X86ISelLowering.cpp

Go to the documentation of this file.

//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that X86 uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//
 
#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>
using namespace llvm;
 
#define DEBUG_TYPE "x86-isel"
 
static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
    "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
    cl::desc(
        "Sets the preferable loop alignment for experiments (as log2 bytes) "
        "for innermost loops only. If specified, this option overrides "
        "alignment set by x86-experimental-pref-loop-alignment."),
    cl::Hidden);
 
static cl::opt<int> BrMergingBaseCostThresh(
    "x86-br-merging-base-cost", cl::init(2),
    cl::desc(
        "Sets the cost threshold for when multiple conditionals will be merged "
        "into one branch versus be split in multiple branches. Merging "
        "conditionals saves branches at the cost of additional instructions. "
        "This value sets the instruction cost limit, below which conditionals "
        "will be merged, and above which conditionals will be split. Set to -1 "
        "to never merge branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingCcmpBias(
    "x86-br-merging-ccmp-bias", cl::init(6),
    cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target "
             "supports conditional compare instructions."),
    cl::Hidden);
 
static cl::opt<bool>
    WidenShift("x86-widen-shift", cl::init(true),
               cl::desc("Replace narrow shifts with wider shifts."),
               cl::Hidden);
 
static cl::opt<int> BrMergingLikelyBias(
    "x86-br-merging-likely-bias", cl::init(0),
    cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely "
             "that all conditionals will be executed. For example for merging "
             "the conditionals (a == b && c > d), if its known that a == b is "
             "likely, then it is likely that if the conditionals are split "
             "both sides will be executed, so it may be desirable to increase "
             "the instruction cost threshold. Set to -1 to never merge likely "
             "branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingUnlikelyBias(
    "x86-br-merging-unlikely-bias", cl::init(-1),
    cl::desc(
        "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely "
        "that all conditionals will be executed. For example for merging "
        "the conditionals (a == b && c > d), if its known that a == b is "
        "unlikely, then it is unlikely that if the conditionals are split "
        "both sides will be executed, so it may be desirable to decrease "
        "the instruction cost threshold. Set to -1 to never merge unlikely "
        "branches."),
    cl::Hidden);
 
static cl::opt<bool> MulConstantOptimization(
    "mul-constant-optimization", cl::init(true),
    cl::desc("Replace 'mul x, Const' with more effective instructions like "
             "SHIFT, LEA, etc."),
    cl::Hidden);
 
X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
                                     const X86Subtarget &STI)
    : TargetLowering(TM, STI), Subtarget(STI) {
  bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
  MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
 
  // Set up the TargetLowering object.
 
  // X86 is weird. It always uses i8 for shift amounts and setcc results.
  setBooleanContents(ZeroOrOneBooleanContent);
  // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
  // X86 instruction cache is coherent with its data cache so we can use the
  // default expansion to a no-op.
  setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Expand);
 
  // For 64-bit, since we have so many registers, use the ILP scheduler.
  // For 32-bit, use the register pressure specific scheduling.
  // For Atom, always use ILP scheduling.
  if (Subtarget.isAtom())
    setSchedulingPreference(Sched::ILP);
  else if (Subtarget.is64Bit())
    setSchedulingPreference(Sched::ILP);
  else
    setSchedulingPreference(Sched::RegPressure);
  const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
  setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
 
  // Bypass expensive divides and use cheaper ones.
  if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
    if (Subtarget.hasSlowDivide32())
      addBypassSlowDiv(32, 8);
    if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
      addBypassSlowDiv(64, 32);
  }
 
  if (Subtarget.canUseCMPXCHG16B())
    setMaxAtomicSizeInBitsSupported(128);
  else if (Subtarget.canUseCMPXCHG8B())
    setMaxAtomicSizeInBitsSupported(64);
  else
    setMaxAtomicSizeInBitsSupported(32);
 
  setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
 
  setMaxLargeFPConvertBitWidthSupported(128);
 
  // Set up the register classes.
  addRegisterClass(MVT::i8, &X86::GR8RegClass);
  addRegisterClass(MVT::i16, &X86::GR16RegClass);
  addRegisterClass(MVT::i32, &X86::GR32RegClass);
  if (Subtarget.is64Bit())
    addRegisterClass(MVT::i64, &X86::GR64RegClass);
 
  for (MVT VT : MVT::integer_valuetypes())
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
 
  // We don't accept any truncstore of integer registers.
  setTruncStoreAction(MVT::i64, MVT::i32, Expand);
  setTruncStoreAction(MVT::i64, MVT::i16, Expand);
  setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
  setTruncStoreAction(MVT::i32, MVT::i16, Expand);
  setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
  setTruncStoreAction(MVT::i16, MVT::i8,  Expand);
 
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
  // SETOEQ and SETUNE require checking two conditions.
  for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
    setCondCodeAction(ISD::SETOEQ, VT, Expand);
    setCondCodeAction(ISD::SETUNE, VT, Expand);
  }
 
  // Integer absolute.
  if (Subtarget.canUseCMOV()) {
    setOperationAction(ISD::ABS            , MVT::i16  , Custom);
    setOperationAction(ISD::ABS            , MVT::i32  , Custom);
    if (Subtarget.is64Bit())
      setOperationAction(ISD::ABS          , MVT::i64  , Custom);
  }
 
  // Absolute difference.
  for (auto Op : {ISD::ABDS, ISD::ABDU}) {
    setOperationAction(Op                  , MVT::i8   , Custom);
    setOperationAction(Op                  , MVT::i16  , Custom);
    setOperationAction(Op                  , MVT::i32  , Custom);
    if (Subtarget.is64Bit())
     setOperationAction(Op                 , MVT::i64  , Custom);
  }
 
  // Signed saturation subtraction.
  setOperationAction(ISD::SSUBSAT          , MVT::i8   , Custom);
  setOperationAction(ISD::SSUBSAT          , MVT::i16  , Custom);
  setOperationAction(ISD::SSUBSAT          , MVT::i32  , Custom);
  if (Subtarget.is64Bit())
    setOperationAction(ISD::SSUBSAT        , MVT::i64  , Custom);
 
  // Funnel shifts.
  for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
    // For slow shld targets we only lower for code size.
    LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
 
    setOperationAction(ShiftOp             , MVT::i8   , Custom);
    setOperationAction(ShiftOp             , MVT::i16  , Custom);
    setOperationAction(ShiftOp             , MVT::i32  , ShiftDoubleAction);
    if (Subtarget.is64Bit())
      setOperationAction(ShiftOp           , MVT::i64  , ShiftDoubleAction);
  }
 
  if (!Subtarget.useSoftFloat()) {
    // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
    // operation.
    setOperationAction(ISD::UINT_TO_FP,        MVT::i8, Promote);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
    setOperationAction(ISD::UINT_TO_FP,        MVT::i16, Promote);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
    // We have an algorithm for SSE2, and we turn this into a 64-bit
    // FILD or VCVTUSI2SS/SD for other targets.
    setOperationAction(ISD::UINT_TO_FP,        MVT::i32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
    // We have an algorithm for SSE2->double, and we turn this into a
    // 64-bit FILD followed by conditional FADD for other targets.
    setOperationAction(ISD::UINT_TO_FP,        MVT::i64, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
 
    // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
    // this operation.
    setOperationAction(ISD::SINT_TO_FP,        MVT::i8, Promote);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
    // SSE has no i16 to fp conversion, only i32. We promote in the handler
    // to allow f80 to use i16 and f64 to use i16 with sse1 only
    setOperationAction(ISD::SINT_TO_FP,        MVT::i16, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
    // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
    setOperationAction(ISD::SINT_TO_FP,        MVT::i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
    // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
    // are Legal, f80 is custom lowered.
    setOperationAction(ISD::SINT_TO_FP,        MVT::i64, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
 
    // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
    // this operation.
    setOperationAction(ISD::FP_TO_SINT,        MVT::i8,  Promote);
    // FIXME: This doesn't generate invalid exception when it should. PR44019.
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8,  Promote);
    setOperationAction(ISD::FP_TO_SINT,        MVT::i16, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
    setOperationAction(ISD::FP_TO_SINT,        MVT::i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
    // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
    // are Legal, f80 is custom lowered.
    setOperationAction(ISD::FP_TO_SINT,        MVT::i64, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
 
    // Handle FP_TO_UINT by promoting the destination to a larger signed
    // conversion.
    setOperationAction(ISD::FP_TO_UINT,        MVT::i8,  Promote);
    // FIXME: This doesn't generate invalid exception when it should. PR44019.
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8,  Promote);
    setOperationAction(ISD::FP_TO_UINT,        MVT::i16, Promote);
    // FIXME: This doesn't generate invalid exception when it should. PR44019.
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
    setOperationAction(ISD::FP_TO_UINT,        MVT::i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT,        MVT::i64, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
 
    setOperationAction(ISD::LRINT,             MVT::f32, Custom);
    setOperationAction(ISD::LRINT,             MVT::f64, Custom);
    setOperationAction(ISD::LLRINT,            MVT::f32, Custom);
    setOperationAction(ISD::LLRINT,            MVT::f64, Custom);
 
    if (!Subtarget.is64Bit() && Subtarget.hasX87()) {
      setOperationAction(ISD::LRINT,  MVT::i64, Custom);
      setOperationAction(ISD::LLRINT, MVT::i64, Custom);
    }
  }
 
  if (Subtarget.hasSSE2()) {
    // Custom lowering for saturating float to int conversions.
    // We handle promotion to larger result types manually.
    for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
      setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
    }
    if (Subtarget.is64Bit()) {
      setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
      setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
    }
  }
  if (Subtarget.hasAVX10_2()) {
    for (MVT VT : {MVT::v8i8, MVT::v16i8, MVT::v32i8}) {
      setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
    }
    setOperationAction(ISD::FP_TO_UINT_SAT, MVT::v2i32, Custom);
    setOperationAction(ISD::FP_TO_SINT_SAT, MVT::v2i32, Custom);
    setOperationAction(ISD::FP_TO_UINT_SAT, MVT::v8i64, Legal);
    setOperationAction(ISD::FP_TO_SINT_SAT, MVT::v8i64, Legal);
    for (MVT VT : {MVT::i32, MVT::v4i32, MVT::v8i32, MVT::v16i32, MVT::v2i64,
                   MVT::v4i64}) {
      setOperationAction(ISD::FP_TO_UINT_SAT, VT, Legal);
      setOperationAction(ISD::FP_TO_SINT_SAT, VT, Legal);
    }
    if (Subtarget.is64Bit()) {
      setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Legal);
      setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Legal);
    }
  }
 
  // Handle address space casts between mixed sized pointers.
  setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
  setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
 
  // TODO: when we have SSE, these could be more efficient, by using movd/movq.
  if (!Subtarget.hasSSE2()) {
    setOperationAction(ISD::BITCAST        , MVT::f32  , Expand);
    setOperationAction(ISD::BITCAST        , MVT::i32  , Expand);
    if (Subtarget.is64Bit()) {
      setOperationAction(ISD::BITCAST      , MVT::f64  , Expand);
      // Without SSE, i64->f64 goes through memory.
      setOperationAction(ISD::BITCAST      , MVT::i64  , Expand);
    }
  } else if (!Subtarget.is64Bit())
    setOperationAction(ISD::BITCAST      , MVT::i64  , Custom);
 
  // Scalar integer divide and remainder are lowered to use operations that
  // produce two results, to match the available instructions. This exposes
  // the two-result form to trivial CSE, which is able to combine x/y and x%y
  // into a single instruction.
  //
  // Scalar integer multiply-high is also lowered to use two-result
  // operations, to match the available instructions. However, plain multiply
  // (low) operations are left as Legal, as there are single-result
  // instructions for this in x86. Using the two-result multiply instructions
  // when both high and low results are needed must be arranged by dagcombine.
  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
    setOperationAction(ISD::MULHS, VT, Expand);
    setOperationAction(ISD::MULHU, VT, Expand);
    setOperationAction(ISD::SDIV, VT, Expand);
    setOperationAction(ISD::UDIV, VT, Expand);
    setOperationAction(ISD::SREM, VT, Expand);
    setOperationAction(ISD::UREM, VT, Expand);
  }
 
  setOperationAction(ISD::BR_JT            , MVT::Other, Expand);
  setOperationAction(ISD::BRCOND           , MVT::Other, Custom);
  for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
                   MVT::i8,  MVT::i16, MVT::i32, MVT::i64 }) {
    setOperationAction(ISD::BR_CC,     VT, Expand);
    setOperationAction(ISD::SELECT_CC, VT, Expand);
  }
  if (Subtarget.is64Bit())
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
 
  setOperationAction(ISD::FREM, MVT::f32, LibCall);
  setOperationAction(ISD::FREM, MVT::f64, LibCall);
  setOperationAction(ISD::FREM, MVT::f80, LibCall);
  setOperationAction(ISD::FREM, MVT::f128, LibCall);
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
    setOperationAction(ISD::GET_ROUNDING   , MVT::i32  , Custom);
    setOperationAction(ISD::SET_ROUNDING   , MVT::Other, Custom);
    setOperationAction(ISD::GET_FPENV_MEM  , MVT::Other, Custom);
    setOperationAction(ISD::SET_FPENV_MEM  , MVT::Other, Custom);
    setOperationAction(ISD::RESET_FPENV    , MVT::Other, Custom);
  }
 
  // Promote the i8 variants and force them on up to i32 which has a shorter
  // encoding.
  setOperationPromotedToType(ISD::CTTZ, MVT::i8, MVT::i32);
  setOperationPromotedToType(ISD::CTTZ_ZERO_POISON, MVT::i8, MVT::i32);
  // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
  // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
  // promote that too.
  setOperationPromotedToType(ISD::CTTZ, MVT::i16, MVT::i32);
  setOperationPromotedToType(ISD::CTTZ_ZERO_POISON, MVT::i16, MVT::i32);
 
  if (!Subtarget.hasBMI()) {
    setOperationAction(ISD::CTTZ, MVT::i32, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::i32, Legal);
    if (Subtarget.is64Bit()) {
      setOperationAction(ISD::CTTZ, MVT::i64, Custom);
      setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::i64, Legal);
    }
  }
 
  if (Subtarget.hasLZCNT()) {
    // When promoting the i8 variants, force them to i32 for a shorter
    // encoding.
    setOperationPromotedToType(ISD::CTLZ, MVT::i8, MVT::i32);
    setOperationPromotedToType(ISD::CTLZ_ZERO_POISON, MVT::i8, MVT::i32);
  } else {
    for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
      if (VT == MVT::i64 && !Subtarget.is64Bit())
        continue;
      setOperationAction(ISD::CTLZ, VT, Custom);
      setOperationAction(ISD::CTLZ_ZERO_POISON, VT, Custom);
    }
  }
 
  for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
                  ISD::STRICT_FP_TO_FP16}) {
    // Special handling for half-precision floating point conversions.
    // If we don't have F16C support, then lower half float conversions
    // into library calls.
    setOperationAction(
        Op, MVT::f32,
        (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
    // There's never any support for operations beyond MVT::f32.
    setOperationAction(Op, MVT::f64, Expand);
    setOperationAction(Op, MVT::f80, Expand);
    setOperationAction(Op, MVT::f128, Expand);
  }
 
  for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
    setOperationAction(ISD::STRICT_FP_TO_BF16, VT, Expand);
    setOperationAction(ISD::STRICT_BF16_TO_FP, VT, Expand);
  }
 
  for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
    setTruncStoreAction(VT, MVT::f16, Expand);
    setTruncStoreAction(VT, MVT::bf16, Expand);
 
    setOperationAction(ISD::BF16_TO_FP, VT, Expand);
    setOperationAction(ISD::FP_TO_BF16, VT, Custom);
  }
 
  setOperationAction(ISD::PARITY, MVT::i8, Custom);
  setOperationAction(ISD::PARITY, MVT::i16, Custom);
  setOperationAction(ISD::PARITY, MVT::i32, Custom);
  if (Subtarget.is64Bit())
    setOperationAction(ISD::PARITY, MVT::i64, Custom);
  if (Subtarget.hasPOPCNT()) {
    setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
    // popcntw is longer to encode than popcntl and also has a false dependency
    // on the dest that popcntl hasn't had since Cannon Lake.
    setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
  } else {
    setOperationAction(ISD::CTPOP          , MVT::i8   , Custom);
    setOperationAction(ISD::CTPOP          , MVT::i16  , Custom);
    setOperationAction(ISD::CTPOP          , MVT::i32  , Custom);
    setOperationAction(ISD::CTPOP          , MVT::i64  , Custom);
  }
 
  if (Subtarget.hasBMI2()) {
    setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i8, Promote);
    setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i16, Promote);
    setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i32, Legal);
    if (Subtarget.is64Bit())
      setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i64, Legal);
  }
 
  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
 
  if (!Subtarget.hasMOVBE())
    setOperationAction(ISD::BSWAP          , MVT::i16  , Expand);
 
  // X86 wants to expand cmov itself.
  for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
    setOperationAction(ISD::SELECT, VT, Custom);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
  }
  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
    if (VT == MVT::i64 && !Subtarget.is64Bit())
      continue;
    setOperationAction(ISD::SELECT, VT, Custom);
    setOperationAction(ISD::SETCC,  VT, Custom);
  }
 
  setOperationAction(ISD::COND_LOOP, MVT::Other, Custom);
 
  // Custom action for SELECT MMX and expand action for SELECT_CC MMX
  setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
 
  setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);
  // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
  // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
  setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
 
  // Darwin ABI issue.
  for (auto VT : { MVT::i32, MVT::i64 }) {
    if (VT == MVT::i64 && !Subtarget.is64Bit())
      continue;
    setOperationAction(ISD::ConstantPool    , VT, Custom);
    setOperationAction(ISD::JumpTable       , VT, Custom);
    setOperationAction(ISD::GlobalAddress   , VT, Custom);
    setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
    setOperationAction(ISD::ExternalSymbol  , VT, Custom);
    setOperationAction(ISD::BlockAddress    , VT, Custom);
  }
 
  // 64-bit shl, sra, srl (iff 32-bit x86)
  for (auto VT : { MVT::i32, MVT::i64 }) {
    if (VT == MVT::i64 && !Subtarget.is64Bit())
      continue;
    setOperationAction(ISD::SHL_PARTS, VT, Custom);
    setOperationAction(ISD::SRA_PARTS, VT, Custom);
    setOperationAction(ISD::SRL_PARTS, VT, Custom);
  }
 
  if (Subtarget.hasSSEPrefetch())
    setOperationAction(ISD::PREFETCH      , MVT::Other, Custom);
 
  setOperationAction(ISD::ATOMIC_FENCE  , MVT::Other, Custom);
 
  // Expand certain atomics
  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
    setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
    setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
  }
 
  if (!Subtarget.is64Bit())
    setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
 
  if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
    // All CPUs supporting AVX will atomically load/store aligned 128-bit
    // values, so we can emit [V]MOVAPS/[V]MOVDQA.
    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
  }
 
  if (Subtarget.canUseCMPXCHG16B())
    setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
 
  // 32-bit Windows non-GNU EH (MSVC/Itanium SEH) does not use per-invoke EH
  // labels, so expand them away. SjLj EH does use them.
  if (Subtarget.isTargetWin32() && !Subtarget.isTargetCygMing() &&
      TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
    setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
  }
 
  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
 
  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
  if (Subtarget.isTargetPS())
    setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
  else
    setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
  // VASTART needs to be custom lowered to use the VarArgsFrameIndex
  setOperationAction(ISD::VASTART           , MVT::Other, Custom);
  setOperationAction(ISD::VAEND             , MVT::Other, Expand);
  bool Is64Bit = Subtarget.is64Bit();
  setOperationAction(ISD::VAARG,  MVT::Other, Is64Bit ? Custom : Expand);
  setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
 
  setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
 
  setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
 
  // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
  setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
  setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
 
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
 
  auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
    setOperationAction(ISD::FABS, VT, Action);
    setOperationAction(ISD::FNEG, VT, Action);
    setOperationAction(ISD::FCOPYSIGN, VT, Expand);
    setOperationAction(ISD::FREM, VT, Action);
    setOperationAction(ISD::FMA, VT, Action);
    setOperationAction(ISD::FMINNUM, VT, Action);
    setOperationAction(ISD::FMAXNUM, VT, Action);
    setOperationAction(ISD::FMINIMUM, VT, Action);
    setOperationAction(ISD::FMAXIMUM, VT, Action);
    setOperationAction(ISD::FMINIMUMNUM, VT, Action);
    setOperationAction(ISD::FMAXIMUMNUM, VT, Action);
    setOperationAction(ISD::FSIN, VT, Action);
    setOperationAction(ISD::FCOS, VT, Action);
    setOperationAction(ISD::FSINCOS, VT, Action);
    setOperationAction(ISD::FTAN, VT, Action);
    setOperationAction(ISD::FSQRT, VT, Action);
    setOperationAction(ISD::FPOW, VT, Action);
    setOperationAction(ISD::FPOWI, VT, Action);
    setOperationAction(ISD::FLOG, VT, Action);
    setOperationAction(ISD::FLOG2, VT, Action);
    setOperationAction(ISD::FLOG10, VT, Action);
    setOperationAction(ISD::FEXP, VT, Action);
    setOperationAction(ISD::FEXP2, VT, Action);
    setOperationAction(ISD::FEXP10, VT, Action);
    setOperationAction(ISD::FCEIL, VT, Action);
    setOperationAction(ISD::FFLOOR, VT, Action);
    setOperationAction(ISD::FNEARBYINT, VT, Action);
    setOperationAction(ISD::FRINT, VT, Action);
    setOperationAction(ISD::BR_CC, VT, Action);
    setOperationAction(ISD::SETCC, VT, Action);
    setOperationAction(ISD::SELECT, VT, Custom);
    setOperationAction(ISD::SELECT_CC, VT, Action);
    setOperationAction(ISD::FROUND, VT, Action);
    setOperationAction(ISD::FROUNDEVEN, VT, Action);
    setOperationAction(ISD::FTRUNC, VT, Action);
    setOperationAction(ISD::FLDEXP, VT, Action);
    setOperationAction(ISD::FFREXP, VT, Action);
    setOperationAction(ISD::FSINCOSPI, VT, Action);
  };
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
    // f16, f32 and f64 use SSE.
    // Set up the FP register classes.
    addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
                                                     : &X86::FR16RegClass);
    addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
                                                     : &X86::FR32RegClass);
    addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
                                                     : &X86::FR64RegClass);
 
    // Disable f32->f64 extload as we can only generate this in one instruction
    // under optsize. So its easier to pattern match (fpext (load)) for that
    // case instead of needing to emit 2 instructions for extload in the
    // non-optsize case.
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
 
    for (auto VT : { MVT::f32, MVT::f64 }) {
      // Use ANDPD to simulate FABS.
      setOperationAction(ISD::FABS, VT, Custom);
 
      // Use XORP to simulate FNEG.
      setOperationAction(ISD::FNEG, VT, Custom);
 
      // Use ANDPD and ORPD to simulate FCOPYSIGN.
      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
 
      // These might be better off as horizontal vector ops.
      setOperationAction(ISD::FADD, VT, Custom);
      setOperationAction(ISD::FSUB, VT, Custom);
 
      // We don't support sin/cos/fmod
      setOperationAction(ISD::FSIN   , VT, Expand);
      setOperationAction(ISD::FCOS   , VT, Expand);
      setOperationAction(ISD::FSINCOS, VT, Expand);
    }
 
    // Half type will be promoted by default.
    setF16Action(MVT::f16, Promote);
    setOperationAction(ISD::FADD, MVT::f16, Promote);
    setOperationAction(ISD::FSUB, MVT::f16, Promote);
    setOperationAction(ISD::FMUL, MVT::f16, Promote);
    setOperationAction(ISD::FDIV, MVT::f16, Promote);
    setOperationAction(ISD::FABS, MVT::f16, Custom);
    setOperationAction(ISD::FNEG, MVT::f16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
    setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
    setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
    setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
 
    setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
 
    setOperationAction(ISD::LLROUND, MVT::f16, Expand);
    setOperationAction(ISD::LROUND, MVT::f16, Expand);
    setOperationAction(ISD::LRINT, MVT::f16, Expand);
    setOperationAction(ISD::LLRINT, MVT::f16, Expand);
 
    setOperationAction(ISD::STRICT_LLROUND, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_LROUND, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_LRINT, MVT::f16, Promote);
    setOperationAction(ISD::STRICT_LLRINT, MVT::f16, Promote);
 
    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
 
    // Lower this to MOVMSK plus an AND.
    setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
    setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
 
  } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
             (UseX87 || Is64Bit)) {
    // Use SSE for f32, x87 for f64.
    // Set up the FP register classes.
    addRegisterClass(MVT::f32, &X86::FR32RegClass);
    if (UseX87)
      addRegisterClass(MVT::f64, &X86::RFP64RegClass);
 
    // Use ANDPS to simulate FABS.
    setOperationAction(ISD::FABS , MVT::f32, Custom);
 
    // Use XORP to simulate FNEG.
    setOperationAction(ISD::FNEG , MVT::f32, Custom);
 
    if (UseX87)
      setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f64, Expand);
 
    // Use ANDPS and ORPS to simulate FCOPYSIGN.
    if (UseX87)
      setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
 
    // We don't support sin/cos/fmod
    setOperationAction(ISD::FSIN   , MVT::f32, Expand);
    setOperationAction(ISD::FCOS   , MVT::f32, Expand);
    setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
 
    if (UseX87) {
      // Always expand sin/cos functions even though x87 has an instruction.
      setOperationAction(ISD::FSIN, MVT::f64, Expand);
      setOperationAction(ISD::FCOS, MVT::f64, Expand);
      setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
    }
  } else if (UseX87) {
    // f32 and f64 in x87.
    // Set up the FP register classes.
    addRegisterClass(MVT::f64, &X86::RFP64RegClass);
    addRegisterClass(MVT::f32, &X86::RFP32RegClass);
 
    for (auto VT : { MVT::f32, MVT::f64 }) {
      setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Expand);
      setOperationAction(ISD::FCOPYSIGN, VT, Expand);
 
      // Always expand sin/cos functions even though x87 has an instruction.
      setOperationAction(ISD::FSIN   , VT, Expand);
      setOperationAction(ISD::FCOS   , VT, Expand);
      setOperationAction(ISD::FSINCOS, VT, Expand);
    }
  }
 
  // Expand FP32 immediates into loads from the stack, save special cases.
  if (isTypeLegal(MVT::f32)) {
    if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
      addLegalFPImmediate(APFloat(+0.0f)); // FLD0
      addLegalFPImmediate(APFloat(+1.0f)); // FLD1
      addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
      addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
    } else // SSE immediates.
      addLegalFPImmediate(APFloat(+0.0f)); // xorps
  }
  // Expand FP64 immediates into loads from the stack, save special cases.
  if (isTypeLegal(MVT::f64)) {
    if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
      addLegalFPImmediate(APFloat(+0.0)); // FLD0
      addLegalFPImmediate(APFloat(+1.0)); // FLD1
      addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
      addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
    } else // SSE immediates.
      addLegalFPImmediate(APFloat(+0.0)); // xorpd
  }
  // Support fp16 0 immediate.
  if (isTypeLegal(MVT::f16))
    addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
 
  // Handle constrained floating-point operations of scalar.
  setOperationAction(ISD::STRICT_FADD,      MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FADD,      MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FSUB,      MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FSUB,      MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FMUL,      MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FMUL,      MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FDIV,      MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FDIV,      MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FSQRT,     MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FSQRT,     MVT::f64, Legal);
 
  // We don't support FMA.
  setOperationAction(ISD::FMA, MVT::f64, Expand);
  setOperationAction(ISD::FMA, MVT::f32, Expand);
 
  // f80 always uses X87.
  if (UseX87) {
    addRegisterClass(MVT::f80, &X86::RFP80RegClass);
    setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f80, Expand);
    setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
    {
      APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
      addLegalFPImmediate(TmpFlt);  // FLD0
      TmpFlt.changeSign();
      addLegalFPImmediate(TmpFlt);  // FLD0/FCHS
 
      bool ignored;
      APFloat TmpFlt2(+1.0);
      TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
                      &ignored);
      addLegalFPImmediate(TmpFlt2);  // FLD1
      TmpFlt2.changeSign();
      addLegalFPImmediate(TmpFlt2);  // FLD1/FCHS
    }
 
    // Always expand sin/cos functions even though x87 has an instruction.
    // clang-format off
    setOperationAction(ISD::FSIN   , MVT::f80, Expand);
    setOperationAction(ISD::FCOS   , MVT::f80, Expand);
    setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
    setOperationAction(ISD::FTAN   , MVT::f80, Expand);
    setOperationAction(ISD::FASIN  , MVT::f80, Expand);
    setOperationAction(ISD::FACOS  , MVT::f80, Expand);
    setOperationAction(ISD::FATAN  , MVT::f80, Expand);
    setOperationAction(ISD::FATAN2 , MVT::f80, Expand);
    setOperationAction(ISD::FSINH  , MVT::f80, Expand);
    setOperationAction(ISD::FCOSH  , MVT::f80, Expand);
    setOperationAction(ISD::FTANH  , MVT::f80, Expand);
    // clang-format on
 
    setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
    setOperationAction(ISD::FCEIL,  MVT::f80, Expand);
    setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
    setOperationAction(ISD::FRINT,  MVT::f80, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
    setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
    setOperationAction(ISD::FMA, MVT::f80, Expand);
    setOperationAction(ISD::LROUND, MVT::f80, LibCall);
    setOperationAction(ISD::LLROUND, MVT::f80, LibCall);
    setOperationAction(ISD::LRINT, MVT::f80, Custom);
    setOperationAction(ISD::LLRINT, MVT::f80, Custom);
 
    // Handle constrained floating-point operations of scalar.
    setOperationAction(ISD::STRICT_FADD     , MVT::f80, Legal);
    setOperationAction(ISD::STRICT_FSUB     , MVT::f80, Legal);
    setOperationAction(ISD::STRICT_FMUL     , MVT::f80, Legal);
    setOperationAction(ISD::STRICT_FDIV     , MVT::f80, Legal);
    setOperationAction(ISD::STRICT_FSQRT    , MVT::f80, Legal);
    setOperationAction(ISD::FCANONICALIZE, MVT::f80, Expand);
    if (isTypeLegal(MVT::f16)) {
      setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
    } else {
      setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
    }
    // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
    // as Custom.
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
  }
 
  // f128 uses xmm registers, but most operations require libcalls.
  if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
    addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                   : &X86::VR128RegClass);
 
    addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
 
    setOperationAction(ISD::FADD,        MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
    setOperationAction(ISD::FSUB,        MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
    setOperationAction(ISD::FDIV,        MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
    setOperationAction(ISD::FMUL,        MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
    setOperationAction(ISD::FMA,         MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FMA,  MVT::f128, LibCall);
 
    setOperationAction(ISD::FABS, MVT::f128, Custom);
    setOperationAction(ISD::FNEG, MVT::f128, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
 
    // clang-format off
    setOperationAction(ISD::FSIN,         MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FSIN,  MVT::f128, LibCall);
    setOperationAction(ISD::FCOS,         MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FCOS,  MVT::f128, LibCall);
    setOperationAction(ISD::FSINCOS,      MVT::f128, LibCall);
    setOperationAction(ISD::FTAN,         MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FTAN,  MVT::f128, LibCall);
    // clang-format on
    // No STRICT_FSINCOS
    setOperationAction(ISD::FSQRT,        MVT::f128, LibCall);
    setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
 
    setOperationAction(ISD::FP_EXTEND,        MVT::f128, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
    // We need to custom handle any FP_ROUND with an f128 input, but
    // LegalizeDAG uses the result type to know when to run a custom handler.
    // So we have to list all legal floating point result types here.
    if (isTypeLegal(MVT::f32)) {
      setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
    }
    if (isTypeLegal(MVT::f64)) {
      setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
    }
    if (isTypeLegal(MVT::f80)) {
      setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
      setOperationAction(ISD::FCANONICALIZE, MVT::f80, Expand);
    }
 
    setOperationAction(ISD::SETCC, MVT::f128, Custom);
 
    setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
    setTruncStoreAction(MVT::f128, MVT::f32, Expand);
    setTruncStoreAction(MVT::f128, MVT::f64, Expand);
    setTruncStoreAction(MVT::f128, MVT::f80, Expand);
  }
 
  // Always use a library call for pow.
  setOperationAction(ISD::FPOW             , MVT::f32  , Expand);
  setOperationAction(ISD::FPOW             , MVT::f64  , Expand);
  setOperationAction(ISD::FPOW             , MVT::f80  , Expand);
  setOperationAction(ISD::FPOW             , MVT::f128 , Expand);
 
  setOperationAction(ISD::FLOG, MVT::f80, Expand);
  setOperationAction(ISD::FLOG2, MVT::f80, Expand);
  setOperationAction(ISD::FLOG10, MVT::f80, Expand);
  setOperationAction(ISD::FEXP, MVT::f80, Expand);
  setOperationAction(ISD::FEXP2, MVT::f80, Expand);
  setOperationAction(ISD::FEXP10, MVT::f80, Expand);
  setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
  setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
 
  // Some FP actions are always expanded for vector types.
  for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
                   MVT::v4f32, MVT::v8f32,  MVT::v16f32,
                   MVT::v2f64, MVT::v4f64,  MVT::v8f64 }) {
    // clang-format off
    setOperationAction(ISD::FSIN,      VT, Expand);
    setOperationAction(ISD::FSINCOS,   VT, Expand);
    setOperationAction(ISD::FCOS,      VT, Expand);
    setOperationAction(ISD::FTAN,      VT, Expand);
    setOperationAction(ISD::FREM,      VT, Expand);
    setOperationAction(ISD::FCOPYSIGN, VT, Expand);
    setOperationAction(ISD::FPOW,      VT, Expand);
    setOperationAction(ISD::FLOG,      VT, Expand);
    setOperationAction(ISD::FLOG2,     VT, Expand);
    setOperationAction(ISD::FLOG10,    VT, Expand);
    setOperationAction(ISD::FEXP,      VT, Expand);
    setOperationAction(ISD::FEXP2,     VT, Expand);
    setOperationAction(ISD::FEXP10,    VT, Expand);
    // clang-format on
  }
 
  // First set operation action for all vector types to either promote
  // (for widening) or expand (for scalarization). Then we will selectively
  // turn on ones that can be effectively codegen'd.
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::SDIV, VT, Expand);
    setOperationAction(ISD::UDIV, VT, Expand);
    setOperationAction(ISD::SREM, VT, Expand);
    setOperationAction(ISD::UREM, VT, Expand);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
    setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
    setOperationAction(ISD::FMA,  VT, Expand);
    setOperationAction(ISD::FFLOOR, VT, Expand);
    setOperationAction(ISD::FCEIL, VT, Expand);
    setOperationAction(ISD::FTRUNC, VT, Expand);
    setOperationAction(ISD::FRINT, VT, Expand);
    setOperationAction(ISD::FNEARBYINT, VT, Expand);
    setOperationAction(ISD::FROUNDEVEN, VT, Expand);
    setOperationAction(ISD::SMUL_LOHI, VT, Expand);
    setOperationAction(ISD::MULHS, VT, Expand);
    setOperationAction(ISD::UMUL_LOHI, VT, Expand);
    setOperationAction(ISD::MULHU, VT, Expand);
    setOperationAction(ISD::SDIVREM, VT, Expand);
    setOperationAction(ISD::UDIVREM, VT, Expand);
    setOperationAction(ISD::CTPOP, VT, Expand);
    setOperationAction(ISD::CTTZ, VT, Expand);
    setOperationAction(ISD::CTLZ, VT, Expand);
    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);
    setOperationAction(ISD::BSWAP, VT, Expand);
    setOperationAction(ISD::SETCC, VT, Expand);
    setOperationAction(ISD::FP_TO_UINT, VT, Expand);
    setOperationAction(ISD::FP_TO_SINT, VT, Expand);
    setOperationAction(ISD::UINT_TO_FP, VT, Expand);
    setOperationAction(ISD::SINT_TO_FP, VT, Expand);
    setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
    setOperationAction(ISD::TRUNCATE, VT, Expand);
    setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
    setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
    setOperationAction(ISD::ANY_EXTEND, VT, Expand);
    setOperationAction(ISD::SELECT_CC, VT, Expand);
    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
      setTruncStoreAction(InnerVT, VT, Expand);
 
      setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
      setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
 
      // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
      // types, we have to deal with them whether we ask for Expansion or not.
      // Setting Expand causes its own optimisation problems though, so leave
      // them legal.
      if (VT.getVectorElementType() == MVT::i1)
        setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
 
      // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
      // split/scalarized right now.
      if (VT.getVectorElementType() == MVT::f16 ||
          VT.getVectorElementType() == MVT::bf16)
        setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
    }
  }
 
  // FIXME: In order to prevent SSE instructions being expanded to MMX ones
  // with -msoft-float, disable use of MMX as well.
  if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
    addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
    // No operations on x86mmx supported, everything uses intrinsics.
  }
 
  auto SetFPMinMaxAction = [&](MVT VT) {
    setOperationAction(ISD::PSEUDO_FMIN, VT, Legal);
    setOperationAction(ISD::PSEUDO_FMAX, VT, Legal);
    setOperationAction(ISD::STRICT_PSEUDO_FMIN, VT, Legal);
    setOperationAction(ISD::STRICT_PSEUDO_FMAX, VT, Legal);
    setOperationAction(ISD::FMAXIMUM, VT, Custom);
    setOperationAction(ISD::FMINIMUM, VT, Custom);
    setOperationAction(ISD::FMAXIMUMNUM, VT, Custom);
    setOperationAction(ISD::FMINIMUMNUM, VT, Custom);
  };
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
    addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
 
    SetFPMinMaxAction(MVT::f32);
 
    setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
    setOperationAction(ISD::FABS,               MVT::v4f32, Custom);
    setOperationAction(ISD::FCOPYSIGN,          MVT::v4f32, Custom);
    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);
    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
    setOperationAction(ISD::VSELECT,            MVT::v4f32, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
    setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);
    setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
 
    setOperationAction(ISD::LOAD,               MVT::v2f32, Custom);
    setOperationAction(ISD::STORE,              MVT::v2f32, Custom);
    setOperationAction(ISD::FCANONICALIZE, MVT::v2f32, Expand);
 
    setOperationAction(ISD::STRICT_FADD,        MVT::v4f32, Legal);
    setOperationAction(ISD::STRICT_FSUB,        MVT::v4f32, Legal);
    setOperationAction(ISD::STRICT_FMUL,        MVT::v4f32, Legal);
    setOperationAction(ISD::STRICT_FDIV,        MVT::v4f32, Legal);
    setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f32, Legal);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
    addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
 
    // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
    // registers cannot be used even for integer operations.
    addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
    addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
    addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
    addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
    addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
                                                    : &X86::VR128RegClass);
 
    for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 })
      SetFPMinMaxAction(VT);
 
    setOperationAction(ISD::MUL,                MVT::v2i8,  Custom);
    setOperationAction(ISD::MUL,                MVT::v4i8,  Custom);
    setOperationAction(ISD::MUL,                MVT::v8i8,  Custom);
 
    setOperationAction(ISD::MUL,                MVT::v16i8, Custom);
    setOperationAction(ISD::MUL,                MVT::v4i32, Custom);
    setOperationAction(ISD::MUL,                MVT::v2i64, Custom);
    setOperationAction(ISD::MULHU,              MVT::v4i32, Custom);
    setOperationAction(ISD::MULHS,              MVT::v4i32, Custom);
    setOperationAction(ISD::MULHU,              MVT::v16i8, Custom);
    setOperationAction(ISD::MULHS,              MVT::v16i8, Custom);
    setOperationAction(ISD::MULHU,              MVT::v8i16, Legal);
    setOperationAction(ISD::MULHS,              MVT::v8i16, Legal);
    setOperationAction(ISD::MUL,                MVT::v8i16, Legal);
    setOperationAction(ISD::AVGCEILU,           MVT::v16i8, Legal);
    setOperationAction(ISD::AVGCEILU,           MVT::v8i16, Legal);
 
    setOperationAction(ISD::SMULO,              MVT::v16i8, Custom);
    setOperationAction(ISD::UMULO,              MVT::v16i8, Custom);
    setOperationAction(ISD::UMULO,              MVT::v2i32, Custom);
 
    setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
    setOperationAction(ISD::FCANONICALIZE, MVT::v2f64, Expand);
    setOperationAction(ISD::FABS,               MVT::v2f64, Custom);
    setOperationAction(ISD::FCOPYSIGN,          MVT::v2f64, Custom);
 
    setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
    setOperationAction(ISD::LRINT, MVT::v2i32, Custom);
 
    setOperationAction(ISD::AND, MVT::i128, Custom);
    setOperationAction(ISD::OR, MVT::i128, Custom);
    setOperationAction(ISD::XOR, MVT::i128, Custom);
    setOperationAction(ISD::SELECT, MVT::i128, Custom);
 
    if (Subtarget.hasPCLMUL()) {
      for (auto VT : {MVT::i64, MVT::v4i32, MVT::v2i64}) {
        setOperationAction(ISD::CLMUL, VT, Custom);
        setOperationAction(ISD::CLMULH, VT, Custom);
      }
      setOperationAction(ISD::CLMUL, MVT::i32, Custom);
      setOperationAction(ISD::CLMUL, MVT::i16, Custom);
      setOperationAction(ISD::CLMUL, MVT::i8, Custom);
    }
 
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
      setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
      setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
      setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
    }
 
    // SSE2 can use basic vector unrolling.
    // SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
    // Fallback to ReplaceNodeResults for vXi64 reductions on 32-bit targets.
    for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::i64}) {
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_MUL,  VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
    }
 
    setOperationAction(ISD::UADDSAT,            MVT::v16i8, Legal);
    setOperationAction(ISD::SADDSAT,            MVT::v16i8, Legal);
    setOperationAction(ISD::USUBSAT,            MVT::v16i8, Legal);
    setOperationAction(ISD::SSUBSAT,            MVT::v16i8, Legal);
    setOperationAction(ISD::UADDSAT,            MVT::v8i16, Legal);
    setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);
    setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);
    setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);
    setOperationAction(ISD::USUBSAT,            MVT::v4i32, Custom);
    setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i8, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
 
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction(ISD::CTPOP, VT, Custom);
      setOperationAction(ISD::ABS, VT, Custom);
      setOperationAction(ISD::ABDS, VT, Custom);
      setOperationAction(ISD::ABDU, VT, Custom);
 
      // The condition codes aren't legal in SSE/AVX and under AVX512 we use
      // setcc all the way to isel and prefer SETGT in some isel patterns.
      setCondCodeAction(ISD::SETLT, VT, Custom);
      setCondCodeAction(ISD::SETLE, VT, Custom);
    }
 
    setOperationAction(ISD::SETCC,          MVT::v2f64, Custom);
    setOperationAction(ISD::SETCC,          MVT::v4f32, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v2f64, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f32, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
 
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
      setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
      setOperationAction(ISD::VSELECT,            VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    }
 
    for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
      setOperationAction(ISD::VSELECT,            VT, Custom);
 
      if (VT == MVT::v2i64 && !Subtarget.is64Bit())
        continue;
 
      setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    }
    setF16Action(MVT::v8f16, Expand);
    setOperationAction(ISD::FADD, MVT::v8f16, Expand);
    setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
    setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
    setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
    setOperationAction(ISD::FNEG, MVT::v8f16, Custom);
    setOperationAction(ISD::FABS, MVT::v8f16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
 
    // Custom lower v2i64 and v2f64 selects.
    setOperationAction(ISD::SELECT,             MVT::v2f64, Custom);
    setOperationAction(ISD::SELECT,             MVT::v2i64, Custom);
    setOperationAction(ISD::SELECT,             MVT::v4i32, Custom);
    setOperationAction(ISD::SELECT,             MVT::v8i16, Custom);
    setOperationAction(ISD::SELECT,             MVT::v8f16, Custom);
    setOperationAction(ISD::SELECT,             MVT::v16i8, Custom);
 
    setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Custom);
    setOperationAction(ISD::FP_TO_UINT,         MVT::v4i32, Custom);
    setOperationAction(ISD::FP_TO_SINT,         MVT::v2i32, Custom);
    setOperationAction(ISD::FP_TO_UINT,         MVT::v2i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v4i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v2i32, Custom);
 
    // Custom legalize these to avoid over promotion or custom promotion.
    for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
      setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
      setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
    }
 
    setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v4i32, Custom);
    setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2i32, Custom);
 
    setOperationAction(ISD::UINT_TO_FP,         MVT::v2i32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2i32, Custom);
 
    setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v4i32, Custom);
 
    // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
    setOperationAction(ISD::SINT_TO_FP,         MVT::v2f32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2f32, Custom);
    setOperationAction(ISD::UINT_TO_FP,         MVT::v2f32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2f32, Custom);
 
    setOperationAction(ISD::FP_EXTEND,          MVT::v2f32, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v2f32, Custom);
    setOperationAction(ISD::FP_ROUND,           MVT::v2f32, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v2f32, Custom);
 
    // We want to legalize this to an f64 load rather than an i64 load on
    // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
    // store.
    setOperationAction(ISD::LOAD,               MVT::v2i32, Custom);
    setOperationAction(ISD::LOAD,               MVT::v4i16, Custom);
    setOperationAction(ISD::LOAD,               MVT::v8i8,  Custom);
    setOperationAction(ISD::STORE,              MVT::v2i32, Custom);
    setOperationAction(ISD::STORE,              MVT::v4i16, Custom);
    setOperationAction(ISD::STORE,              MVT::v8i8,  Custom);
 
    // Add 32-bit vector stores to help vectorization opportunities.
    setOperationAction(ISD::STORE,              MVT::v2i16, Custom);
    setOperationAction(ISD::STORE,              MVT::v4i8,  Custom);
 
    setOperationAction(ISD::BITCAST,            MVT::v2i32, Custom);
    setOperationAction(ISD::BITCAST,            MVT::v4i16, Custom);
    setOperationAction(ISD::BITCAST,            MVT::v8i8,  Custom);
    if (!Subtarget.hasAVX512())
      setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
 
    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
    setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
 
    setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
 
    setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v2i64, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v4i32, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v4i64, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v8i16, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v8i32, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v8i64, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v16i8, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v16i32, Custom);
    setOperationAction(ISD::TRUNCATE,    MVT::v16i64, Custom);
 
    // In the customized shift lowering, the legal v4i32/v2i64 cases
    // in AVX2 will be recognized.
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SRL,              VT, Custom);
      setOperationAction(ISD::SHL,              VT, Custom);
      setOperationAction(ISD::SRA,              VT, Custom);
      if (VT == MVT::v2i64) continue;
      setOperationAction(ISD::ROTL,             VT, Custom);
      setOperationAction(ISD::ROTR,             VT, Custom);
      setOperationAction(ISD::FSHL,             VT, Custom);
      setOperationAction(ISD::FSHR,             VT, Custom);
    }
 
    setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);
    setOperationAction(ISD::STRICT_FADD,        MVT::v2f64, Legal);
    setOperationAction(ISD::STRICT_FSUB,        MVT::v2f64, Legal);
    setOperationAction(ISD::STRICT_FMUL,        MVT::v2f64, Legal);
    setOperationAction(ISD::STRICT_FDIV,        MVT::v2f64, Legal);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasGFNI()) {
    setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::i64, Custom);
 
    for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
      setOperationAction(ISD::BITREVERSE, VT, Custom);
    }
 
    setOperationAction(ISD::CTLZ, MVT::v16i8, Custom);
    setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
    setOperationAction(ISD::ABS,                MVT::v16i8, Legal);
    setOperationAction(ISD::ABS,                MVT::v8i16, Legal);
    setOperationAction(ISD::ABS,                MVT::v4i32, Legal);
 
    for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
      setOperationAction(ISD::BITREVERSE,       VT, Custom);
      setOperationAction(ISD::CTLZ,             VT, Custom);
    }
    setOperationAction(ISD::BITREVERSE,        MVT::i128, Custom);
 
    // These might be better off as horizontal vector ops.
    setOperationAction(ISD::ADD,                MVT::i16, Custom);
    setOperationAction(ISD::ADD,                MVT::i32, Custom);
    setOperationAction(ISD::SUB,                MVT::i16, Custom);
    setOperationAction(ISD::SUB,                MVT::i32, Custom);
  }
  if (Subtarget.hasNDD()) {
    // Enable custom lowering for scalar USUBSAT to optimize usub.sat(X,1)
    // with cmp+adc when NDD is available.
    setOperationAction(ISD::USUBSAT, MVT::i8, Custom);
    setOperationAction(ISD::USUBSAT, MVT::i16, Custom);
    setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
    setOperationAction(ISD::USUBSAT, MVT::i64, Custom);
  }
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
    for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
      setOperationAction(ISD::FFLOOR,            RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FFLOOR,     RoundedTy,  Legal);
      setOperationAction(ISD::FCEIL,             RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FCEIL,      RoundedTy,  Legal);
      setOperationAction(ISD::FTRUNC,            RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FTRUNC,     RoundedTy,  Legal);
      setOperationAction(ISD::FRINT,             RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FRINT,      RoundedTy,  Legal);
      setOperationAction(ISD::FNEARBYINT,        RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy,  Legal);
      setOperationAction(ISD::FROUNDEVEN,        RoundedTy,  Legal);
      setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy,  Legal);
 
      setOperationAction(ISD::FROUND,            RoundedTy,  Custom);
    }
 
    setOperationAction(ISD::SMAX,               MVT::v16i8, Legal);
    setOperationAction(ISD::SMAX,               MVT::v4i32, Legal);
    setOperationAction(ISD::UMAX,               MVT::v8i16, Legal);
    setOperationAction(ISD::UMAX,               MVT::v4i32, Legal);
    setOperationAction(ISD::SMIN,               MVT::v16i8, Legal);
    setOperationAction(ISD::SMIN,               MVT::v4i32, Legal);
    setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
    setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
 
    setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);
    setOperationAction(ISD::SADDSAT,            MVT::v2i64, Custom);
    setOperationAction(ISD::SSUBSAT,            MVT::v2i64, Custom);
 
    // FIXME: Do we need to handle scalar-to-vector here?
    setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
    setOperationAction(ISD::SMULO,              MVT::v2i32, Custom);
 
    // We directly match byte blends in the backend as they match the VSELECT
    // condition form.
    setOperationAction(ISD::VSELECT,            MVT::v16i8, Legal);
 
    // SSE41 brings specific instructions for doing vector sign extend even in
    // cases where we don't have SRA.
    for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
    }
 
    // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
    for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
      setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8,  Legal);
      setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8,  Legal);
      setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8,  Legal);
      setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
      setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
      setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
    }
 
    if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
      // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
      // do the pre and post work in the vector domain.
      setOperationAction(ISD::UINT_TO_FP,        MVT::v4i64, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
      // We need to mark SINT_TO_FP as Custom even though we want to expand it
      // so that DAG combine doesn't try to turn it into uint_to_fp.
      setOperationAction(ISD::SINT_TO_FP,        MVT::v4i64, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
    }
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
    setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
    for (MVT VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
                    MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
      setOperationAction(ISD::ROTL, VT, VT.is128BitVector() ? Legal : Custom);
      setOperationAction(ISD::ROTR, VT, Custom);
    }
 
    // XOP can efficiently perform BITREVERSE with VPPERM.
    for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
      setOperationAction(ISD::BITREVERSE, VT, Custom);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
    bool HasInt256 = Subtarget.hasInt256();
 
    addRegisterClass(MVT::v32i8,  Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v8i32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v8f32,  Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v4i64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
    addRegisterClass(MVT::v4f64,  Subtarget.hasVLX() ? &X86::VR256XRegClass
                                                     : &X86::VR256RegClass);
 
    for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
      setOperationAction(ISD::FFLOOR,            VT, Legal);
      setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
      setOperationAction(ISD::FCEIL,             VT, Legal);
      setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
      setOperationAction(ISD::FTRUNC,            VT, Legal);
      setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
      setOperationAction(ISD::FRINT,             VT, Legal);
      setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
      setOperationAction(ISD::FNEARBYINT,        VT, Legal);
      setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
      setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
      setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
 
      setOperationAction(ISD::FROUND,            VT, Custom);
 
      setOperationAction(ISD::FNEG,              VT, Custom);
      setOperationAction(ISD::FABS,              VT, Custom);
      setOperationAction(ISD::FCOPYSIGN,         VT, Custom);
      setOperationAction(ISD::FCANONICALIZE,     VT, Expand);
      SetFPMinMaxAction(VT);
    }
 
    setOperationAction(ISD::LRINT, MVT::v8f32, Custom);
    setOperationAction(ISD::LRINT, MVT::v4f64, Custom);
 
    setOperationAction(ISD::AND, MVT::i256, Custom);
    setOperationAction(ISD::OR, MVT::i256, Custom);
    setOperationAction(ISD::XOR, MVT::i256, Custom);
    setOperationAction(ISD::SELECT, MVT::i256, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::i256, Custom);
 
    // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
    // even though v8i16 is a legal type.
    setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i16, MVT::v8i32);
    setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i16, MVT::v8i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
    setOperationAction(ISD::FP_TO_SINT,                MVT::v8i32, Custom);
    setOperationAction(ISD::FP_TO_UINT,                MVT::v8i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v8i32, Custom);
 
    setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i32, Custom);
    setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Expand);
    setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Expand);
    setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Custom);
 
    setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v4f32, Legal);
    setOperationAction(ISD::STRICT_FADD,        MVT::v8f32, Legal);
    setOperationAction(ISD::STRICT_FADD,        MVT::v4f64, Legal);
    setOperationAction(ISD::STRICT_FSUB,        MVT::v8f32, Legal);
    setOperationAction(ISD::STRICT_FSUB,        MVT::v4f64, Legal);
    setOperationAction(ISD::STRICT_FMUL,        MVT::v8f32, Legal);
    setOperationAction(ISD::STRICT_FMUL,        MVT::v4f64, Legal);
    setOperationAction(ISD::STRICT_FDIV,        MVT::v8f32, Legal);
    setOperationAction(ISD::STRICT_FDIV,        MVT::v4f64, Legal);
    setOperationAction(ISD::STRICT_FSQRT,       MVT::v8f32, Legal);
    setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f64, Legal);
 
    if (!Subtarget.hasAVX512())
      setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
 
    // In the customized shift lowering, the legal v8i32/v4i64 cases
    // in AVX2 will be recognized.
    for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
      setOperationAction(ISD::SRL,             VT, Custom);
      setOperationAction(ISD::SHL,             VT, Custom);
      setOperationAction(ISD::SRA,             VT, Custom);
      setOperationAction(ISD::ABDS,            VT, Custom);
      setOperationAction(ISD::ABDU,            VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR,    VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_MUL,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX,  VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN,  VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX,  VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN,  VT, Custom);
      if (VT == MVT::v4i64) continue;
      setOperationAction(ISD::ROTL,            VT, Custom);
      setOperationAction(ISD::ROTR,            VT, Custom);
      setOperationAction(ISD::FSHL,            VT, Custom);
      setOperationAction(ISD::FSHR,            VT, Custom);
    }
 
    // These types need custom splitting if their input is a 128-bit vector.
    setOperationAction(ISD::SIGN_EXTEND,       MVT::v8i64,  Custom);
    setOperationAction(ISD::SIGN_EXTEND,       MVT::v16i32, Custom);
    setOperationAction(ISD::ZERO_EXTEND,       MVT::v8i64,  Custom);
    setOperationAction(ISD::ZERO_EXTEND,       MVT::v16i32, Custom);
 
    setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);
    setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);
    setOperationAction(ISD::SELECT,            MVT::v8i32, Custom);
    setOperationAction(ISD::SELECT,            MVT::v16i16, Custom);
    setOperationAction(ISD::SELECT,            MVT::v16f16, Custom);
    setOperationAction(ISD::SELECT,            MVT::v32i8, Custom);
    setOperationAction(ISD::SELECT,            MVT::v8f32, Custom);
 
    for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
      setOperationAction(ISD::SIGN_EXTEND,     VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND,     VT, Custom);
      setOperationAction(ISD::ANY_EXTEND,      VT, Custom);
    }
 
    setOperationAction(ISD::TRUNCATE,          MVT::v32i8, Custom);
    setOperationAction(ISD::TRUNCATE,          MVT::v32i16, Custom);
    setOperationAction(ISD::TRUNCATE,          MVT::v32i32, Custom);
    setOperationAction(ISD::TRUNCATE,          MVT::v32i64, Custom);
 
    for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
      setOperationAction(ISD::SETCC,           VT, Custom);
      setOperationAction(ISD::CTPOP,           VT, Custom);
      setOperationAction(ISD::CTLZ,            VT, Custom);
      setOperationAction(ISD::BITREVERSE,      VT, Custom);
 
      // The condition codes aren't legal in SSE/AVX and under AVX512 we use
      // setcc all the way to isel and prefer SETGT in some isel patterns.
      setCondCodeAction(ISD::SETLT, VT, Custom);
      setCondCodeAction(ISD::SETLE, VT, Custom);
    }
 
    setOperationAction(ISD::SETCC,          MVT::v4f64, Custom);
    setOperationAction(ISD::SETCC,          MVT::v8f32, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v4f64, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f32, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
 
    if (Subtarget.hasAnyFMA()) {
      for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
                       MVT::v2f64, MVT::v4f64 }) {
        setOperationAction(ISD::FMA, VT, Legal);
        setOperationAction(ISD::STRICT_FMA, VT, Legal);
      }
    }
 
    for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
      setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
      setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
    }
 
    setOperationAction(ISD::MUL,       MVT::v4i64,  Custom);
    setOperationAction(ISD::MUL,       MVT::v8i32,  HasInt256 ? Legal : Custom);
    setOperationAction(ISD::MUL,       MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::MUL,       MVT::v32i8,  Custom);
 
    setOperationAction(ISD::MULHU,     MVT::v8i32,  Custom);
    setOperationAction(ISD::MULHS,     MVT::v8i32,  Custom);
    setOperationAction(ISD::MULHU,     MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::MULHS,     MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::MULHU,     MVT::v32i8,  Custom);
    setOperationAction(ISD::MULHS,     MVT::v32i8,  Custom);
    setOperationAction(ISD::AVGCEILU,  MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::AVGCEILU,  MVT::v32i8,  HasInt256 ? Legal : Custom);
 
    setOperationAction(ISD::SMULO,     MVT::v32i8, Custom);
    setOperationAction(ISD::UMULO,     MVT::v32i8, Custom);
 
    setOperationAction(ISD::ABS,       MVT::v4i64,  Custom);
    setOperationAction(ISD::SMAX,      MVT::v4i64,  Custom);
    setOperationAction(ISD::UMAX,      MVT::v4i64,  Custom);
    setOperationAction(ISD::SMIN,      MVT::v4i64,  Custom);
    setOperationAction(ISD::UMIN,      MVT::v4i64,  Custom);
 
    setOperationAction(ISD::UADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
    setOperationAction(ISD::SADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
    setOperationAction(ISD::USUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
    setOperationAction(ISD::SSUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);
    setOperationAction(ISD::UADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::SADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::USUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::SSUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);
    setOperationAction(ISD::UADDSAT,   MVT::v8i32, Custom);
    setOperationAction(ISD::USUBSAT,   MVT::v8i32, Custom);
    setOperationAction(ISD::UADDSAT,   MVT::v4i64, Custom);
    setOperationAction(ISD::USUBSAT,   MVT::v4i64, Custom);
 
    for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
      setOperationAction(ISD::ABS,  VT, HasInt256 ? Legal : Custom);
      setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
      setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
      setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
      setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
    }
 
    for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
    }
 
    if (HasInt256) {
      // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
      // when we have a 256bit-wide blend with immediate.
      setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
 
      // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
      for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
        setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
        setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i8,  Legal);
        setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i8,  Legal);
        setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i16, Legal);
        setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i16, Legal);
        setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i32, Legal);
      }
    }
 
    for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
                     MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
      setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
      setOperationAction(ISD::MSTORE, VT, Legal);
    }
 
    // Extract subvector is special because the value type
    // (result) is 128-bit but the source is 256-bit wide.
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
                     MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
    }
 
    // Custom lower several nodes for 256-bit types.
    for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
                    MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
      setOperationAction(ISD::VSELECT,            VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
      setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
      setOperationAction(ISD::STORE,              VT, Custom);
    }
    setF16Action(MVT::v16f16, Expand);
    setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
    setOperationAction(ISD::FABS, MVT::v16f16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::v16f16, Custom);
    setOperationAction(ISD::FADD, MVT::v16f16, Expand);
    setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
    setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
    setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
 
    // Only PCLMUL required as we always unroll clmul vectors.
    if (Subtarget.hasPCLMUL()) {
      for (auto VT : {MVT::v8i32, MVT::v4i64}) {
        setOperationAction(ISD::CLMUL, VT, Custom);
        setOperationAction(ISD::CLMULH, VT, Custom);
      }
    }
 
    if (HasInt256) {
      setOperationAction(ISD::MULHU, MVT::v4i64, Custom);
      // Custom so the combiner keeps full products as [SU]MUL_LOHI, not
      // MULH[SU].
      setOperationAction(ISD::UMUL_LOHI, MVT::v4i64, Custom);
      setOperationAction(ISD::SMUL_LOHI, MVT::v4i64, Custom);
      setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
 
      // Custom legalize 2x32 to get a little better code.
      setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
      setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
 
      for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
                       MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
        setOperationAction(ISD::MGATHER,  VT, Custom);
    }
 
    if (Subtarget.hasGFNI()) {
      setOperationAction(ISD::CTLZ, MVT::v32i8, Custom);
      setOperationAction(ISD::CTTZ, MVT::v32i8, Custom);
    }
  }
 
  if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
      Subtarget.hasF16C()) {
    for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
      setOperationAction(ISD::FP_ROUND,           VT, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND,    VT, Custom);
    }
    for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
      setOperationAction(ISD::FP_EXTEND,          VT, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,   VT, Custom);
    }
    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
      setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
      setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
    }
    setOperationAction(ISD::SETCC, MVT::v8f16, Custom);
    setOperationAction(ISD::SETCC, MVT::v16f16, Custom);
  }
 
  // This block controls legalization of the mask vector sizes that are
  // available with AVX512. 512-bit vectors are in a separate block controlled
  // by useAVX512Regs.
  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
    addRegisterClass(MVT::v1i1,   &X86::VK1RegClass);
    addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);
    addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);
    addRegisterClass(MVT::v8i1,   &X86::VK8RegClass);
    addRegisterClass(MVT::v16i1,  &X86::VK16RegClass);
 
    setOperationAction(ISD::SELECT,             MVT::v1i1, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
    setOperationAction(ISD::BUILD_VECTOR,       MVT::v1i1, Custom);
 
    setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i1,  MVT::v8i32);
    setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i1,  MVT::v8i32);
    setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v4i1,  MVT::v4i32);
    setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v4i1,  MVT::v4i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1,  MVT::v8i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1,  MVT::v8i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1,  MVT::v4i32);
    setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1,  MVT::v4i32);
    setOperationAction(ISD::FP_TO_SINT,                MVT::v2i1,  Custom);
    setOperationAction(ISD::FP_TO_UINT,                MVT::v2i1,  Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v2i1,  Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT,         MVT::v2i1,  Custom);
    setOperationAction(ISD::FCANONICALIZE, MVT::v8f16, Expand);
    setOperationAction(ISD::FCANONICALIZE, MVT::v16f16, Expand);
    setOperationAction(ISD::FCANONICALIZE, MVT::v32f16, Expand);
 
    // There is no byte sized k-register load or store without AVX512DQ.
    if (!Subtarget.hasDQI()) {
      setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
      setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
      setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
      setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
 
      setOperationAction(ISD::STORE, MVT::v1i1, Custom);
      setOperationAction(ISD::STORE, MVT::v2i1, Custom);
      setOperationAction(ISD::STORE, MVT::v4i1, Custom);
      setOperationAction(ISD::STORE, MVT::v8i1, Custom);
    }
 
    // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
    for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
      setOperationAction(ISD::ANY_EXTEND,  VT, Custom);
    }
 
    for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
      setOperationAction(ISD::VSELECT,          VT, Expand);
 
    for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
      setOperationAction(ISD::SETCC,            VT, Custom);
      setOperationAction(ISD::SELECT,           VT, Custom);
      setOperationAction(ISD::TRUNCATE,         VT, Custom);
 
      setOperationAction(ISD::BUILD_VECTOR,     VT, Custom);
      setOperationAction(ISD::CONCAT_VECTORS,   VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,   VT,  Custom);
    }
 
    for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
  }
  if (Subtarget.hasDQI() && Subtarget.hasVLX()) {
    for (MVT VT : {MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
      setOperationAction(ISD::LRINT, VT, Legal);
      setOperationAction(ISD::LLRINT, VT, Legal);
    }
  }
 
  // This block controls legalization for 512-bit operations with 8/16/32/64 bit
  // elements. 512-bits can be disabled based on prefer-vector-width and
  // required-vector-width function attributes.
  if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
    bool HasBWI = Subtarget.hasBWI();
 
    addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
    addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
    addRegisterClass(MVT::v8i64,  &X86::VR512RegClass);
    addRegisterClass(MVT::v8f64,  &X86::VR512RegClass);
    addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
    addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
    addRegisterClass(MVT::v64i8,  &X86::VR512RegClass);
 
    for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
      setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8,  Legal);
      setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
      setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i8,   Legal);
      setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i16,  Legal);
      setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i32,  Legal);
      if (HasBWI)
        setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
    }
 
    for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
      SetFPMinMaxAction(VT);
      setOperationAction(ISD::FNEG,  VT, Custom);
      setOperationAction(ISD::FABS,  VT, Custom);
      setOperationAction(ISD::FMA,   VT, Legal);
      setOperationAction(ISD::STRICT_FMA, VT, Legal);
      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
      setOperationAction(ISD::FLDEXP, VT, Custom);
    }
    setOperationAction(ISD::LRINT, MVT::v16f32,
                       Subtarget.hasDQI() ? Legal : Custom);
    setOperationAction(ISD::LRINT, MVT::v8f64,
                       Subtarget.hasDQI() ? Legal : Custom);
    if (Subtarget.hasDQI())
      setOperationAction(ISD::LLRINT, MVT::v8f64, Legal);
 
    setOperationAction(ISD::AND, MVT::i512, Custom);
    setOperationAction(ISD::OR, MVT::i512, Custom);
    setOperationAction(ISD::XOR, MVT::i512, Custom);
    setOperationAction(ISD::ADD, MVT::i512, Custom);
    setOperationAction(ISD::SUB, MVT::i512, Custom);
    setOperationAction(ISD::SRL, MVT::i512, Custom);
    setOperationAction(ISD::SHL, MVT::i512, Custom);
    setOperationAction(ISD::SRA, MVT::i512, Custom);
    setOperationAction(ISD::FSHR, MVT::i512, Custom);
    setOperationAction(ISD::FSHL, MVT::i512, Custom);
    setOperationAction(ISD::FSHR, MVT::i256, Custom);
    setOperationAction(ISD::FSHL, MVT::i256, Custom);
    setOperationAction(ISD::SELECT, MVT::i512, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::i512, Custom);
 
    for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
      setOperationPromotedToType(ISD::FP_TO_SINT       , VT, MVT::v16i32);
      setOperationPromotedToType(ISD::FP_TO_UINT       , VT, MVT::v16i32);
      setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
      setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
    }
 
    for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
      setOperationAction(ISD::FP_TO_SINT,        VT, Custom);
      setOperationAction(ISD::FP_TO_UINT,        VT, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
    }
 
    setOperationAction(ISD::SINT_TO_FP,        MVT::v16i32, Custom);
    setOperationAction(ISD::UINT_TO_FP,        MVT::v16i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
    setOperationAction(ISD::FP_EXTEND,         MVT::v8f64,  Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v8f64,  Custom);
 
    setOperationAction(ISD::STRICT_FADD,      MVT::v16f32, Legal);
    setOperationAction(ISD::STRICT_FADD,      MVT::v8f64,  Legal);
    setOperationAction(ISD::STRICT_FSUB,      MVT::v16f32, Legal);
    setOperationAction(ISD::STRICT_FSUB,      MVT::v8f64,  Legal);
    setOperationAction(ISD::STRICT_FMUL,      MVT::v16f32, Legal);
    setOperationAction(ISD::STRICT_FMUL,      MVT::v8f64,  Legal);
    setOperationAction(ISD::STRICT_FDIV,      MVT::v16f32, Legal);
    setOperationAction(ISD::STRICT_FDIV,      MVT::v8f64,  Legal);
    setOperationAction(ISD::STRICT_FSQRT,     MVT::v16f32, Legal);
    setOperationAction(ISD::STRICT_FSQRT,     MVT::v8f64,  Legal);
    setOperationAction(ISD::STRICT_FP_ROUND,  MVT::v8f32,  Legal);
 
    setTruncStoreAction(MVT::v8i64,   MVT::v8i8,   Legal);
    setTruncStoreAction(MVT::v8i64,   MVT::v8i16,  Legal);
    setTruncStoreAction(MVT::v8i64,   MVT::v8i32,  Legal);
    setTruncStoreAction(MVT::v16i32,  MVT::v16i8,  Legal);
    setTruncStoreAction(MVT::v16i32,  MVT::v16i16, Legal);
    if (HasBWI)
      setTruncStoreAction(MVT::v32i16,  MVT::v32i8, Legal);
 
    // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
    // to 512-bit rather than use the AVX2 instructions so that we can use
    // k-masks.
    if (!Subtarget.hasVLX()) {
      for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
           MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
        setOperationAction(ISD::MLOAD,  VT, Custom);
        setOperationAction(ISD::MSTORE, VT, Custom);
      }
    }
 
    setOperationAction(ISD::TRUNCATE,    MVT::v8i32,  Legal);
    setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Legal);
    setOperationAction(ISD::TRUNCATE,    MVT::v32i8,  HasBWI ? Legal : Custom);
    setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
    setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
    setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64,  Custom);
    setOperationAction(ISD::ANY_EXTEND,  MVT::v32i16, Custom);
    setOperationAction(ISD::ANY_EXTEND,  MVT::v16i32, Custom);
    setOperationAction(ISD::ANY_EXTEND,  MVT::v8i64,  Custom);
    setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
    setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
    setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64,  Custom);
 
    if (HasBWI) {
      // Extends from v64i1 masks to 512-bit vectors.
      setOperationAction(ISD::SIGN_EXTEND,        MVT::v64i8, Custom);
      setOperationAction(ISD::ZERO_EXTEND,        MVT::v64i8, Custom);
      setOperationAction(ISD::ANY_EXTEND,         MVT::v64i8, Custom);
    }
 
    for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
      setOperationAction(ISD::FFLOOR,            VT, Legal);
      setOperationAction(ISD::STRICT_FFLOOR,     VT, Legal);
      setOperationAction(ISD::FCEIL,             VT, Legal);
      setOperationAction(ISD::STRICT_FCEIL,      VT, Legal);
      setOperationAction(ISD::FTRUNC,            VT, Legal);
      setOperationAction(ISD::STRICT_FTRUNC,     VT, Legal);
      setOperationAction(ISD::FRINT,             VT, Legal);
      setOperationAction(ISD::STRICT_FRINT,      VT, Legal);
      setOperationAction(ISD::FNEARBYINT,        VT, Legal);
      setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
      setOperationAction(ISD::FROUNDEVEN,        VT, Legal);
      setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
 
      setOperationAction(ISD::FROUND,            VT, Custom);
    }
 
    for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
    }
 
    setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::ADD, MVT::v64i8,  HasBWI ? Legal : Custom);
    setOperationAction(ISD::SUB, MVT::v64i8,  HasBWI ? Legal : Custom);
 
    setOperationAction(ISD::MUL, MVT::v8i64,  Custom);
    setOperationAction(ISD::MUL, MVT::v16i32, Legal);
    setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::MUL, MVT::v64i8,  Custom);
 
    setOperationAction(ISD::MULHU, MVT::v8i64, Custom);
    setOperationAction(ISD::UMUL_LOHI, MVT::v8i64, Custom);
    setOperationAction(ISD::SMUL_LOHI, MVT::v8i64, Custom);
    setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
    setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
    setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::MULHS, MVT::v64i8,  Custom);
    setOperationAction(ISD::MULHU, MVT::v64i8,  Custom);
    setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
    setOperationAction(ISD::AVGCEILU, MVT::v64i8,  HasBWI ? Legal : Custom);
 
    setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
    setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
 
    for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
      setOperationAction(ISD::SRL,              VT, Custom);
      setOperationAction(ISD::SHL,              VT, Custom);
      setOperationAction(ISD::SRA,              VT, Custom);
      setOperationAction(ISD::ROTL,             VT, Custom);
      setOperationAction(ISD::ROTR,             VT, Custom);
      setOperationAction(ISD::SETCC,            VT, Custom);
      setOperationAction(ISD::ABDS,             VT, Custom);
      setOperationAction(ISD::ABDU,             VT, Custom);
      setOperationAction(ISD::BITREVERSE,       VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND,    VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR,     VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR,    VT, Custom);
      setOperationAction(ISD::VECREDUCE_MUL,    VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX,   VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN,   VT, Custom);
 
      // The condition codes aren't legal in SSE/AVX and under AVX512 we use
      // setcc all the way to isel and prefer SETGT in some isel patterns.
      setCondCodeAction(ISD::SETLT, VT, Custom);
      setCondCodeAction(ISD::SETLE, VT, Custom);
    }
 
    setOperationAction(ISD::SETCC,          MVT::v8f64, Custom);
    setOperationAction(ISD::SETCC,          MVT::v16f32, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v8f64, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::v16f32, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
 
    for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
      setOperationAction(ISD::SMAX,             VT, Legal);
      setOperationAction(ISD::UMAX,             VT, Legal);
      setOperationAction(ISD::SMIN,             VT, Legal);
      setOperationAction(ISD::UMIN,             VT, Legal);
      setOperationAction(ISD::ROTL,             VT, Legal);
      setOperationAction(ISD::ROTR,             VT, Legal);
      setOperationAction(ISD::ABS,              VT, Legal);
      setOperationAction(ISD::CTPOP,            VT, Custom);
    }
 
    for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
      setOperationAction(ISD::ABS,     VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::CTPOP,   VT, Subtarget.hasBITALG() ? Legal : Custom);
      setOperationAction(ISD::CTLZ,    VT, Custom);
      setOperationAction(ISD::SMAX,    VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::UMAX,    VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::SMIN,    VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::UMIN,    VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
      setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
    }
 
    setOperationAction(ISD::FSHL,       MVT::v64i8, Custom);
    setOperationAction(ISD::FSHR,       MVT::v64i8, Custom);
    setOperationAction(ISD::FSHL,      MVT::v32i16, Custom);
    setOperationAction(ISD::FSHR,      MVT::v32i16, Custom);
    setOperationAction(ISD::FSHL,      MVT::v16i32, Custom);
    setOperationAction(ISD::FSHR,      MVT::v16i32, Custom);
 
    if (Subtarget.hasDQI() || Subtarget.hasFP16())
      for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
                       ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                       ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
        setOperationAction(Opc,           MVT::v8i64, Custom);
 
    if (Subtarget.hasDQI()) {
      setOperationAction(ISD::MUL,        MVT::v8i64, Legal);
 
      // MULHS needs vpmullq (AVX512DQ) for its low multiply to be a win.
      setOperationAction(ISD::MULHS, MVT::v8i64, Custom);
    }
 
    if (Subtarget.hasCDI()) {
      // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
      for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
        setOperationAction(ISD::CTLZ,            VT, Legal);
      }
    } // Subtarget.hasCDI()
 
    if (Subtarget.hasVPOPCNTDQ()) {
      for (auto VT : { MVT::v16i32, MVT::v8i64 })
        setOperationAction(ISD::CTPOP, VT, Legal);
      setOperationAction(ISD::CTPOP, MVT::i512, Custom);
    }
 
    // Extract subvector is special because the value type
    // (result) is 256-bit but the source is 512-bit wide.
    // 128-bit was made Legal under AVX1.
    for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
                     MVT::v16f16, MVT::v8f32, MVT::v4f64 })
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
 
    for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
                     MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
      setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);
      setOperationAction(ISD::SELECT,             VT, Custom);
      setOperationAction(ISD::VSELECT,            VT, Custom);
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
      setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
    }
    setF16Action(MVT::v32f16, Expand);
    setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
    setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
      setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
    setOperationAction(ISD::SETCC, MVT::v32f16, Custom);
 
    for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
      setOperationAction(ISD::MLOAD,               VT, Legal);
      setOperationAction(ISD::MSTORE,              VT, Legal);
      setOperationAction(ISD::MGATHER,             VT, Custom);
      setOperationAction(ISD::MSCATTER,            VT, Custom);
    }
    if (HasBWI) {
      for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
        setOperationAction(ISD::MLOAD,        VT, Legal);
        setOperationAction(ISD::MSTORE,       VT, Legal);
      }
    } else {
      setOperationAction(ISD::STORE, MVT::v32i16, Custom);
      setOperationAction(ISD::STORE, MVT::v64i8,  Custom);
    }
 
    if (Subtarget.hasVBMI2()) {
      for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
        setOperationAction(ISD::FSHL, VT, Legal);
        setOperationAction(ISD::FSHR, VT, Legal);
      }
 
      setOperationAction(ISD::ROTL, MVT::v32i16, Legal);
      setOperationAction(ISD::ROTR, MVT::v32i16, Legal);
    }
 
    // Only PCLMUL required as we always unroll clmul vectors.
    if (Subtarget.hasPCLMUL()) {
      for (auto VT : {MVT::v16i32, MVT::v8i64}) {
        setOperationAction(ISD::CLMUL, VT, Custom);
        setOperationAction(ISD::CLMULH, VT, Custom);
      }
    }
 
    setOperationAction(ISD::FNEG, MVT::v32f16, Custom);
    setOperationAction(ISD::FABS, MVT::v32f16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::v32f16, Custom);
    setOperationAction(ISD::FLDEXP, MVT::v32f16, Custom);
 
    if (Subtarget.hasGFNI()) {
      setOperationAction(ISD::CTLZ, MVT::v64i8, Custom);
      setOperationAction(ISD::CTTZ, MVT::v64i8, Custom);
    }
  }// useAVX512Regs
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
    for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
                    MVT::v4i64}) {
      setOperationAction(ISD::FSHL, VT, Legal);
      setOperationAction(ISD::FSHR, VT, Legal);
    }
 
    setOperationAction(ISD::ROTL, MVT::v16i16, Legal);
    setOperationAction(ISD::ROTR, MVT::v16i16, Legal);
    setOperationAction(ISD::ROTL, MVT::v8i16, Legal);
    setOperationAction(ISD::ROTR, MVT::v8i16, Legal);
  }
 
  // This block controls legalization for operations that don't have
  // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
  // narrower widths.
  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
    for (MVT VT : {MVT::f16, MVT::f32, MVT::f64, MVT::v8f16, MVT::v4f32,
                   MVT::v2f64, MVT::v16f16, MVT::v8f32, MVT::v4f64})
      setOperationAction(ISD::FLDEXP, VT, Custom);
 
    // These operations are handled on non-VLX by artificially widening in
    // isel patterns.
    setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v4i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v2i32, Custom);
 
    if (Subtarget.hasDQI()) {
      // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
      // v2f32 UINT_TO_FP is already custom under SSE2.
      assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
             isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
             "Unexpected operation action!");
      // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
      setOperationAction(ISD::FP_TO_SINT,        MVT::v2f32, Custom);
      setOperationAction(ISD::FP_TO_UINT,        MVT::v2f32, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
    }
 
    for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
      setOperationAction(ISD::SMAX, VT, Legal);
      setOperationAction(ISD::UMAX, VT, Legal);
      setOperationAction(ISD::SMIN, VT, Legal);
      setOperationAction(ISD::UMIN, VT, Legal);
      setOperationAction(ISD::ABS,  VT, Legal);
    }
 
    for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
      setOperationAction(ISD::ROTL,     VT, Legal);
      setOperationAction(ISD::ROTR,     VT, Legal);
    }
 
    // Custom legalize 2x32 to get a little better code.
    setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
    setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
 
    for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
                     MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
      setOperationAction(ISD::MSCATTER, VT, Custom);
 
    if (Subtarget.hasDQI()) {
      for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
                       ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                       ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
        setOperationAction(Opc, MVT::v2i64, Custom);
        setOperationAction(Opc, MVT::v4i64, Custom);
      }
      setOperationAction(ISD::MUL, MVT::v2i64, Legal);
      setOperationAction(ISD::MUL, MVT::v4i64, Legal);
 
      // MULHS is only a win when the low multiply can use vpmullq; non-VLX
      // targets handle VPMULLQ by implicit widening.
      setOperationAction(ISD::MULHS, MVT::v4i64, Custom);
    }
 
    if (Subtarget.hasCDI()) {
      for (auto VT : {MVT::i256, MVT::i512}) {
        if (VT == MVT::i512 && !Subtarget.useAVX512Regs())
          continue;
        setOperationAction(ISD::CTLZ, VT, Custom);
        setOperationAction(ISD::CTTZ, VT, Custom);
        setOperationAction(ISD::CTLZ_ZERO_POISON, VT, Custom);
        setOperationAction(ISD::CTTZ_ZERO_POISON, VT, Custom);
      }
      for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
        setOperationAction(ISD::CTLZ,            VT, Legal);
      }
    } // Subtarget.hasCDI()
 
    if (Subtarget.hasVPOPCNTDQ()) {
      for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64})
        setOperationAction(ISD::CTPOP, VT, Legal);
      setOperationAction(ISD::CTPOP, MVT::i256, Custom);
    }
 
    // We can try to convert vectors to different sizes to leverage legal
    // `vpcompress` cases. So we mark these supported vector sizes as Custom and
    // then specialize to Legal below.
    for (MVT VT : {MVT::v8i32, MVT::v8f32, MVT::v4i32, MVT::v4f32, MVT::v4i64,
                   MVT::v4f64, MVT::v2i64, MVT::v2f64, MVT::v16i8, MVT::v8i16,
                   MVT::v16i16, MVT::v8i8})
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
 
    // Legal vpcompress depends on various AVX512 extensions.
    // Legal in AVX512F
    for (MVT VT : {MVT::v16i32, MVT::v16f32, MVT::v8i64, MVT::v8f64})
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Legal);
 
    // Legal in AVX512F + AVX512VL
    if (Subtarget.hasVLX())
      for (MVT VT : {MVT::v8i32, MVT::v8f32, MVT::v4i32, MVT::v4f32, MVT::v4i64,
                     MVT::v4f64, MVT::v2i64, MVT::v2f64})
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Legal);
 
    // Legal in AVX512F + AVX512VBMI2
    if (Subtarget.hasVBMI2())
      for (MVT VT : {MVT::v32i16, MVT::v64i8})
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Legal);
 
    // Legal in AVX512F + AVX512VL + AVX512VBMI2
    if (Subtarget.hasVBMI2() && Subtarget.hasVLX())
      for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v32i8, MVT::v16i16})
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Legal);
  }
 
  // This block control legalization of v32i1/v64i1 which are available with
  // AVX512BW..
  if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
    addRegisterClass(MVT::v32i1,  &X86::VK32RegClass);
    addRegisterClass(MVT::v64i1,  &X86::VK64RegClass);
 
    for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
      setOperationAction(ISD::VSELECT,            VT, Expand);
      setOperationAction(ISD::TRUNCATE,           VT, Custom);
      setOperationAction(ISD::SETCC,              VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);
      setOperationAction(ISD::SELECT,             VT, Custom);
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
      setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Custom);
    }
 
    for (auto VT : { MVT::v16i1, MVT::v32i1 })
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
    // Extends from v32i1 masks to 256-bit vectors.
    setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);
    setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);
    setOperationAction(ISD::ANY_EXTEND,         MVT::v32i8, Custom);
 
    for (auto VT : {MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16,
                    MVT::v16f16, MVT::v8f16}) {
      setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);
      setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
    }
 
    // These operations are handled on non-VLX by artificially widening in
    // isel patterns.
    // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
 
    if (Subtarget.hasBITALG()) {
      for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
        setOperationAction(ISD::CTPOP, VT, Legal);
    }
 
    if (Subtarget.hasBMM()) {
      setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
      setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
      setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
      setOperationAction(ISD::BITREVERSE, MVT::i64, Custom);
 
      for (auto VT : {MVT::v16i8, MVT::v32i8, MVT::v64i8})
        setOperationAction(ISD::BITREVERSE, VT, Legal);
    }
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
    auto setGroup = [&] (MVT VT) {
      setOperationAction(ISD::FADD,               VT, Legal);
      setOperationAction(ISD::STRICT_FADD,        VT, Legal);
      setOperationAction(ISD::FSUB,               VT, Legal);
      setOperationAction(ISD::STRICT_FSUB,        VT, Legal);
      setOperationAction(ISD::FMUL,               VT, Legal);
      setOperationAction(ISD::STRICT_FMUL,        VT, Legal);
      setOperationAction(ISD::FDIV,               VT, Legal);
      setOperationAction(ISD::STRICT_FDIV,        VT, Legal);
      setOperationAction(ISD::FSQRT,              VT, Legal);
      setOperationAction(ISD::STRICT_FSQRT,       VT, Legal);
 
      setOperationAction(ISD::FFLOOR,             VT, Legal);
      setOperationAction(ISD::STRICT_FFLOOR,      VT, Legal);
      setOperationAction(ISD::FCEIL,              VT, Legal);
      setOperationAction(ISD::STRICT_FCEIL,       VT, Legal);
      setOperationAction(ISD::FTRUNC,             VT, Legal);
      setOperationAction(ISD::STRICT_FTRUNC,      VT, Legal);
      setOperationAction(ISD::FRINT,              VT, Legal);
      setOperationAction(ISD::STRICT_FRINT,       VT, Legal);
      setOperationAction(ISD::FNEARBYINT,         VT, Legal);
      setOperationAction(ISD::STRICT_FNEARBYINT,  VT, Legal);
      setOperationAction(ISD::FROUNDEVEN, VT, Legal);
      setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
 
      setOperationAction(ISD::FROUND,             VT, Custom);
 
      setOperationAction(ISD::LOAD,               VT, Legal);
      setOperationAction(ISD::STORE,              VT, Legal);
 
      setOperationAction(ISD::FMA,                VT, Legal);
      setOperationAction(ISD::STRICT_FMA,         VT, Legal);
      setOperationAction(ISD::VSELECT,            VT, Legal);
      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
      setOperationAction(ISD::SELECT,             VT, Custom);
 
      setOperationAction(ISD::FNEG,               VT, Custom);
      setOperationAction(ISD::FABS,               VT, Custom);
      setOperationAction(ISD::FCOPYSIGN,          VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
 
      setOperationAction(ISD::SETCC,              VT, Custom);
      setOperationAction(ISD::STRICT_FSETCC,      VT, Custom);
      setOperationAction(ISD::STRICT_FSETCCS,     VT, Custom);
    };
 
    // AVX512_FP16 scalar operations
    setGroup(MVT::f16);
    SetFPMinMaxAction(MVT::f16);
    setOperationAction(ISD::FREM,                 MVT::f16, Promote);
    setOperationAction(ISD::STRICT_FREM,          MVT::f16, Promote);
    setOperationAction(ISD::SELECT_CC,            MVT::f16, Expand);
    setOperationAction(ISD::BR_CC,                MVT::f16, Expand);
    setOperationAction(ISD::STRICT_FROUND,        MVT::f16, Promote);
    setOperationAction(ISD::FROUNDEVEN,           MVT::f16, Legal);
    setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);
    setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);
    setOperationAction(ISD::FP_EXTEND,            MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);
    setOperationAction(ISD::LRINT,                MVT::f16, Legal);
    setOperationAction(ISD::LLRINT,               MVT::f16, Legal);
 
    setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
    setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
 
    if (Subtarget.useAVX512Regs()) {
      setGroup(MVT::v32f16);
      setOperationAction(ISD::SCALAR_TO_VECTOR,       MVT::v32f16, Custom);
      setOperationAction(ISD::SINT_TO_FP,             MVT::v32i16, Legal);
      setOperationAction(ISD::STRICT_SINT_TO_FP,      MVT::v32i16, Legal);
      setOperationAction(ISD::UINT_TO_FP,             MVT::v32i16, Legal);
      setOperationAction(ISD::STRICT_UINT_TO_FP,      MVT::v32i16, Legal);
      setOperationAction(ISD::FP_ROUND,               MVT::v16f16, Legal);
      setOperationAction(ISD::STRICT_FP_ROUND,        MVT::v16f16, Legal);
      setOperationAction(ISD::FP_EXTEND,              MVT::v16f32, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v16f32, Legal);
      setOperationAction(ISD::FP_EXTEND,              MVT::v8f64,  Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v8f64,  Legal);
      setOperationAction(ISD::INSERT_VECTOR_ELT,      MVT::v32f16, Custom);
 
      setOperationAction(ISD::FP_TO_SINT,             MVT::v32i16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT,      MVT::v32i16, Custom);
      setOperationAction(ISD::FP_TO_UINT,             MVT::v32i16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT,      MVT::v32i16, Custom);
      setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i8,  MVT::v32i16);
      setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
                                 MVT::v32i16);
      setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i8,  MVT::v32i16);
      setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
                                 MVT::v32i16);
      setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i1,  MVT::v32i16);
      setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
                                 MVT::v32i16);
      setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i1,  MVT::v32i16);
      setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
                                 MVT::v32i16);
 
      setOperationAction(ISD::EXTRACT_SUBVECTOR,      MVT::v16f16, Legal);
      setOperationAction(ISD::INSERT_SUBVECTOR,       MVT::v32f16, Legal);
      setOperationAction(ISD::CONCAT_VECTORS,         MVT::v32f16, Custom);
 
      setLoadExtAction(ISD::EXTLOAD, MVT::v8f64,  MVT::v8f16,  Legal);
      setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
 
      SetFPMinMaxAction(MVT::v32f16);
      setOperationAction(ISD::LRINT, MVT::v32f16, Legal);
      setOperationAction(ISD::LLRINT, MVT::v8f16, Legal);
    }
 
    setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);
    setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);
 
    if (Subtarget.hasVLX()) {
      setGroup(MVT::v8f16);
      setGroup(MVT::v16f16);
 
      setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8f16,  Legal);
      setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16f16, Custom);
      setOperationAction(ISD::SINT_TO_FP,         MVT::v16i16, Legal);
      setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v16i16, Legal);
      setOperationAction(ISD::SINT_TO_FP,         MVT::v8i16,  Legal);
      setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i16,  Legal);
      setOperationAction(ISD::UINT_TO_FP,         MVT::v16i16, Legal);
      setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v16i16, Legal);
      setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16,  Legal);
      setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v8i16,  Legal);
 
      setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Legal);
      setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v8f16, Legal);
      setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v8f32, Legal);
      setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Legal);
 
      // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
      setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v8f16,  Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v16f16, Custom);
 
      setOperationAction(ISD::EXTRACT_SUBVECTOR,    MVT::v8f16, Legal);
      setOperationAction(ISD::INSERT_SUBVECTOR,     MVT::v16f16, Legal);
      setOperationAction(ISD::CONCAT_VECTORS,       MVT::v16f16, Custom);
 
      setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
      setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
      setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
      setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
 
      // Need to custom widen these to prevent scalarization.
      setOperationAction(ISD::LOAD,  MVT::v4f16, Custom);
      setOperationAction(ISD::STORE, MVT::v4f16, Custom);
 
      SetFPMinMaxAction(MVT::v8f16);
      SetFPMinMaxAction(MVT::v16f16);
 
      setOperationAction(ISD::LRINT, MVT::v8f16, Legal);
      setOperationAction(ISD::LRINT, MVT::v16f16, Legal);
    }
  }
 
  if (!Subtarget.useSoftFloat() &&
      (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
    addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
                                                        : &X86::VR128RegClass);
    addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
                                                         : &X86::VR256RegClass);
    // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
    // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
    // Set the operation action Custom to do the customization later.
    setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
    for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
      setF16Action(VT, Expand);
      if (!Subtarget.hasBF16())
        setOperationAction(ISD::VSELECT, VT, Custom);
      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
    }
    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
      setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
      setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
    }
    setOperationAction(ISD::SETCC, MVT::v8bf16, Custom);
    setOperationAction(ISD::SETCC, MVT::v16bf16, Custom);
    setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
    addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasBF16() &&
      Subtarget.useAVX512Regs()) {
    addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
    setF16Action(MVT::v32bf16, Expand);
    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
      setOperationPromotedToType(Opc, MVT::v32bf16, MVT::v32f32);
    setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
    setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
    setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
    setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal);
    setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom);
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX10_2()) {
    // Lower scalar bf16 arithmetic by widening to a vector op and extracting
    // the low element.
    setOperationAction(ISD::FADD, MVT::bf16, Custom);
    setOperationAction(ISD::FSUB, MVT::bf16, Custom);
    setOperationAction(ISD::FMUL, MVT::bf16, Custom);
    setOperationAction(ISD::FDIV, MVT::bf16, Custom);
    setOperationAction(ISD::FSQRT, MVT::bf16, Custom);
    setOperationAction(ISD::FMA, MVT::bf16, Custom);
 
    setOperationAction(ISD::FADD, MVT::v32bf16, Legal);
    setOperationAction(ISD::FSUB, MVT::v32bf16, Legal);
    setOperationAction(ISD::FMUL, MVT::v32bf16, Legal);
    setOperationAction(ISD::FDIV, MVT::v32bf16, Legal);
    setOperationAction(ISD::FSQRT, MVT::v32bf16, Legal);
    setOperationAction(ISD::FMA, MVT::v32bf16, Legal);
    setOperationAction(ISD::SETCC, MVT::v32bf16, Custom);
    SetFPMinMaxAction(MVT::v32bf16);
    for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
      setOperationAction(ISD::FADD, VT, Legal);
      setOperationAction(ISD::FSUB, VT, Legal);
      setOperationAction(ISD::FMUL, VT, Legal);
      setOperationAction(ISD::FDIV, VT, Legal);
      setOperationAction(ISD::FSQRT, VT, Legal);
      setOperationAction(ISD::FMA, VT, Legal);
      setOperationAction(ISD::SETCC, VT, Custom);
      SetFPMinMaxAction(VT);
    }
    for (auto VT : {MVT::f16, MVT::f32, MVT::f64}) {
      setCondCodeAction(ISD::SETOEQ, VT, Custom);
      setCondCodeAction(ISD::SETUNE, VT, Custom);
    }
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
    setTruncStoreAction(MVT::v4i64, MVT::v4i8,  Legal);
    setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
    setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
    setTruncStoreAction(MVT::v8i32, MVT::v8i8,  Legal);
    setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
 
    setTruncStoreAction(MVT::v2i64, MVT::v2i8,  Legal);
    setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
    setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
    setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
    setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
 
    if (Subtarget.hasBWI()) {
      setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);
      setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
    }
 
    if (Subtarget.hasFP16()) {
      // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
      setOperationAction(ISD::FP_TO_SINT,        MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
      setOperationAction(ISD::FP_TO_UINT,        MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
      setOperationAction(ISD::FP_TO_SINT,        MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
      setOperationAction(ISD::FP_TO_UINT,        MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
      // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
      setOperationAction(ISD::SINT_TO_FP,        MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
      setOperationAction(ISD::UINT_TO_FP,        MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
      setOperationAction(ISD::SINT_TO_FP,        MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
      setOperationAction(ISD::UINT_TO_FP,        MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
      // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
      setOperationAction(ISD::FP_ROUND,          MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v2f16, Custom);
      setOperationAction(ISD::FP_ROUND,          MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v4f16, Custom);
      // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
      setOperationAction(ISD::FP_EXTEND,         MVT::v2f16, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v2f16, Custom);
      setOperationAction(ISD::FP_EXTEND,         MVT::v4f16, Custom);
      setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v4f16, Custom);
    }
  }
 
  if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
    addRegisterClass(MVT::x86amx, &X86::TILERegClass);
  }
 
  // We want to custom lower some of our intrinsics.
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
  if (!Subtarget.is64Bit()) {
    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
  }
 
  // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
  // handle type legalization for these operations here.
  //
  // FIXME: We really should do custom legalization for addition and
  // subtraction on x86-32 once PR3203 is fixed.  We really can't do much better
  // than generic legalization for 64-bit multiplication-with-overflow, though.
  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
    if (VT == MVT::i64 && !Subtarget.is64Bit())
      continue;
    // Add/Sub/Mul with overflow operations are custom lowered.
    setOperationAction(ISD::SADDO, VT, Custom);
    setOperationAction(ISD::UADDO, VT, Custom);
    setOperationAction(ISD::SSUBO, VT, Custom);
    setOperationAction(ISD::USUBO, VT, Custom);
    setOperationAction(ISD::SMULO, VT, Custom);
    setOperationAction(ISD::UMULO, VT, Custom);
 
    // Support carry in as value rather than glue.
    setOperationAction(ISD::UADDO_CARRY, VT, Custom);
    setOperationAction(ISD::USUBO_CARRY, VT, Custom);
    setOperationAction(ISD::SETCCCARRY, VT, Custom);
    setOperationAction(ISD::SADDO_CARRY, VT, Custom);
    setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
  }
 
  // Combine sin / cos into _sincos_stret if it is available.
  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
 
  if (Subtarget.isTargetWin64()) {
    setOperationAction(ISD::SDIV, MVT::i128, Custom);
    setOperationAction(ISD::UDIV, MVT::i128, Custom);
    setOperationAction(ISD::SREM, MVT::i128, Custom);
    setOperationAction(ISD::UREM, MVT::i128, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
    setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
  }
 
  // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
  // is. We should promote the value to 64-bits to solve this.
  // This is what the CRT headers do - `fmodf` is an inline header
  // function casting to f64 and calling `fmod`.
  if (Subtarget.is32Bit() &&
      (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
    // clang-format off
   for (ISD::NodeType Op :
         {ISD::FACOS,  ISD::STRICT_FACOS,
          ISD::FASIN,  ISD::STRICT_FASIN,
          ISD::FATAN,  ISD::STRICT_FATAN,
          ISD::FATAN2, ISD::STRICT_FATAN2,
          ISD::FCEIL,  ISD::STRICT_FCEIL,
          ISD::FCOS,   ISD::STRICT_FCOS,
          ISD::FCOSH,  ISD::STRICT_FCOSH,
          ISD::FEXP,   ISD::STRICT_FEXP,
          ISD::FFLOOR, ISD::STRICT_FFLOOR,
          ISD::FREM,   ISD::STRICT_FREM,
          ISD::FLOG,   ISD::STRICT_FLOG,
          ISD::FLOG10, ISD::STRICT_FLOG10,
          ISD::FPOW,   ISD::STRICT_FPOW,
          ISD::FSIN,   ISD::STRICT_FSIN,
          ISD::FSINH,  ISD::STRICT_FSINH,
          ISD::FTAN,   ISD::STRICT_FTAN,
          ISD::FTANH,  ISD::STRICT_FTANH,
          // TODO: Add ISD:::STRICT_FMODF too once implemented.
          ISD::FMODF})
     if (isOperationExpandOrLibCall(Op, MVT::f32))
        setOperationAction(Op, MVT::f32, Promote);
  // clang-format on
 
  // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined.  MinGW has
  // it, but it's just a wrapper around ldexp.
  if (Subtarget.isOSWindows()) {
    for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
      if (isOperationExpand(Op, MVT::f32))
        setOperationAction(Op, MVT::f32, Promote);
  }
 
  setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f16, MVT::i16);
  setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f32, MVT::i32);
  setOperationPromotedToType(ISD::ATOMIC_LOAD, MVT::f64, MVT::i64);
 
  setOperationPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16);
  setOperationPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
  setOperationPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64);
 
  // We have target-specific dag combine patterns for the following nodes:
  setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
                       ISD::SCALAR_TO_VECTOR,
                       ISD::INSERT_VECTOR_ELT,
                       ISD::EXTRACT_VECTOR_ELT,
                       ISD::CONCAT_VECTORS,
                       ISD::INSERT_SUBVECTOR,
                       ISD::EXTRACT_SUBVECTOR,
                       ISD::BITCAST,
                       ISD::VSELECT,
                       ISD::SELECT,
                       ISD::SHL,
                       ISD::SRA,
                       ISD::SRL,
                       ISD::OR,
                       ISD::AND,
                       ISD::AVGCEILS,
                       ISD::AVGCEILU,
                       ISD::AVGFLOORS,
                       ISD::AVGFLOORU,
                       ISD::BITREVERSE,
                       ISD::ADD,
                       ISD::SADDSAT,
                       ISD::SSUBSAT,
                       ISD::FADD,
                       ISD::FSUB,
                       ISD::FNEG,
                       ISD::FMA,
                       ISD::STRICT_FMA,
                       ISD::FMINNUM,
                       ISD::FMAXNUM,
                       ISD::PSEUDO_FMIN,
                       ISD::PSEUDO_FMAX,
                       ISD::SUB,
                       ISD::ATOMIC_LOAD,
                       ISD::LOAD,
                       ISD::LRINT,
                       ISD::LLRINT,
                       ISD::MLOAD,
                       ISD::STORE,
                       ISD::MSTORE,
                       ISD::TRUNCATE,
                       ISD::ZERO_EXTEND,
                       ISD::ANY_EXTEND,
                       ISD::SIGN_EXTEND,
                       ISD::SIGN_EXTEND_INREG,
                       ISD::ANY_EXTEND_VECTOR_INREG,
                       ISD::SIGN_EXTEND_VECTOR_INREG,
                       ISD::ZERO_EXTEND_VECTOR_INREG,
                       ISD::VECREDUCE_AND,
                       ISD::VECREDUCE_OR,
                       ISD::VECREDUCE_XOR,
                       ISD::VECREDUCE_MUL,
                       ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP,
                       ISD::FP_TO_SINT,
                       ISD::STRICT_SINT_TO_FP,
                       ISD::STRICT_UINT_TO_FP,
                       ISD::FP_TO_SINT_SAT,
                       ISD::FP_TO_UINT_SAT,
                       ISD::SETCC,
                       ISD::MUL,
                       ISD::UDIV,
                       ISD::SDIV,
                       ISD::UREM,
                       ISD::SREM,
                       ISD::MASKED_UDIV,
                       ISD::MASKED_SDIV,
                       ISD::MASKED_UREM,
                       ISD::MASKED_SREM,
                       ISD::XOR,
                       ISD::MSCATTER,
                       ISD::MGATHER,
                       ISD::FP16_TO_FP,
                       ISD::FP_EXTEND,
                       ISD::STRICT_FP_EXTEND,
                       ISD::FP_ROUND,
                       ISD::STRICT_FP_ROUND,
                       ISD::ROTL,
                       ISD::ROTR,
                       ISD::FSHL,
                       ISD::FSHR,
                       ISD::INTRINSIC_VOID,
                       ISD::INTRINSIC_WO_CHAIN,
                       ISD::INTRINSIC_W_CHAIN});
 
  computeRegisterProperties(Subtarget.getRegisterInfo());
 
  MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
  MaxStoresPerMemsetOptSize = 8;
  MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
  MaxStoresPerMemcpyOptSize = 4;
  MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
  MaxStoresPerMemmoveOptSize = 4;
 
  // TODO: These control memcmp expansion in CGP and could be raised higher, but
  // that needs to benchmarked and balanced with the potential use of vector
  // load/store types (PR33329, PR33914).
  MaxLoadsPerMemcmp = 2;
  MaxLoadsPerMemcmpOptSize = 2;
 
  // Default loop alignment, which can be overridden by -align-loops.
  setPrefLoopAlignment(Align(16));
 
  // An out-of-order CPU can speculatively execute past a predictable branch,
  // but a conditional move could be stalled by an expensive earlier operation.
  PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
  EnableExtLdPromotion = true;
  setPrefFunctionAlignment(Align(16));
 
  verifyIntrinsicTables();
 
  // Default to having -disable-strictnode-mutation on
  IsStrictFPEnabled = true;
}
 
// This has so far only been implemented for 64-bit MachO.
bool X86TargetLowering::useLoadStackGuardNode(const Module &M) const {
  return Subtarget.isTargetMachO() && Subtarget.is64Bit();
}
 
bool X86TargetLowering::useStackGuardMixFP() const {
  // Currently only MSVC CRTs mix the frame pointer into the stack guard value.
  return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
}
 
SDValue X86TargetLowering::emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val,
                                               const SDLoc &DL) const {
  EVT PtrTy = getPointerTy(DAG.getDataLayout());
  unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
  MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
  return SDValue(Node, 0);
}
 
TargetLoweringBase::LegalizeTypeAction
X86TargetLowering::getPreferredVectorAction(MVT VT) const {
  if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
      !Subtarget.hasBWI())
    return TypeSplitVector;
 
  // Since v8f16 is legal, widen anything over v4f16.
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getVectorNumElements() <= 4 && !Subtarget.hasF16C() &&
      VT.getVectorElementType() == MVT::f16)
    return TypeSplitVector;
 
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getVectorElementType() != MVT::i1)
    return TypeWidenVector;
 
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
FastISel *X86TargetLowering::createFastISel(
    FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
    const LibcallLoweringInfo *libcallLowering) const {
  return X86::createFastISel(funcInfo, libInfo, libcallLowering);
}
 
//===----------------------------------------------------------------------===//
//                           Other Lowering Hooks
//===----------------------------------------------------------------------===//
 
bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
                      bool AssumeSingleUse, bool IgnoreAlignment) {
  if (!AssumeSingleUse && !Op.hasOneUse())
    return false;
  if (!ISD::isNormalLoad(Op.getNode()))
    return false;
 
  // If this is an unaligned vector, make sure the target supports folding it.
  auto *Ld = cast<LoadSDNode>(Op.getNode());
  if (!IgnoreAlignment && !Subtarget.hasAVX() &&
      !Subtarget.hasSSEUnalignedMem() && Ld->getValueSizeInBits(0) == 128 &&
      Ld->getAlign() < Align(16))
    return false;
 
  // TODO: If this is a non-temporal load and the target has an instruction
  //       for it, it should not be folded. See "useNonTemporalLoad()".
 
  return true;
}
 
bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
                                          const X86Subtarget &Subtarget,
                                          bool AssumeSingleUse) {
  assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
  if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
    return false;
 
  // We can not replace a wide volatile load with a broadcast-from-memory,
  // because that would narrow the load, which isn't legal for volatiles.
  auto *Ld = cast<LoadSDNode>(Op.getNode());
  return !Ld->isVolatile() ||
         Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
}
 
bool X86::mayFoldIntoStore(SDValue Op) {
  if (!Op.hasOneUse())
    return false;
  // Peek through (oneuse) bitcast users
  SDNode *User = *Op->user_begin();
  while (User->getOpcode() == ISD::BITCAST) {
    if (!User->hasOneUse())
      return false;
    User = *User->user_begin();
  }
  return ISD::isNormalStore(User) || User->getOpcode() == ISD::ATOMIC_STORE;
}
 
bool X86::mayFoldIntoZeroExtend(SDValue Op) {
  if (Op.hasOneUse()) {
    unsigned Opcode = Op.getNode()->user_begin()->getOpcode();
    return (ISD::ZERO_EXTEND == Opcode);
  }
  return false;
}
 
// Return true if its cheap to bitcast this to a vector type.
static bool mayFoldIntoVector(SDValue Op, const SelectionDAG &DAG,
                              const X86Subtarget &Subtarget) {
  if (peekThroughBitcasts(Op).getValueType().isVector())
    return true;
  if (isa<ConstantSDNode>(Op) || isa<ConstantFPSDNode>(Op))
    return true;
 
  EVT VT = Op.getValueType();
  unsigned Opcode = Op.getOpcode();
  if ((VT == MVT::i128 || VT == MVT::i256 || VT == MVT::i512) &&
      DAG.getTargetLoweringInfo().getOperationAction(Opcode, VT) ==
          TargetLowering::LegalizeAction::Custom) {
    // Check for larger than legal scalar integer ops that might have been
    // custom lowered to vector instruction.
    switch (Opcode) {
    case ISD::BITREVERSE:
      return true;
    case ISD::SHL:
    case ISD::SRL:
    case ISD::SRA:
      return mayFoldIntoVector(Op.getOperand(0), DAG, Subtarget);
    case ISD::AND:
    case ISD::OR:
    case ISD::XOR:
    case ISD::ADD:
    case ISD::SUB:
    case ISD::FSHL:
    case ISD::FSHR:
      return mayFoldIntoVector(Op.getOperand(0), DAG, Subtarget) &&
             mayFoldIntoVector(Op.getOperand(1), DAG, Subtarget);
    case ISD::SELECT:
      return mayFoldIntoVector(Op.getOperand(1), DAG, Subtarget) &&
             mayFoldIntoVector(Op.getOperand(2), DAG, Subtarget);
    }
  }
  return X86::mayFoldLoad(Op, Subtarget, /*AssumeSingleUse=*/true,
                          /*IgnoreAlignment=*/true);
}
 
static bool isLogicOp(unsigned Opcode) {
  // TODO: Add support for X86ISD::FAND/FOR/FXOR/FANDN with test coverage.
  return ISD::isBitwiseLogicOp(Opcode) || X86ISD::ANDNP == Opcode;
}
 
static bool isTargetShuffle(unsigned Opcode) {
  switch(Opcode) {
  default: return false;
  case X86ISD::BLENDI:
  case X86ISD::PSHUFB:
  case X86ISD::PSHUFD:
  case X86ISD::PSHUFHW:
  case X86ISD::PSHUFLW:
  case X86ISD::SHUFP:
  case X86ISD::INSERTPS:
  case X86ISD::EXTRQI:
  case X86ISD::INSERTQI:
  case X86ISD::VALIGN:
  case X86ISD::PALIGNR:
  case X86ISD::VSHLDQ:
  case X86ISD::VSRLDQ:
  case X86ISD::MOVLHPS:
  case X86ISD::MOVHLPS:
  case X86ISD::MOVSHDUP:
  case X86ISD::MOVSLDUP:
  case X86ISD::MOVDDUP:
  case X86ISD::MOVSS:
  case X86ISD::MOVSD:
  case X86ISD::MOVSH:
  case X86ISD::UNPCKL:
  case X86ISD::UNPCKH:
  case X86ISD::VBROADCAST:
  case X86ISD::VPERMILPI:
  case X86ISD::VPERMILPV:
  case X86ISD::VPERM2X128:
  case X86ISD::SHUF128:
  case X86ISD::VPERMIL2:
  case X86ISD::VPERMI:
  case X86ISD::VPPERM:
  case X86ISD::VPERMV:
  case X86ISD::VPERMV3:
  case X86ISD::VZEXT_MOVL:
  case X86ISD::COMPRESS:
  case X86ISD::EXPAND:
    return true;
  }
}
 
static bool isTargetShuffleVariableMask(unsigned Opcode) {
  switch (Opcode) {
  default: return false;
  // Target Shuffles.
  case X86ISD::PSHUFB:
  case X86ISD::VPERMILPV:
  case X86ISD::VPERMIL2:
  case X86ISD::VPPERM:
  case X86ISD::VPERMV:
  case X86ISD::VPERMV3:
    return true;
  // 'Faux' Target Shuffles.
  case ISD::OR:
  case ISD::AND:
  case X86ISD::ANDNP:
    return true;
  }
}
 
SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
  int ReturnAddrIndex = FuncInfo->getRAIndex();
 
  if (ReturnAddrIndex == 0) {
    // Set up a frame object for the return address.
    unsigned SlotSize = RegInfo->getSlotSize();
    ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
                                                          -(int64_t)SlotSize,
                                                          false);
    FuncInfo->setRAIndex(ReturnAddrIndex);
  }
 
  return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
}
 
bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
                                       bool HasSymbolicDisplacement) {
  // Offset should fit into 32 bit immediate field.
  if (!isInt<32>(Offset))
    return false;
 
  // If we don't have a symbolic displacement - we don't have any extra
  // restrictions.
  if (!HasSymbolicDisplacement)
    return true;
 
  // We can fold large offsets in the large code model because we always use
  // 64-bit offsets.
  if (CM == CodeModel::Large)
    return true;
 
  // For kernel code model we know that all object resist in the negative half
  // of 32bits address space. We may not accept negative offsets, since they may
  // be just off and we may accept pretty large positive ones.
  if (CM == CodeModel::Kernel)
    return Offset >= 0;
 
  // For other non-large code models we assume that latest small object is 16MB
  // before end of 31 bits boundary. We may also accept pretty large negative
  // constants knowing that all objects are in the positive half of address
  // space.
  return Offset < 16 * 1024 * 1024;
}
 
/// Return true if the condition is an signed comparison operation.
static bool isX86CCSigned(X86::CondCode X86CC) {
  switch (X86CC) {
  default:
    llvm_unreachable("Invalid integer condition!");
  case X86::COND_E:
  case X86::COND_NE:
  case X86::COND_B:
  case X86::COND_A:
  case X86::COND_BE:
  case X86::COND_AE:
    return false;
  case X86::COND_G:
  case X86::COND_GE:
  case X86::COND_L:
  case X86::COND_LE:
    return true;
  }
}
 
static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
  switch (SetCCOpcode) {
  // clang-format off
  default: llvm_unreachable("Invalid integer condition!");
  case ISD::SETEQ:  return X86::COND_E;
  case ISD::SETGT:  return X86::COND_G;
  case ISD::SETGE:  return X86::COND_GE;
  case ISD::SETLT:  return X86::COND_L;
  case ISD::SETLE:  return X86::COND_LE;
  case ISD::SETNE:  return X86::COND_NE;
  case ISD::SETULT: return X86::COND_B;
  case ISD::SETUGT: return X86::COND_A;
  case ISD::SETULE: return X86::COND_BE;
  case ISD::SETUGE: return X86::COND_AE;
  // clang-format on
  }
}
 
/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
/// condition code, returning the condition code and the LHS/RHS of the
/// comparison to make.
static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
                                    bool isFP, SDValue &LHS, SDValue &RHS,
                                    SelectionDAG &DAG) {
  if (!isFP) {
    if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
      if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
        // X > -1   -> X == 0, jump !sign.
        RHS = DAG.getConstant(0, DL, RHS.getValueType());
        return X86::COND_NS;
      }
      if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
        // X < 0   -> X == 0, jump on sign.
        return X86::COND_S;
      }
      if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
        // X >= 0   -> X == 0, jump on !sign.
        return X86::COND_NS;
      }
      if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
        // X < 1   -> X <= 0
        RHS = DAG.getConstant(0, DL, RHS.getValueType());
        return X86::COND_LE;
      }
    }
 
    return TranslateIntegerX86CC(SetCCOpcode);
  }
 
  // First determine if it is required or is profitable to flip the operands.
 
  // If LHS is a foldable load, but RHS is not, flip the condition.
  if (ISD::isNON_EXTLoad(LHS.getNode()) &&
      !ISD::isNON_EXTLoad(RHS.getNode())) {
    SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
    std::swap(LHS, RHS);
  }
 
  switch (SetCCOpcode) {
  default: break;
  case ISD::SETOLT:
  case ISD::SETOLE:
  case ISD::SETUGT:
  case ISD::SETUGE:
    std::swap(LHS, RHS);
    break;
  }
 
  // On a floating point condition, the flags are set as follows:
  // ZF  PF  CF   op
  //  0 | 0 | 0 | X > Y
  //  0 | 0 | 1 | X < Y
  //  1 | 0 | 0 | X == Y
  //  1 | 1 | 1 | unordered
  switch (SetCCOpcode) {
  // clang-format off
  default: llvm_unreachable("Condcode should be pre-legalized away");
  case ISD::SETUEQ:
  case ISD::SETEQ:   return X86::COND_E;
  case ISD::SETOLT:              // flipped
  case ISD::SETOGT:
  case ISD::SETGT:   return X86::COND_A;
  case ISD::SETOLE:              // flipped
  case ISD::SETOGE:
  case ISD::SETGE:   return X86::COND_AE;
  case ISD::SETUGT:              // flipped
  case ISD::SETULT:
  case ISD::SETLT:   return X86::COND_B;
  case ISD::SETUGE:              // flipped
  case ISD::SETULE:
  case ISD::SETLE:   return X86::COND_BE;
  case ISD::SETONE:
  case ISD::SETNE:   return X86::COND_NE;
  case ISD::SETUO:   return X86::COND_P;
  case ISD::SETO:    return X86::COND_NP;
  case ISD::SETOEQ:
  case ISD::SETUNE:  return X86::COND_INVALID;
  // clang-format on
  }
}
 
/// Is there a floating point cmov for the specific X86 condition code?
/// Current x86 isa includes the following FP cmov instructions:
/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
static bool hasFPCMov(unsigned X86CC) {
  switch (X86CC) {
  default:
    return false;
  case X86::COND_B:
  case X86::COND_BE:
  case X86::COND_E:
  case X86::COND_P:
  case X86::COND_A:
  case X86::COND_AE:
  case X86::COND_NE:
  case X86::COND_NP:
    return true;
  }
}
 
static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
  return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
         VT.is512BitVector();
}
 
void X86TargetLowering::getTgtMemIntrinsic(
    SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  IntrinsicInfo Info;
  Info.flags = MachineMemOperand::MONone;
  Info.offset = 0;
 
  const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
  if (!IntrData) {
    switch (Intrinsic) {
    case Intrinsic::x86_aesenc128kl:
    case Intrinsic::x86_aesdec128kl:
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(1);
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
      Info.align = Align(1);
      Info.flags |= MachineMemOperand::MOLoad;
      Infos.push_back(Info);
      return;
    case Intrinsic::x86_aesenc256kl:
    case Intrinsic::x86_aesdec256kl:
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(1);
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
      Info.align = Align(1);
      Info.flags |= MachineMemOperand::MOLoad;
      Infos.push_back(Info);
      return;
    case Intrinsic::x86_aesencwide128kl:
    case Intrinsic::x86_aesdecwide128kl:
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(0);
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
      Info.align = Align(1);
      Info.flags |= MachineMemOperand::MOLoad;
      Infos.push_back(Info);
      return;
    case Intrinsic::x86_aesencwide256kl:
    case Intrinsic::x86_aesdecwide256kl:
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(0);
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
      Info.align = Align(1);
      Info.flags |= MachineMemOperand::MOLoad;
      Infos.push_back(Info);
      return;
    case Intrinsic::x86_cmpccxadd32:
    case Intrinsic::x86_cmpccxadd64:
    case Intrinsic::x86_atomic_bts:
    case Intrinsic::x86_atomic_btc:
    case Intrinsic::x86_atomic_btr: {
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(0);
      unsigned Size = I.getType()->getScalarSizeInBits();
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
      Info.align = Align(Size);
      Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                    MachineMemOperand::MOVolatile;
      Infos.push_back(Info);
      return;
    }
    case Intrinsic::x86_atomic_bts_rm:
    case Intrinsic::x86_atomic_btc_rm:
    case Intrinsic::x86_atomic_btr_rm: {
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(0);
      unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
      Info.align = Align(Size);
      Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                    MachineMemOperand::MOVolatile;
      Infos.push_back(Info);
      return;
    }
    case Intrinsic::x86_aadd32:
    case Intrinsic::x86_aadd64:
    case Intrinsic::x86_aand32:
    case Intrinsic::x86_aand64:
    case Intrinsic::x86_aor32:
    case Intrinsic::x86_aor64:
    case Intrinsic::x86_axor32:
    case Intrinsic::x86_axor64:
    case Intrinsic::x86_atomic_add_cc:
    case Intrinsic::x86_atomic_sub_cc:
    case Intrinsic::x86_atomic_or_cc:
    case Intrinsic::x86_atomic_and_cc:
    case Intrinsic::x86_atomic_xor_cc: {
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.ptrVal = I.getArgOperand(0);
      unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
      Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
      Info.align = Align(Size);
      Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                    MachineMemOperand::MOVolatile;
      Infos.push_back(Info);
      return;
    }
    }
    return;
  }
 
  switch (IntrData->Type) {
  case TRUNCATE_TO_MEM_VI8:
  case TRUNCATE_TO_MEM_VI16:
  case TRUNCATE_TO_MEM_VI32: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = I.getArgOperand(0);
    MVT VT  = MVT::getVT(I.getArgOperand(1)->getType());
    MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
    if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
      ScalarVT = MVT::i8;
    else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
      ScalarVT = MVT::i16;
    else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
      ScalarVT = MVT::i32;
 
    Info.memVT = VT.changeElementType(ScalarVT);
    Info.align = Align(1);
    Info.flags |= MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  case GATHER:
  case GATHER_AVX2: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.ptrVal = nullptr;
    MVT DataVT = MVT::getVT(I.getType());
    MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
    unsigned NumElts = std::min(DataVT.getVectorNumElements(),
                                IndexVT.getVectorNumElements());
    Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
    Info.align = Align(1);
    Info.flags |= MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  }
  case SCATTER: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = nullptr;
    MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
    MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
    unsigned NumElts = std::min(DataVT.getVectorNumElements(),
                                IndexVT.getVectorNumElements());
    Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
    Info.align = Align(1);
    Info.flags |= MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  default:
    return;
  }
}
 
/// Returns true if the target can instruction select the
/// specified FP immediate natively. If false, the legalizer will
/// materialize the FP immediate as a load from a constant pool.
bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                     bool ForCodeSize) const {
  for (const APFloat &FPImm : LegalFPImmediates)
    if (Imm.bitwiseIsEqual(FPImm))
      return true;
  return false;
}
 
bool X86TargetLowering::shouldReduceLoadWidth(
    SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
    std::optional<unsigned> ByteOffset) const {
  assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
 
  auto PeekThroughOneUserBitcasts = [](const SDNode *N) {
    while (N->getOpcode() == ISD::BITCAST && N->hasOneUse())
      N = *N->user_begin();
    return N;
  };
 
  // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
  // relocation target a movq or addq instruction: don't let the load shrink.
  SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
  if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
    if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
      return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
 
  // If this is a (1) 128-bit or wider vector load, and any use will be used by
  // a legal full width instruction, then the load can typically be memory
  // folded into that instruction, so it's probably not worth splitting the
  // load. Additionally, for (2) AVX vector loads with (3) multiple uses where
  // (4) all of those uses are extracted directly into a store, the extract +
  // store can be store-folded, so it's again not worth splitting.
  EVT VT = Load->getValueType(0);
  if (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) {
    bool FullWidthUse = false;
    // The extract + store folding only helps the AVX split case, which requires
    // multiple uses of the load.
    bool AllExtractStores = (VT.is256BitVector() || VT.is512BitVector()) &&
                            !SDValue(Load, 0).hasOneUse();
    for (SDUse &Use : Load->uses()) {
      // Skip uses of the chain value. Result 0 of the node is the load value.
      if (Use.getResNo() != 0)
        continue;
 
      const SDNode *User = PeekThroughOneUserBitcasts(Use.getUser());
 
      // If this use is an extract + store, it's probably not worth splitting.
      if (AllExtractStores && User->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
          all_of(User->uses(), [&](const SDUse &U) {
            const SDNode *Inner = PeekThroughOneUserBitcasts(U.getUser());
            return Inner->getOpcode() == ISD::STORE;
          }))
        continue;
 
      AllExtractStores = false;
 
      // If any use is a full width legal/target bin op, then assume its legal
      // and won't split.
      if (isBinOp(User->getOpcode()) &&
          (isOperationLegal(User->getOpcode(), User->getValueType(0)) ||
           User->getOpcode() > ISD::BUILTIN_OP_END))
        FullWidthUse = true;
    }
 
    if (AllExtractStores)
      return false;
 
    // If we have an user that uses the full vector width, then this use is
    // only worth splitting if the offset isn't 0 (to avoid an
    // EXTRACT_SUBVECTOR) or we're loading a scalar integer.
    if (FullWidthUse)
      return (ByteOffset.value_or(0) > 0) || NewVT.isScalarInteger();
  }
 
  return true;
}
 
/// Returns true if it is beneficial to convert a load of a constant
/// to just the constant itself.
bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                          Type *Ty) const {
  assert(Ty->isIntegerTy());
 
  unsigned BitSize = Ty->getPrimitiveSizeInBits();
  if (BitSize == 0 || BitSize > 64)
    return false;
  return true;
}
 
bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
  // If we are using XMM registers in the ABI and the condition of the select is
  // a floating-point compare and we have blendv or conditional move, then it is
  // cheaper to select instead of doing a cross-register move and creating a
  // load that depends on the compare result.
  bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
  return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
}
 
bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
  // TODO: It might be a win to ease or lift this restriction, but the generic
  // folds in DAGCombiner conflict with vector folds for an AVX512 target.
  if (VT.isVector() && Subtarget.hasAVX512())
    return false;
 
  return true;
}
 
bool X86TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &, EVT VT,
                                                        EVT) const {
  // With CCMP, keep and/or(setcc, setcc) trees intact so LowerSELECT can
  // emit them as CCMP chains rather than splitting into chained selects.
  return !(Subtarget.hasCCMP() && VT.isScalarInteger());
}
 
bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
                                               SDValue C) const {
  // TODO: We handle scalars using custom code, but generic combining could make
  // that unnecessary.
  APInt MulC;
  if (!ISD::isConstantSplatVector(C.getNode(), MulC))
    return false;
 
  if (VT.isVector() && VT.getScalarSizeInBits() == 8) {
    // Check whether a vXi8 multiply can be decomposed into two shifts
    // (decomposing 2^m ± 2^n as 2^(a+b) ± 2^b). Similar to
    // DAGCombiner::visitMUL, consider the constant `2` decomposable as
    // (2^0 + 1).
    APInt ShiftedMulC = MulC.abs();
    unsigned TZeros = ShiftedMulC == 2 ? 0 : ShiftedMulC.countr_zero();
    ShiftedMulC.lshrInPlace(TZeros);
    if ((ShiftedMulC - 1).isPowerOf2() || (ShiftedMulC + 1).isPowerOf2())
      return true;
  }
 
  // Find the type this will be legalized too. Otherwise we might prematurely
  // convert this to shl+add/sub and then still have to type legalize those ops.
  // Another choice would be to defer the decision for illegal types until
  // after type legalization. But constant splat vectors of i64 can't make it
  // through type legalization on 32-bit targets so we would need to special
  // case vXi64.
  while (getTypeAction(Context, VT) != TypeLegal)
    VT = getTypeToTransformTo(Context, VT);
 
  // If vector multiply is legal, assume that's faster than shl + add/sub.
  // Multiply is a complex op with higher latency and lower throughput in
  // most implementations, sub-vXi32 vector multiplies are always fast,
  // vXi32 mustn't have a SlowMULLD implementation, and anything larger (vXi64)
  // is always going to be slow.
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 &&
      (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow()))
    return false;
 
  // shl+add, shl+sub, shl+add+neg
  return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
         (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
}
 
TargetLowering::ExtractSubvectorCost
X86TargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
                                           unsigned Index) const {
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return ExtractSubvectorCost::Expensive;
 
  // Mask vectors support all subregister combinations and operations that
  // extract half of vector.
  if (ResVT.getVectorElementType() == MVT::i1) {
    if (Index == 0 || ((ResVT.getSizeInBits() * 2 == SrcVT.getSizeInBits()) &&
                       (Index == ResVT.getVectorNumElements())))
      return ExtractSubvectorCost::Free;
    return ExtractSubvectorCost::Expensive;
  }
 
  if (Index == 0)
    return ExtractSubvectorCost::Free;
  else if ((Index % ResVT.getVectorNumElements()) == 0)
    return ExtractSubvectorCost::Cheap;
  return ExtractSubvectorCost::Expensive;
}
 
bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
  unsigned Opc = VecOp.getOpcode();
 
  // Assume target opcodes can't be scalarized.
  // TODO - do we have any exceptions?
  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
    return false;
 
  // If the vector op is not supported, try to convert to scalar.
  EVT VecVT = VecOp.getValueType();
  if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
    return true;
 
  // If the vector op is supported, but the scalar op is not, the transform may
  // not be worthwhile.
  EVT ScalarVT = VecVT.getScalarType();
  return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
}
 
bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT,
                                             bool) const {
  // TODO: Allow vectors?
  if (VT.isVector())
    return false;
  return VT.isSimple() || !isOperationExpand(Opcode, VT);
}
 
bool X86TargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
  // Speculate cttz only if we can directly use TZCNT/CMOV, can promote to
  // i32/i64 or can rely on BSF passthrough value.
  return Subtarget.hasBMI() || Subtarget.canUseCMOV() ||
         Subtarget.hasBitScanPassThrough() ||
         (!Ty->isVectorTy() &&
          Ty->getScalarSizeInBits() < (Subtarget.is64Bit() ? 64u : 32u));
}
 
bool X86TargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
  // Speculate ctlz only if we can directly use LZCNT/CMOV, or can rely on BSR
  // passthrough value.
  return Subtarget.hasLZCNT() || Subtarget.canUseCMOV() ||
         Subtarget.hasBitScanPassThrough();
}
 
bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const {
  // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
  // expensive than a straight movsd. On the other hand, it's important to
  // shrink long double fp constant since fldt is very slow.
  return !Subtarget.hasSSE2() || VT == MVT::f80;
}
 
bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
  return (VT == MVT::f64 && Subtarget.hasSSE2()) ||
         (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
}
 
bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
                                                const SelectionDAG &DAG,
                                                const MachineMemOperand &MMO) const {
  if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
      BitcastVT.getVectorElementType() == MVT::i1)
    return false;
 
  if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
    return false;
 
  if (LoadVT.isVector() && BitcastVT.isVector()) {
    // If both types are legal vectors, it's always ok to convert them.
    // Don't convert to an illegal type.
    if (isTypeLegal(LoadVT))
      return isTypeLegal(BitcastVT);
  }
 
  // If we have a large vector type (even if illegal), don't bitcast to large
  // (illegal) scalar types. Better to load fewer vectors and extract.
  if (LoadVT.isVector() && !BitcastVT.isVector() && LoadVT.isInteger() &&
      BitcastVT.isInteger() && (LoadVT.getSizeInBits() % 128) == 0)
    return false;
 
  return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
}
 
bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
                                         const MachineFunction &MF) const {
  // Do not merge to float value size (128 bytes) if no implicit
  // float attribute is set.
  bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
 
  if (NoFloat) {
    unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
    return (MemVT.getSizeInBits() <= MaxIntSize);
  }
  // Make sure we don't merge greater than our preferred vector
  // width.
  if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
    return false;
 
  return true;
}
 
bool X86TargetLowering::isCtlzFast() const {
  return Subtarget.hasFastLZCNT();
}
 
bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
    const Instruction &AndI) const {
  return true;
}
 
bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
  // Scalar integer and-not compares are efficiently handled by NOT+TEST (or
  // BMI ANDN).
  return Y.getValueType().isScalarInteger();
}
 
bool X86TargetLowering::hasAndNot(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  if (!VT.isVector()) {
    if (!Subtarget.hasBMI())
      return false;
 
    // There are only 32-bit and 64-bit forms for 'andn'.
    if (VT != MVT::i32 && VT != MVT::i64)
      return false;
    return !isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque();
  }
 
  // Vector.
  if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)
    return false;
 
  if (VT == MVT::v4i32)
    return true;
 
  return Subtarget.hasSSE2();
}
 
bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
  return X.getValueType().isScalarInteger(); // 'bt'
}
 
bool X86TargetLowering::
    shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
        SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
        unsigned OldShiftOpcode, unsigned NewShiftOpcode,
        SelectionDAG &DAG) const {
  // Does baseline recommend not to perform the fold by default?
  if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
          X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
    return false;
  // For scalars this transform is always beneficial.
  if (X.getValueType().isScalarInteger())
    return true;
  // If all the shift amounts are identical, then transform is beneficial even
  // with rudimentary SSE2 shifts.
  if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
    return true;
  // If we have AVX2 with it's powerful shift operations, then it's also good.
  if (Subtarget.hasAVX2())
    return true;
  // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'.
  return NewShiftOpcode == ISD::SHL;
}
 
unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
    EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
    const APInt &ShiftOrRotateAmt, const std::optional<APInt> &AndMask) const {
  if (!VT.isInteger())
    return ShiftOpc;
 
  bool PreferRotate = false;
  if (VT.isVector()) {
    // For vectors, if we have rotate instruction support, then its definetly
    // best. Otherwise its not clear what the best so just don't make changed.
    PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 ||
                                             VT.getScalarType() == MVT::i64);
  } else {
    // For scalar, if we have bmi prefer rotate for rorx. Otherwise prefer
    // rotate unless we have a zext mask+shr.
    PreferRotate = Subtarget.hasBMI2();
    if (!PreferRotate) {
      unsigned MaskBits =
          VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue();
      PreferRotate = (MaskBits != 8) && (MaskBits != 16) && (MaskBits != 32);
    }
  }
 
  if (ShiftOpc == ISD::SHL || ShiftOpc == ISD::SRL) {
    assert(AndMask.has_value() && "Null andmask when querying about shift+and");
 
    if (PreferRotate && MayTransformRotate)
      return ISD::ROTL;
 
    // If vector we don't really get much benefit swapping around constants.
    // Maybe we could check if the DAG has the flipped node already in the
    // future.
    if (VT.isVector())
      return ShiftOpc;
 
    // See if the beneficial to swap shift type.
    if (ShiftOpc == ISD::SHL) {
      // If the current setup has imm64 mask, then inverse will have
      // at least imm32 mask (or be zext i32 -> i64).
      if (VT == MVT::i64)
        return AndMask->getSignificantBits() > 32 ? (unsigned)ISD::SRL
                                                  : ShiftOpc;
 
      // We can only benefit if req at least 7-bit for the mask. We
      // don't want to replace shl of 1,2,3 as they can be implemented
      // with lea/add.
      return ShiftOrRotateAmt.uge(7) ? (unsigned)ISD::SRL : ShiftOpc;
    }
 
    if (VT == MVT::i64)
      // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is
      // extremely efficient.
      return AndMask->getSignificantBits() > 33 ? (unsigned)ISD::SHL : ShiftOpc;
 
    // Keep small shifts as shl so we can generate add/lea.
    return ShiftOrRotateAmt.ult(7) ? (unsigned)ISD::SHL : ShiftOpc;
  }
 
  // We prefer rotate for vectors of if we won't get a zext mask with SRL
  // (PreferRotate will be set in the latter case).
  if (PreferRotate || !MayTransformRotate || VT.isVector())
    return ShiftOpc;
 
  // Non-vector type and we have a zext mask with SRL.
  return ISD::SRL;
}
 
TargetLoweringBase::CondMergingParams
X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
                                                 const Value *Lhs,
                                                 const Value *Rhs,
                                                 const Function *) const {
  using namespace llvm::PatternMatch;
  int BaseCost = BrMergingBaseCostThresh.getValue();
  // With CCMP, branches can be merged in a more efficient way.
  if (BaseCost >= 0 && Subtarget.hasCCMP())
    BaseCost += BrMergingCcmpBias;
  // a == b && a == c is a fast pattern on x86.
  if (BaseCost >= 0 && Opc == Instruction::And &&
      match(Lhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())) &&
      match(Rhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())))
    BaseCost += 1;
 
  // For OR conditions with EQ comparisons, prefer splitting into branches
  // (unless CCMP is available). OR+EQ cannot be optimized via bitwise ops,
  // unlike OR+NE which becomes (P|Q)!=0. Similarly, don't split signed
  // comparisons (SLT, SGT) that can be optimized.
  if (BaseCost >= 0 && !Subtarget.hasCCMP() && Opc == Instruction::Or &&
      match(Lhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())) &&
      match(Rhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())))
    return {-1, -1, -1};
 
  return {BaseCost, BrMergingLikelyBias.getValue(),
          BrMergingUnlikelyBias.getValue()};
}
 
bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const {
  return N->getOpcode() != ISD::FP_EXTEND;
}
 
bool X86TargetLowering::shouldFoldConstantShiftPairToMask(
    const SDNode *N) const {
  assert(((N->getOpcode() == ISD::SHL &&
           N->getOperand(0).getOpcode() == ISD::SRL) ||
          (N->getOpcode() == ISD::SRL &&
           N->getOperand(0).getOpcode() == ISD::SHL)) &&
         "Expected shift-shift mask");
  // TODO: Should we always create i64 masks? Or only folded immediates?
  EVT VT = N->getValueType(0);
  if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) ||
      (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) {
    // Only fold if the shift values are equal - so it folds to AND.
    // TODO - we should fold if either is a non-uniform vector but we don't do
    // the fold for non-splats yet.
    return N->getOperand(1) == N->getOperand(0).getOperand(1);
  }
  return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N);
}
 
bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  // For vectors, we don't have a preference, but we probably want a mask.
  if (VT.isVector())
    return false;
 
  unsigned MaxWidth = Subtarget.is64Bit() ? 64 : 32;
  return VT.getScalarSizeInBits() <= MaxWidth;
}
 
TargetLowering::ShiftLegalizationStrategy
X86TargetLowering::preferredShiftLegalizationStrategy(
    SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
  if (DAG.getMachineFunction().getFunction().hasMinSize() &&
      !Subtarget.isOSWindows())
    return ShiftLegalizationStrategy::LowerToLibcall;
  return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
                                                            ExpansionFactor);
}
 
bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {
  // Any legal vector type can be splatted more efficiently than
  // loading/spilling from memory.
  return isTypeLegal(VT);
}
 
MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {
  MVT VT = MVT::getIntegerVT(NumBits);
  if (isTypeLegal(VT))
    return VT;
 
  // PMOVMSKB can handle this.
  if (NumBits == 128 && isTypeLegal(MVT::v16i8))
    return MVT::v16i8;
 
  // VPMOVMSKB can handle this.
  if (NumBits == 256 && isTypeLegal(MVT::v32i8))
    return MVT::v32i8;
 
  // TODO: Allow 64-bit type for 32-bit target.
  // TODO: 512-bit types should be allowed, but make sure that those
  // cases are handled in combineVectorSizedSetCCEquality().
 
  return MVT::INVALID_SIMPLE_VALUE_TYPE;
}
 
/// Val is the undef sentinel value or equal to the specified value.
static bool isUndefOrEqual(int Val, int CmpVal) {
  return ((Val == SM_SentinelUndef) || (Val == CmpVal));
}
 
/// Return true if every element in Mask is the undef sentinel value or equal to
/// the specified value.
static bool isUndefOrEqual(ArrayRef<int> Mask, int CmpVal) {
  return llvm::all_of(Mask, [CmpVal](int M) {
    return (M == SM_SentinelUndef) || (M == CmpVal);
  });
}
 
/// Return true if every element in Mask, beginning from position Pos and ending
/// in Pos+Size is the undef sentinel value or equal to the specified value.
static bool isUndefOrEqualInRange(ArrayRef<int> Mask, int CmpVal, unsigned Pos,
                                  unsigned Size) {
  return llvm::all_of(Mask.slice(Pos, Size),
                      [CmpVal](int M) { return isUndefOrEqual(M, CmpVal); });
}
 
/// Val is either the undef or zero sentinel value.
static bool isUndefOrZero(int Val) {
  return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
}
 
/// Return true if every element in Mask, beginning from position Pos and ending
/// in Pos+Size is the undef sentinel value.
static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
  return llvm::all_of(Mask.slice(Pos, Size), equal_to(SM_SentinelUndef));
}
 
/// Return true if the mask creates a vector whose lower half is undefined.
static bool isUndefLowerHalf(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  return isUndefInRange(Mask, 0, NumElts / 2);
}
 
/// Return true if the mask creates a vector whose upper half is undefined.
static bool isUndefUpperHalf(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
}
 
/// Return true if Val falls within the specified range (L, H].
static bool isInRange(int Val, int Low, int Hi) {
  return (Val >= Low && Val < Hi);
}
 
/// Return true if the value of any element in Mask falls within the specified
/// range (L, H].
static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
  return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
}
 
/// Return true if the value of any element in Mask is the zero sentinel value.
static bool isAnyZero(ArrayRef<int> Mask) {
  return llvm::any_of(Mask, equal_to(SM_SentinelZero));
}
 
/// Return true if Val is undef or if its value falls within the
/// specified range (L, H].
static bool isUndefOrInRange(int Val, int Low, int Hi) {
  return (Val == SM_SentinelUndef) || isInRange(Val, Low, Hi);
}
 
/// Return true if every element in Mask is undef or if its value
/// falls within the specified range (L, H].
static bool isUndefOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
  return llvm::all_of(
      Mask, [Low, Hi](int M) { return isUndefOrInRange(M, Low, Hi); });
}
 
/// Return true if Val is undef, zero or if its value falls within the
/// specified range (L, H].
static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi) {
  return isUndefOrZero(Val) || isInRange(Val, Low, Hi);
}
 
/// Return true if every element in Mask is undef, zero or if its value
/// falls within the specified range (L, H].
static bool isUndefOrZeroOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
  return llvm::all_of(
      Mask, [Low, Hi](int M) { return isUndefOrZeroOrInRange(M, Low, Hi); });
}
 
/// Return true if every element in Mask, is an in-place blend/select mask or is
/// undef.
[[maybe_unused]] static bool isBlendOrUndef(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  for (auto [I, M] : enumerate(Mask))
    if (!isUndefOrEqual(M, I) && !isUndefOrEqual(M, I + NumElts))
      return false;
  return true;
}
 
/// Return true if every element in Mask, beginning
/// from position Pos and ending in Pos + Size, falls within the specified
/// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, unsigned Pos,
                                       unsigned Size, int Low, int Step = 1) {
  for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
    if (!isUndefOrEqual(Mask[i], Low))
      return false;
  return true;
}
 
/// Return true if every element in Mask, beginning
/// from position Pos and ending in Pos+Size, falls within the specified
/// sequential range (Low, Low+Size], or is undef or is zero.
static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
                                             unsigned Size, int Low,
                                             int Step = 1) {
  for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
    if (!isUndefOrZero(Mask[i]) && Mask[i] != Low)
      return false;
  return true;
}
 
/// Return true if every element in Mask, beginning
/// from position Pos and ending in Pos+Size is undef or is zero.
static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
                                 unsigned Size) {
  return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
}
 
/// Return true if every element of a single input is referenced by the shuffle
/// mask. i.e. it just permutes them all.
static bool isCompletePermute(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  APInt DemandedElts = APInt::getZero(NumElts);
  for (int M : Mask)
    if (isInRange(M, 0, NumElts))
      DemandedElts.setBit(M);
  return DemandedElts.isAllOnes();
}
 
/// Helper function to test whether a shuffle mask could be
/// simplified by widening the elements being shuffled.
///
/// Appends the mask for wider elements in WidenedMask if valid. Otherwise
/// leaves it in an unspecified state.
///
/// NOTE: This must handle normal vector shuffle masks and *target* vector
/// shuffle masks. The latter have the special property of a '-2' representing
/// a zero-ed lane of a vector.
static bool canWidenShuffleElements(ArrayRef<int> Mask,
                                    SmallVectorImpl<int> &WidenedMask) {
  WidenedMask.assign(Mask.size() / 2, 0);
  for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
    int M0 = Mask[i];
    int M1 = Mask[i + 1];
 
    // If both elements are undef, its trivial.
    if (M0 == SM_SentinelUndef && M1 == SM_SentinelUndef) {
      WidenedMask[i / 2] = SM_SentinelUndef;
      continue;
    }
 
    // Check for an undef mask and a mask value properly aligned to fit with
    // a pair of values. If we find such a case, use the non-undef mask's value.
    if (M0 == SM_SentinelUndef && M1 >= 0 && (M1 % 2) == 1) {
      WidenedMask[i / 2] = M1 / 2;
      continue;
    }
    if (M1 == SM_SentinelUndef && M0 >= 0 && (M0 % 2) == 0) {
      WidenedMask[i / 2] = M0 / 2;
      continue;
    }
 
    // When zeroing, we need to spread the zeroing across both lanes to widen.
    if (M0 == SM_SentinelZero || M1 == SM_SentinelZero) {
      if ((M0 == SM_SentinelZero || M0 == SM_SentinelUndef) &&
          (M1 == SM_SentinelZero || M1 == SM_SentinelUndef)) {
        WidenedMask[i / 2] = SM_SentinelZero;
        continue;
      }
      return false;
    }
 
    // Finally check if the two mask values are adjacent and aligned with
    // a pair.
    if (M0 != SM_SentinelUndef && (M0 % 2) == 0 && (M0 + 1) == M1) {
      WidenedMask[i / 2] = M0 / 2;
      continue;
    }
 
    // Otherwise we can't safely widen the elements used in this shuffle.
    return false;
  }
  assert(WidenedMask.size() == Mask.size() / 2 &&
         "Incorrect size of mask after widening the elements!");
 
  return true;
}
 
static bool canWidenShuffleElements(ArrayRef<int> Mask,
                                    const APInt &Zeroable,
                                    bool V2IsZero,
                                    SmallVectorImpl<int> &WidenedMask) {
  // Create an alternative mask with info about zeroable elements.
  // Here we do not set undef elements as zeroable.
  SmallVector<int, 64> ZeroableMask(Mask);
  if (V2IsZero) {
    assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
    for (int i = 0, Size = Mask.size(); i != Size; ++i)
      if (Mask[i] != SM_SentinelUndef && Zeroable[i])
        ZeroableMask[i] = SM_SentinelZero;
  }
  return canWidenShuffleElements(ZeroableMask, WidenedMask);
}
 
static bool canWidenShuffleElements(ArrayRef<int> Mask) {
  SmallVector<int, 32> WidenedMask;
  return canWidenShuffleElements(Mask, WidenedMask);
}
 
// Attempt to narrow/widen shuffle mask until it matches the target number of
// elements.
static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
                                 SmallVectorImpl<int> &ScaledMask) {
  unsigned NumSrcElts = Mask.size();
  assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) &&
         "Illegal shuffle scale factor");
 
  // Narrowing is guaranteed to work.
  if (NumDstElts >= NumSrcElts) {
    int Scale = NumDstElts / NumSrcElts;
    llvm::narrowShuffleMaskElts(Scale, Mask, ScaledMask);
    return true;
  }
 
  // We have to repeat the widening until we reach the target size, but we can
  // split out the first widening as it sets up ScaledMask for us.
  if (canWidenShuffleElements(Mask, ScaledMask)) {
    while (ScaledMask.size() > NumDstElts) {
      SmallVector<int, 16> WidenedMask;
      if (!canWidenShuffleElements(ScaledMask, WidenedMask))
        return false;
      ScaledMask = std::move(WidenedMask);
    }
    return true;
  }
 
  return false;
}
 
static bool canScaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts) {
  SmallVector<int, 32> ScaledMask;
  return scaleShuffleElements(Mask, NumDstElts, ScaledMask);
}
 
// Helper to grow the shuffle mask for a larger value type.
// NOTE: This is different to scaleShuffleElements which is a same size type.
static void growShuffleMask(ArrayRef<int> SrcMask,
                            SmallVectorImpl<int> &DstMask,
                            unsigned SrcSizeInBits, unsigned DstSizeInBits) {
  assert(DstMask.empty() && "Expected an empty shuffle mas");
  assert((DstSizeInBits % SrcSizeInBits) == 0 && "Illegal shuffle scale");
  unsigned Scale = DstSizeInBits / SrcSizeInBits;
  unsigned NumSrcElts = SrcMask.size();
  DstMask.assign(SrcMask.begin(), SrcMask.end());
  for (int &M : DstMask) {
    if (M < 0)
      continue;
    M = (M % NumSrcElts) + ((M / NumSrcElts) * Scale * NumSrcElts);
  }
  DstMask.append((Scale - 1) * NumSrcElts, SM_SentinelUndef);
}
 
/// Returns true if Elt is a constant zero or a floating point constant +0.0.
bool X86::isZeroNode(SDValue Elt) {
  return isNullConstant(Elt) || isNullFPConstant(Elt);
}
 
// Build a vector of constants.
// Use an UNDEF node if MaskElt == -1.
// Split 64-bit constants in the 32-bit mode.
static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
                              const SDLoc &dl, bool IsMask = false) {
 
  SmallVector<SDValue, 32>  Ops;
  bool Split = false;
 
  MVT ConstVecVT = VT;
  unsigned NumElts = VT.getVectorNumElements();
  bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
  if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
    ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
    Split = true;
  }
 
  MVT EltVT = ConstVecVT.getVectorElementType();
  for (unsigned i = 0; i < NumElts; ++i) {
    bool IsUndef = Values[i] < 0 && IsMask;
    SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
      DAG.getConstant(Values[i], dl, EltVT);
    Ops.push_back(OpNode);
    if (Split)
      Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
                    DAG.getConstant(0, dl, EltVT));
  }
  SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
  if (Split)
    ConstsNode = DAG.getBitcast(VT, ConstsNode);
  return ConstsNode;
}
 
static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
                              MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
  assert(Bits.size() == Undefs.getBitWidth() &&
         "Unequal constant and undef arrays");
  SmallVector<SDValue, 32> Ops;
  bool Split = false;
 
  MVT ConstVecVT = VT;
  unsigned NumElts = VT.getVectorNumElements();
  bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
  if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
    ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
    Split = true;
  }
 
  MVT EltVT = ConstVecVT.getVectorElementType();
  MVT EltIntVT = EltVT.changeTypeToInteger();
  for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
    if (Undefs[i]) {
      Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
      continue;
    }
    const APInt &V = Bits[i];
    assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes");
    if (Split) {
      Ops.push_back(DAG.getConstant(V.extractBits(32, 0), dl, EltVT));
      Ops.push_back(DAG.getConstant(V.extractBits(32, 32), dl, EltVT));
    } else {
      Ops.push_back(DAG.getBitcast(EltVT, DAG.getConstant(V, dl, EltIntVT)));
    }
  }
 
  SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
  return DAG.getBitcast(VT, ConstsNode);
}
 
static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
                              SelectionDAG &DAG, const SDLoc &dl) {
  APInt Undefs = APInt::getZero(Bits.size());
  return getConstVector(Bits, Undefs, VT, DAG, dl);
}
 
/// Returns a vector of specified type with all zero elements.
static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
                             SelectionDAG &DAG, const SDLoc &dl) {
  assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||
          VT.getVectorElementType() == MVT::i1) &&
         "Unexpected vector type");
 
  // Try to build SSE/AVX zero vectors as <N x i32> bitcasted to their dest
  // type. This ensures they get CSE'd. But if the integer type is not
  // available, use a floating-point +0.0 instead.
  SDValue Vec;
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (!Subtarget.hasSSE2() && VT.is128BitVector()) {
    Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
  } else if (VT.isFloatingPoint() &&
             TLI.isTypeLegal(VT.getVectorElementType())) {
    Vec = DAG.getConstantFP(+0.0, dl, VT);
  } else if (VT.getVectorElementType() == MVT::i1) {
    assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
           "Unexpected vector type");
    Vec = DAG.getConstant(0, dl, VT);
  } else {
    unsigned Num32BitElts = VT.getSizeInBits() / 32;
    Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
  }
  return DAG.getBitcast(VT, Vec);
}
 
// Helper to determine if the ops are all the extracted subvectors come from a
// single source. If we allow commute they don't have to be in order (Lo/Hi).
static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute) {
  if (LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
      RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
      LHS.getValueType() != RHS.getValueType() ||
      LHS.getOperand(0) != RHS.getOperand(0))
    return SDValue();
 
  SDValue Src = LHS.getOperand(0);
  if (Src.getValueSizeInBits() != (LHS.getValueSizeInBits() * 2))
    return SDValue();
 
  unsigned NumElts = LHS.getValueType().getVectorNumElements();
  if ((LHS.getConstantOperandAPInt(1) == 0 &&
       RHS.getConstantOperandAPInt(1) == NumElts) ||
      (AllowCommute && RHS.getConstantOperandAPInt(1) == 0 &&
       LHS.getConstantOperandAPInt(1) == NumElts))
    return Src;
 
  return SDValue();
}
 
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
                                const SDLoc &dl, unsigned vectorWidth) {
  EVT VT = Vec.getValueType();
  EVT ElVT = VT.getVectorElementType();
  unsigned ResultNumElts =
      (VT.getVectorNumElements() * vectorWidth) / VT.getSizeInBits();
  EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, ResultNumElts);
 
  assert(ResultVT.getSizeInBits() == vectorWidth &&
         "Illegal subvector extraction");
 
  // Extract the relevant vectorWidth bits.  Generate an EXTRACT_SUBVECTOR
  unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
  assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
 
  // This is the index of the first element of the vectorWidth-bit chunk
  // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
  IdxVal &= ~(ElemsPerChunk - 1);
 
  // If the input is a buildvector just emit a smaller one.
  if (Vec.getOpcode() == ISD::BUILD_VECTOR)
    return DAG.getBuildVector(ResultVT, dl,
                              Vec->ops().slice(IdxVal, ElemsPerChunk));
 
  // Check if we're extracting the upper undef of a widening pattern.
  if (Vec.getOpcode() == ISD::INSERT_SUBVECTOR && Vec.getOperand(0).isUndef() &&
      Vec.getOperand(1).getValueType().getVectorNumElements() <= IdxVal &&
      isNullConstant(Vec.getOperand(2)))
    return DAG.getUNDEF(ResultVT);
 
  return DAG.getExtractSubvector(dl, ResultVT, Vec, IdxVal);
}
 
/// Generate a DAG to grab 128-bits from a vector > 128 bits.  This
/// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
/// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
/// instructions or a simple subregister reference. Idx is an index in the
/// 128 bits we want.  It need not be aligned to a 128-bit boundary.  That makes
/// lowering EXTRACT_VECTOR_ELT operations easier.
static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
                                   SelectionDAG &DAG, const SDLoc &dl) {
  assert((Vec.getValueType().is256BitVector() ||
          Vec.getValueType().is512BitVector()) &&
         "Unexpected vector size!");
  return extractSubVector(Vec, IdxVal, DAG, dl, 128);
}
 
/// Generate a DAG to grab 256-bits from a 512-bit vector.
static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
                                   SelectionDAG &DAG, const SDLoc &dl) {
  assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
  return extractSubVector(Vec, IdxVal, DAG, dl, 256);
}
 
static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
                               SelectionDAG &DAG, const SDLoc &dl,
                               unsigned vectorWidth) {
  assert((vectorWidth == 128 || vectorWidth == 256) &&
         "Unsupported vector width");
  // Inserting UNDEF is Result
  if (Vec.isUndef())
    return Result;
 
  // Insert the relevant vectorWidth bits.
  EVT VT = Vec.getValueType();
  unsigned ElemsPerChunk = vectorWidth / VT.getScalarSizeInBits();
  assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
 
  // This is the index of the first element of the vectorWidth-bit chunk
  // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
  IdxVal &= ~(ElemsPerChunk - 1);
  return DAG.getInsertSubvector(dl, Result, Vec, IdxVal);
}
 
/// Generate a DAG to put 128-bits into a vector > 128 bits.  This
/// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
/// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
/// simple superregister reference.  Idx is an index in the 128 bits
/// we want.  It need not be aligned to a 128-bit boundary.  That makes
/// lowering INSERT_VECTOR_ELT operations easier.
static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
                                  SelectionDAG &DAG, const SDLoc &dl) {
  assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
  return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
}
 
/// Widen a vector to a larger size with the same scalar type, with the new
/// elements either zero or undef.
static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
                              const X86Subtarget &Subtarget, SelectionDAG &DAG,
                              const SDLoc &dl) {
  EVT VecVT = Vec.getValueType();
  assert(VecVT.getFixedSizeInBits() <= VT.getFixedSizeInBits() &&
         VecVT.getScalarType() == VT.getScalarType() &&
         "Unsupported vector widening type");
  // If the upper 128-bits of a build vector are already undef/zero, then try to
  // widen from the lower 128-bits.
  if (Vec.getOpcode() == ISD::BUILD_VECTOR && VecVT.is256BitVector()) {
    unsigned NumSrcElts = VecVT.getVectorNumElements();
    ArrayRef<SDUse> Hi = Vec->ops().drop_front(NumSrcElts / 2);
    if (all_of(Hi, [&](SDValue V) {
          return V.isUndef() || (ZeroNewElements && X86::isZeroNode(V));
        }))
      Vec = extract128BitVector(Vec, 0, DAG, dl);
  }
  SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
                                : DAG.getUNDEF(VT);
  return DAG.getInsertSubvector(dl, Res, Vec, 0);
}
 
/// Widen a vector to a larger size with the same scalar type, with the new
/// elements either zero or undef.
static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
                              const X86Subtarget &Subtarget, SelectionDAG &DAG,
                              const SDLoc &dl, unsigned WideSizeInBits) {
  assert(Vec.getValueSizeInBits() <= WideSizeInBits &&
         (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 &&
         "Unsupported vector widening type");
  unsigned WideNumElts = WideSizeInBits / Vec.getScalarValueSizeInBits();
  MVT SVT = Vec.getSimpleValueType().getScalarType();
  MVT VT = MVT::getVectorVT(SVT, WideNumElts);
  return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
}
 
/// Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT
/// and bitcast with integer types.
static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget) {
  assert(VT.getVectorElementType() == MVT::i1 && "Expected bool vector");
  unsigned NumElts = VT.getVectorNumElements();
  if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
    return Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
  return VT;
}
 
/// Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and
/// bitcast with integer types.
static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements,
                               const X86Subtarget &Subtarget, SelectionDAG &DAG,
                               const SDLoc &dl) {
  MVT VT = widenMaskVectorType(Vec.getSimpleValueType(), Subtarget);
  return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
}
 
// Helper function to collect subvector ops that are concatenated together,
// either by ISD::CONCAT_VECTORS or a ISD::INSERT_SUBVECTOR series.
// The subvectors in Ops are guaranteed to be the same type.
static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
                             SelectionDAG &DAG) {
  assert(Ops.empty() && "Expected an empty ops vector");
 
  if (N->getOpcode() == ISD::CONCAT_VECTORS) {
    Ops.append(N->op_begin(), N->op_end());
    return true;
  }
 
  if (N->getOpcode() == ISD::INSERT_SUBVECTOR) {
    SDValue Src = N->getOperand(0);
    SDValue Sub = N->getOperand(1);
    const APInt &Idx = N->getConstantOperandAPInt(2);
    EVT VT = Src.getValueType();
    EVT SubVT = Sub.getValueType();
 
    if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) {
      // insert_subvector(undef, x, lo)
      if (Idx == 0 && Src.isUndef()) {
        Ops.push_back(Sub);
        Ops.push_back(DAG.getUNDEF(SubVT));
        return true;
      }
      if (Idx == (VT.getVectorNumElements() / 2)) {
        // insert_subvector(insert_subvector(undef, x, lo), y, hi)
        if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
            Src.getOperand(1).getValueType() == SubVT &&
            isNullConstant(Src.getOperand(2))) {
          // Attempt to recurse into inner (matching) concats.
          SDValue Lo = Src.getOperand(1);
          SDValue Hi = Sub;
          SmallVector<SDValue, 2> LoOps, HiOps;
          if (collectConcatOps(Lo.getNode(), LoOps, DAG) &&
              collectConcatOps(Hi.getNode(), HiOps, DAG) &&
              LoOps.size() == HiOps.size()) {
            Ops.append(LoOps);
            Ops.append(HiOps);
            return true;
          }
          Ops.push_back(Lo);
          Ops.push_back(Hi);
          return true;
        }
        // insert_subvector(x, extract_subvector(x, lo), hi)
        if (Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
            Sub.getOperand(0) == Src && isNullConstant(Sub.getOperand(1))) {
          Ops.append(2, Sub);
          return true;
        }
        // insert_subvector(undef, x, hi)
        if (Src.isUndef()) {
          Ops.push_back(DAG.getUNDEF(SubVT));
          Ops.push_back(Sub);
          return true;
        }
      }
    }
  }
 
  if (N->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    EVT VT = N->getValueType(0);
    SDValue Src = N->getOperand(0);
    uint64_t Idx = N->getConstantOperandVal(1);
 
    // Collect all the subvectors from the source vector and slice off the
    // extraction.
    SmallVector<SDValue, 4> SrcOps;
    if (collectConcatOps(Src.getNode(), SrcOps, DAG) &&
        VT.getSizeInBits() > SrcOps[0].getValueSizeInBits() &&
        (VT.getSizeInBits() % SrcOps[0].getValueSizeInBits()) == 0 &&
        (Idx % SrcOps[0].getValueType().getVectorNumElements()) == 0) {
      unsigned SubIdx = Idx / SrcOps[0].getValueType().getVectorNumElements();
      unsigned NumSubs = VT.getSizeInBits() / SrcOps[0].getValueSizeInBits();
      Ops.append(SrcOps.begin() + SubIdx, SrcOps.begin() + SubIdx + NumSubs);
      return true;
    }
  }
 
  assert(Ops.empty() && "Expected an empty ops vector");
  return false;
}
 
// Helper to check if \p V can be split into subvectors and the upper subvectors
// are all undef. In which case return the lower subvector.
static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL,
                                     SelectionDAG &DAG) {
  SmallVector<SDValue> SubOps;
  if (!collectConcatOps(V.getNode(), SubOps, DAG))
    return SDValue();
 
  unsigned NumSubOps = SubOps.size();
  unsigned HalfNumSubOps = NumSubOps / 2;
  assert((NumSubOps % 2) == 0 && "Unexpected number of subvectors");
 
  ArrayRef<SDValue> UpperOps(SubOps.begin() + HalfNumSubOps, SubOps.end());
  if (any_of(UpperOps, [](SDValue Op) { return !Op.isUndef(); }))
    return SDValue();
 
  EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
  ArrayRef<SDValue> LowerOps(SubOps.begin(), SubOps.begin() + HalfNumSubOps);
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps);
}
 
// Helper to check if we can access all the constituent subvectors without any
// extract ops.
static bool isFreeToSplitVector(SDValue V, SelectionDAG &DAG) {
  SmallVector<SDValue> Ops;
  return collectConcatOps(V.getNode(), Ops, DAG);
}
 
static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
                                               const SDLoc &dl) {
  EVT VT = Op.getValueType();
  unsigned NumElems = VT.getVectorNumElements();
  unsigned SizeInBits = VT.getSizeInBits();
  assert((NumElems % 2) == 0 && (SizeInBits % 2) == 0 &&
         "Can't split odd sized vector");
 
  SmallVector<SDValue, 4> SubOps;
  if (collectConcatOps(Op.getNode(), SubOps, DAG)) {
    assert((SubOps.size() % 2) == 0 && "Can't split odd sized vector concat");
    unsigned HalfOps = SubOps.size() / 2;
    EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
    SmallVector<SDValue, 2> LoOps(SubOps.begin(), SubOps.begin() + HalfOps);
    SmallVector<SDValue, 2> HiOps(SubOps.begin() + HalfOps, SubOps.end());
    SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, LoOps);
    SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, HiOps);
    return std::make_pair(Lo, Hi);
  }
 
  // If this is a splat value (with no-undefs) then use the lower subvector,
  // which should be a free extraction.
  SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
  if (DAG.isSplatValue(Op, /*AllowUndefs*/ false))
    return std::make_pair(Lo, Lo);
 
  SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
  return std::make_pair(Lo, Hi);
}
 
/// Break an operation into 2 half sized ops and then concatenate the results.
static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) {
  unsigned NumOps = Op.getNumOperands();
  EVT VT = Op.getValueType();
 
  // Extract the LHS Lo/Hi vectors
  SmallVector<SDValue> LoOps(NumOps, SDValue());
  SmallVector<SDValue> HiOps(NumOps, SDValue());
  for (unsigned I = 0; I != NumOps; ++I) {
    SDValue SrcOp = Op.getOperand(I);
    if (!SrcOp.getValueType().isVector()) {
      LoOps[I] = HiOps[I] = SrcOp;
      continue;
    }
    std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl);
  }
 
  EVT LoVT, HiVT;
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
                     DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps),
                     DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps));
}
 
/// Break an unary integer operation into 2 half sized ops and then
/// concatenate the result back.
static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG,
                                   const SDLoc &dl) {
  // Make sure we only try to split 256/512-bit types to avoid creating
  // narrow vectors.
  [[maybe_unused]] EVT VT = Op.getValueType();
  assert((Op.getOperand(0).getValueType().is256BitVector() ||
          Op.getOperand(0).getValueType().is512BitVector()) &&
         (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
  assert(Op.getOperand(0).getValueType().getVectorNumElements() ==
             VT.getVectorNumElements() &&
         "Unexpected VTs!");
  return splitVectorOp(Op, DAG, dl);
}
 
/// Break a binary integer operation into 2 half sized ops and then
/// concatenate the result back.
static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG,
                                    const SDLoc &dl) {
  // Assert that all the types match.
  [[maybe_unused]] EVT VT = Op.getValueType();
  assert(Op.getOperand(0).getValueType() == VT &&
         Op.getOperand(1).getValueType() == VT && "Unexpected VTs!");
  assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
  return splitVectorOp(Op, DAG, dl);
}
 
// Helper for splitting operands of an operation to legal target size and
// apply a function on each part.
// Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
// 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
// deciding if/how to split Ops. Ops elements do *not* have to be of type VT.
// The argument Builder is a function that will be applied on each split part:
// SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
template <typename F>
SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
                         const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
                         F Builder, bool CheckBWI = true,
                         bool AllowAVX512 = true) {
  assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2");
  unsigned NumSubs = 1;
  if (AllowAVX512 && ((CheckBWI && Subtarget.useBWIRegs()) ||
                      (!CheckBWI && Subtarget.useAVX512Regs()))) {
    if (VT.getSizeInBits() > 512) {
      NumSubs = VT.getSizeInBits() / 512;
      assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");
    }
  } else if (Subtarget.hasAVX2()) {
    if (VT.getSizeInBits() > 256) {
      NumSubs = VT.getSizeInBits() / 256;
      assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size");
    }
  } else {
    if (VT.getSizeInBits() > 128) {
      NumSubs = VT.getSizeInBits() / 128;
      assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size");
    }
  }
 
  if (NumSubs == 1)
    return Builder(DAG, DL, Ops);
 
  SmallVector<SDValue, 4> Subs;
  for (unsigned i = 0; i != NumSubs; ++i) {
    SmallVector<SDValue, 2> SubOps;
    for (SDValue Op : Ops) {
      EVT OpVT = Op.getValueType();
      unsigned NumSubElts = OpVT.getVectorNumElements() / NumSubs;
      unsigned SizeSub = OpVT.getSizeInBits() / NumSubs;
      SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
    }
    Subs.push_back(Builder(DAG, DL, SubOps));
  }
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
}
 
// Helper function that extends a non-512-bit vector op to 512-bits on non-VLX
// targets.
static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,
                             ArrayRef<SDValue> Ops, SelectionDAG &DAG,
                             const X86Subtarget &Subtarget) {
  assert(Subtarget.hasAVX512() && "AVX512 target expected");
  MVT SVT = VT.getScalarType();
 
  // If we have a 32/64 splatted constant, splat it to DstTy to
  // encourage a foldable broadcast'd operand.
  auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {
    unsigned OpEltSizeInBits = OpVT.getScalarSizeInBits();
    // AVX512 broadcasts 32/64-bit operands.
    // TODO: Support float once getAVX512Node is used by fp-ops.
    if (!OpVT.isInteger() || OpEltSizeInBits < 32 ||
        !DAG.getTargetLoweringInfo().isTypeLegal(SVT))
      return SDValue();
    // If we're not widening, don't bother if we're not bitcasting.
    if (OpVT == DstVT && Op.getOpcode() != ISD::BITCAST)
      return SDValue();
    if (auto *BV = dyn_cast<BuildVectorSDNode>(peekThroughBitcasts(Op))) {
      APInt SplatValue, SplatUndef;
      unsigned SplatBitSize;
      bool HasAnyUndefs;
      if (BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                              HasAnyUndefs, OpEltSizeInBits) &&
          !HasAnyUndefs && SplatValue.getBitWidth() == OpEltSizeInBits)
        return DAG.getConstant(SplatValue, DL, DstVT);
    }
    return SDValue();
  };
 
  bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector());
 
  MVT DstVT = VT;
  if (Widen)
    DstVT = MVT::getVectorVT(SVT, 512 / SVT.getSizeInBits());
 
  // Canonicalize src operands.
  SmallVector<SDValue> SrcOps(Ops);
  for (SDValue &Op : SrcOps) {
    MVT OpVT = Op.getSimpleValueType();
    // Just pass through scalar operands.
    if (!OpVT.isVector())
      continue;
    assert(OpVT == VT && "Vector type mismatch");
 
    if (SDValue BroadcastOp = MakeBroadcastOp(Op, OpVT, DstVT)) {
      Op = BroadcastOp;
      continue;
    }
 
    // Just widen the subvector by inserting into an undef wide vector.
    if (Widen)
      Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512);
  }
 
  SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
 
  // Perform the 512-bit op then extract the bottom subvector.
  if (Widen)
    Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
  return Res;
}
 
/// Insert i1-subvector to i1-vector.
static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
                                const X86Subtarget &Subtarget) {
 
  SDLoc dl(Op);
  SDValue Vec = Op.getOperand(0);
  SDValue SubVec = Op.getOperand(1);
  SDValue Idx = Op.getOperand(2);
  unsigned IdxVal = Op.getConstantOperandVal(2);
 
  // Inserting undef is a nop. We can just return the original vector.
  if (SubVec.isUndef())
    return Vec;
 
  if (IdxVal == 0 && Vec.isUndef()) // the operation is legal
    return Op;
 
  MVT OpVT = Op.getSimpleValueType();
  unsigned NumElems = OpVT.getVectorNumElements();
  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
 
  // Extend to natively supported kshift.
  MVT WideOpVT = widenMaskVectorType(OpVT, Subtarget);
 
  // Inserting into the lsbs of a zero vector is legal. ISel will insert shifts
  // if necessary.
  if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
    // May need to promote to a legal type.
    Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                     DAG.getConstant(0, dl, WideOpVT),
                     SubVec, Idx);
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
  }
 
  MVT SubVecVT = SubVec.getSimpleValueType();
  unsigned SubVecNumElems = SubVecVT.getVectorNumElements();
  assert(IdxVal + SubVecNumElems <= NumElems &&
         IdxVal % SubVecVT.getSizeInBits() == 0 &&
         "Unexpected index value in INSERT_SUBVECTOR");
 
  SDValue Undef = DAG.getUNDEF(WideOpVT);
 
  if (IdxVal == 0) {
    // Zero lower bits of the Vec
    SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
                      ZeroIdx);
    Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
    Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
    // Merge them together, SubVec should be zero extended.
    SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                         DAG.getConstant(0, dl, WideOpVT),
                         SubVec, ZeroIdx);
    Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
  }
 
  SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                       Undef, SubVec, ZeroIdx);
 
  if (Vec.isUndef()) {
    assert(IdxVal != 0 && "Unexpected index");
    SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                         DAG.getTargetConstant(IdxVal, dl, MVT::i8));
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
  }
 
  if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
    assert(IdxVal != 0 && "Unexpected index");
    // If upper elements of Vec are known undef, then just shift into place.
    if (llvm::all_of(Vec->ops().slice(IdxVal + SubVecNumElems),
                     [](SDValue V) { return V.isUndef(); })) {
      SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                           DAG.getTargetConstant(IdxVal, dl, MVT::i8));
    } else {
      NumElems = WideOpVT.getVectorNumElements();
      unsigned ShiftLeft = NumElems - SubVecNumElems;
      unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
      SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                           DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
      if (ShiftRight != 0)
        SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
                             DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
    }
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
  }
 
  // Simple case when we put subvector in the upper part
  if (IdxVal + SubVecNumElems == NumElems) {
    SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                         DAG.getTargetConstant(IdxVal, dl, MVT::i8));
    if (SubVecNumElems * 2 == NumElems) {
      // Special case, use legal zero extending insert_subvector. This allows
      // isel to optimize when bits are known zero.
      Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
      Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                        DAG.getConstant(0, dl, WideOpVT),
                        Vec, ZeroIdx);
    } else {
      // Otherwise use explicit shifts to zero the bits.
      Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                        Undef, Vec, ZeroIdx);
      NumElems = WideOpVT.getVectorNumElements();
      SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
      Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
      Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
    }
    Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
  }
 
  // Inserting into the middle is more complicated.
 
  NumElems = WideOpVT.getVectorNumElements();
 
  // Widen the vector if needed.
  Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
 
  unsigned ShiftLeft = NumElems - SubVecNumElems;
  unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
 
  // Do an optimization for the most frequently used types.
  if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {
    APInt Mask0 = APInt::getBitsSet(NumElems, IdxVal, IdxVal + SubVecNumElems);
    Mask0.flipAllBits();
    SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
    SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
    Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
    SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                         DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
    SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
                         DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
    Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
 
    // Reduce to original width if needed.
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
  }
 
  // Clear the upper bits of the subvector and move it to its insert position.
  SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
                       DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
  SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
                       DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
 
  // Isolate the bits below the insertion point.
  unsigned LowShift = NumElems - IdxVal;
  SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
                            DAG.getTargetConstant(LowShift, dl, MVT::i8));
  Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
                    DAG.getTargetConstant(LowShift, dl, MVT::i8));
 
  // Isolate the bits after the last inserted bit.
  unsigned HighShift = IdxVal + SubVecNumElems;
  SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
                            DAG.getTargetConstant(HighShift, dl, MVT::i8));
  High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
                    DAG.getTargetConstant(HighShift, dl, MVT::i8));
 
  // Now OR all 3 pieces together.
  Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
  SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
 
  // Reduce to original width if needed.
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
}
 
static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
                                const SDLoc &dl) {
  assert(V1.getValueType() == V2.getValueType() && "subvector type mismatch");
  EVT SubVT = V1.getValueType();
  EVT SubSVT = SubVT.getScalarType();
  unsigned SubNumElts = SubVT.getVectorNumElements();
  unsigned SubVectorWidth = SubVT.getSizeInBits();
  EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
  SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
  return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
}
 
/// Returns a vector of specified type with all bits set.
/// Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>.
/// Then bitcast to their original type, ensuring they get CSE'd.
static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
  assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
         "Expected a 128/256/512-bit vector type");
  unsigned NumElts = VT.getSizeInBits() / 32;
  SDValue Vec = DAG.getAllOnesConstant(dl, MVT::getVectorVT(MVT::i32, NumElts));
  return DAG.getBitcast(VT, Vec);
}
 
// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
  switch (Opc) {
  case ISD::SHL:
  case X86ISD::VSHL:
  case X86ISD::VSHLI:
    return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
  case ISD::SRL:
  case X86ISD::VSRL:
  case X86ISD::VSRLI:
    return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
  case ISD::SRA:
  case X86ISD::VSRA:
  case X86ISD::VSRAI:
    return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
  }
  llvm_unreachable("Unknown target vector shift node");
}
 
/// Handle vector element shifts where the shift amount is a constant.
/// Takes immediate version of shift as input.
static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
                                          SDValue SrcOp, uint64_t ShiftAmt,
                                          SelectionDAG &DAG) {
  assert(
      (Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI) &&
      "Unknown target vector shift-by-constant node");
 
  // Bitcast the source vector to the output type, this is mainly necessary for
  // vXi8/vXi64 shifts.
  SrcOp = DAG.getBitcast(VT, SrcOp);
 
  // Fold this packed shift into its first operand if ShiftAmt is 0.
  if (ShiftAmt == 0)
    return SrcOp;
 
  // Check for ShiftAmt >= element width
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  if (ShiftAmt >= EltSizeInBits) {
    if (Opc == X86ISD::VSRAI)
      ShiftAmt = EltSizeInBits - 1;
    else
      return DAG.getConstant(0, dl, VT);
  }
 
  // Fold this packed vector shift into a build vector if SrcOp is a
  // vector of Constants or UNDEFs.
  if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
    unsigned ShiftOpc;
    switch (Opc) {
    default:
      llvm_unreachable("Unknown opcode!");
    case X86ISD::VSHLI:
      ShiftOpc = ISD::SHL;
      break;
    case X86ISD::VSRLI:
      ShiftOpc = ISD::SRL;
      break;
    case X86ISD::VSRAI:
      ShiftOpc = ISD::SRA;
      break;
    }
 
    SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
    if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
      return C;
  }
 
  return DAG.getNode(Opc, dl, VT, SrcOp,
                     DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
}
 
/// Handle vector element shifts by a splat shift amount
static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
                                   SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
                                   const X86Subtarget &Subtarget,
                                   SelectionDAG &DAG) {
  MVT AmtVT = ShAmt.getSimpleValueType();
  assert(AmtVT.isVector() && "Vector shift type mismatch");
  assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
         "Illegal vector splat index");
 
  // Move the splat element to the bottom element.
  if (ShAmtIdx != 0) {
    SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
    Mask[0] = ShAmtIdx;
    ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
  }
 
  // Peek through any zext node if we can get back to a 128-bit source.
  if (AmtVT.getScalarSizeInBits() == 64 &&
      (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
       ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
      ShAmt.getOperand(0).getValueType().isSimple() &&
      ShAmt.getOperand(0).getValueType().is128BitVector()) {
    ShAmt = ShAmt.getOperand(0);
    AmtVT = ShAmt.getSimpleValueType();
  }
 
  // See if we can mask off the upper elements using the existing source node.
  // The shift uses the entire lower 64-bits of the amount vector, so no need to
  // do this for vXi64 types.
  bool IsMasked = false;
  if (AmtVT.getScalarSizeInBits() < 64) {
    if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
        ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
      // If the shift amount has come from a scalar, then zero-extend the scalar
      // before moving to the vector.
      ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
      ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
      ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
      AmtVT = MVT::v4i32;
      IsMasked = true;
    } else if (ShAmt.getOpcode() == ISD::AND) {
      // See if the shift amount is already masked (e.g. for rotation modulo),
      // then we can zero-extend it by setting all the other mask elements to
      // zero.
      SmallVector<SDValue> MaskElts(
          AmtVT.getVectorNumElements(),
          DAG.getConstant(0, dl, AmtVT.getScalarType()));
      MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
      SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
      if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
                                             {ShAmt.getOperand(1), Mask}))) {
        ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
        IsMasked = true;
      }
    }
  }
 
  // Extract if the shift amount vector is larger than 128-bits.
  if (AmtVT.getSizeInBits() > 128) {
    ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
    AmtVT = ShAmt.getSimpleValueType();
  }
 
  // Zero-extend bottom element to v2i64 vector type, either by extension or
  // shuffle masking.
  if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
    if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
                                ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
      ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
    } else if (Subtarget.hasSSE41()) {
      ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
                          MVT::v2i64, ShAmt);
    } else {
      SDValue ByteShift = DAG.getTargetConstant(
          (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
      ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
      ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
                          ByteShift);
      ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
                          ByteShift);
    }
  }
 
  // Change opcode to non-immediate version.
  Opc = getTargetVShiftUniformOpcode(Opc, true);
 
  // The return type has to be a 128-bit type with the same element
  // type as the input type.
  MVT EltVT = VT.getVectorElementType();
  MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
 
  ShAmt = DAG.getBitcast(ShVT, ShAmt);
  return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
}
 
static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
                                      SDValue In, SelectionDAG &DAG) {
  EVT InVT = In.getValueType();
  assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.");
 
  // Canonicalize Opcode to general extension version.
  switch (Opcode) {
  case ISD::ANY_EXTEND:
  case ISD::ANY_EXTEND_VECTOR_INREG:
    Opcode = ISD::ANY_EXTEND;
    break;
  case ISD::SIGN_EXTEND:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
    Opcode = ISD::SIGN_EXTEND;
    break;
  case ISD::ZERO_EXTEND:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    Opcode = ISD::ZERO_EXTEND;
    break;
  default:
    llvm_unreachable("Unknown extension opcode");
  }
 
  // For 256-bit vectors, we only need the lower (128-bit) input half.
  // For 512-bit vectors, we only need the lower input half or quarter.
  if (InVT.getSizeInBits() > 128) {
    assert(VT.getSizeInBits() == InVT.getSizeInBits() &&
           "Expected VTs to be the same size!");
    unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();
    In = extractSubVector(In, 0, DAG, DL,
                          std::max(128U, (unsigned)VT.getSizeInBits() / Scale));
    InVT = In.getValueType();
  }
 
  if (VT.getVectorNumElements() != InVT.getVectorNumElements())
    Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode);
 
  return DAG.getNode(Opcode, DL, VT, In);
}
 
// Create OR(AND(LHS,MASK),AND(RHS,~MASK)) bit select pattern
static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
                            SDValue Mask, SelectionDAG &DAG) {
  LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask);
  RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS);
  return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
}
 
void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
                                   bool Lo, bool Unary) {
  assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
         "Illegal vector type to unpack");
  assert(Mask.empty() && "Expected an empty shuffle mask vector");
  int NumElts = VT.getVectorNumElements();
  int NumEltsInLane = 128 / VT.getScalarSizeInBits();
  for (int i = 0; i < NumElts; ++i) {
    unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
    int Pos = (i % NumEltsInLane) / 2 + LaneStart;
    Pos += (Unary ? 0 : NumElts * (i % 2));
    Pos += (Lo ? 0 : NumEltsInLane / 2);
    Mask.push_back(Pos);
  }
}
 
/// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
/// imposed by AVX and specific to the unary pattern. Example:
/// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
/// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
                                   bool Lo) {
  assert(Mask.empty() && "Expected an empty shuffle mask vector");
  int NumElts = VT.getVectorNumElements();
  for (int i = 0; i < NumElts; ++i) {
    int Pos = i / 2;
    Pos += (Lo ? 0 : NumElts / 2);
    Mask.push_back(Pos);
  }
}
 
// Attempt to constant fold, else just create a VECTOR_SHUFFLE.
static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
                                SDValue V1, SDValue V2, ArrayRef<int> Mask) {
  if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) &&
      (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) {
    SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
    for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) {
      int M = Mask[I];
      if (M < 0)
        continue;
      SDValue V = (M < NumElts) ? V1 : V2;
      if (V.isUndef())
        continue;
      Ops[I] = V.getOperand(M % NumElts);
    }
    return DAG.getBuildVector(VT, dl, Ops);
  }
 
  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
}
 
/// Returns a vector_shuffle node for an unpackl operation.
static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                          SDValue V1, SDValue V2) {
  SmallVector<int, 8> Mask;
  createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
  return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
}
 
/// Returns a vector_shuffle node for an unpackh operation.
static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                          SDValue V1, SDValue V2) {
  SmallVector<int, 8> Mask;
  createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
  return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
}
 
/// Returns a node that packs the LHS + RHS nodes together at half width.
/// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half.
/// TODO: Add subvector splitting if/when we have a need for it.
static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
                       const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
                       bool PackHiHalf = false) {
  MVT OpVT = LHS.getSimpleValueType();
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  bool UsePackUS = Subtarget.hasSSE41() || EltSizeInBits == 8;
  assert(OpVT == RHS.getSimpleValueType() &&
         VT.getSizeInBits() == OpVT.getSizeInBits() &&
         (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() &&
         "Unexpected PACK operand types");
  assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) &&
         "Unexpected PACK result type");
 
  // Rely on vector shuffles for vXi64 -> vXi32 packing.
  if (EltSizeInBits == 32) {
    SmallVector<int> PackMask;
    int Offset = PackHiHalf ? 1 : 0;
    int NumElts = VT.getVectorNumElements();
    for (int I = 0; I != NumElts; I += 4) {
      PackMask.push_back(I + Offset);
      PackMask.push_back(I + Offset + 2);
      PackMask.push_back(I + Offset + NumElts);
      PackMask.push_back(I + Offset + NumElts + 2);
    }
    return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
                                DAG.getBitcast(VT, RHS), PackMask);
  }
 
  // See if we already have sufficient leading bits for PACKSS/PACKUS.
  if (!PackHiHalf) {
    if (UsePackUS &&
        DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits &&
        DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits)
      return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
 
    if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits &&
        DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits)
      return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
  }
 
  // Fallback to sign/zero extending the requested half and pack.
  SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
  if (UsePackUS) {
    if (PackHiHalf) {
      LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt);
      RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt);
    } else {
      SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
      LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask);
      RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask);
    };
    return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
  };
 
  if (!PackHiHalf) {
    LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt);
    RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt);
  }
  LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt);
  RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt);
  return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
}
 
/// Return a vector_shuffle of the specified vector of zero or undef vector.
/// This produces a shuffle where the low element of V2 is swizzled into the
/// zero/undef vector, landing at element Idx.
/// This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
                                           bool IsZero,
                                           const X86Subtarget &Subtarget,
                                           SelectionDAG &DAG) {
  MVT VT = V2.getSimpleValueType();
  SDValue V1 = IsZero
    ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
  int NumElems = VT.getVectorNumElements();
  SmallVector<int, 16> MaskVec(NumElems);
  for (int i = 0; i != NumElems; ++i)
    // If this is the insertion idx, put the low elt of V2 here.
    MaskVec[i] = (i == Idx) ? NumElems : i;
  return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
}
 
static ConstantPoolSDNode *getTargetConstantPoolFromBasePtr(SDValue Ptr) {
  if (Ptr.getOpcode() == X86ISD::Wrapper ||
      Ptr.getOpcode() == X86ISD::WrapperRIP)
    Ptr = Ptr.getOperand(0);
  return dyn_cast<ConstantPoolSDNode>(Ptr);
}
 
// TODO: Add support for non-zero offsets.
static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
  ConstantPoolSDNode *CNode = getTargetConstantPoolFromBasePtr(Ptr);
  if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0)
    return nullptr;
  return CNode->getConstVal();
}
 
static const Constant *getTargetConstantFromNode(LoadSDNode *Load) {
  if (!Load || !ISD::isNormalLoad(Load))
    return nullptr;
  return getTargetConstantFromBasePtr(Load->getBasePtr());
}
 
static const Constant *getTargetConstantFromNode(SDValue Op) {
  Op = peekThroughBitcasts(Op);
  return getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op));
}
 
const Constant *
X86TargetLowering::getTargetConstantFromLoad(LoadSDNode *LD) const {
  assert(LD && "Unexpected null LoadSDNode");
  return getTargetConstantFromNode(LD);
}
 
bool X86TargetLowering::isTargetCanonicalSelect(SDNode *N) const {
  // Do not fold (vselect not(C), X, 0s) to (vselect C, Os, X)
  SDValue Cond = N->getOperand(0);
  SDValue RHS = N->getOperand(2);
  EVT CondVT = Cond.getValueType();
  return N->getOpcode() == ISD::VSELECT && Subtarget.hasAVX512() &&
         CondVT.getVectorElementType() == MVT::i1 &&
         ISD::isBuildVectorAllZeros(RHS.getNode());
}
 
// Extract raw constant bits from constant pools.
static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
                                          APInt &UndefElts,
                                          SmallVectorImpl<APInt> &EltBits,
                                          bool AllowWholeUndefs = true,
                                          bool AllowPartialUndefs = false) {
  assert(EltBits.empty() && "Expected an empty EltBits vector");
 
  Op = peekThroughBitcasts(Op);
 
  EVT VT = Op.getValueType();
  unsigned SizeInBits = VT.getSizeInBits();
  unsigned NumElts = SizeInBits / EltSizeInBits;
 
  // Can't split constant.
  if ((SizeInBits % EltSizeInBits) != 0)
    return false;
 
  // Bitcast a source array of element bits to the target size.
  auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
    unsigned NumSrcElts = UndefSrcElts.getBitWidth();
    unsigned SrcEltSizeInBits = SrcEltBits[0].getBitWidth();
    assert((NumSrcElts * SrcEltSizeInBits) == SizeInBits &&
           "Constant bit sizes don't match");
 
    // Don't split if we don't allow undef bits.
    bool AllowUndefs = AllowWholeUndefs || AllowPartialUndefs;
    if (UndefSrcElts.getBoolValue() && !AllowUndefs)
      return false;
 
    // If we're already the right size, don't bother bitcasting.
    if (NumSrcElts == NumElts) {
      UndefElts = UndefSrcElts;
      EltBits.assign(SrcEltBits.begin(), SrcEltBits.end());
      return true;
    }
 
    // Extract all the undef/constant element data and pack into single bitsets.
    APInt UndefBits(SizeInBits, 0);
    APInt MaskBits(SizeInBits, 0);
 
    for (unsigned i = 0; i != NumSrcElts; ++i) {
      unsigned BitOffset = i * SrcEltSizeInBits;
      if (UndefSrcElts[i])
        UndefBits.setBits(BitOffset, BitOffset + SrcEltSizeInBits);
      MaskBits.insertBits(SrcEltBits[i], BitOffset);
    }
 
    // Split the undef/constant single bitset data into the target elements.
    UndefElts = APInt(NumElts, 0);
    EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
 
    for (unsigned i = 0; i != NumElts; ++i) {
      unsigned BitOffset = i * EltSizeInBits;
      APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
 
      // Only treat an element as UNDEF if all bits are UNDEF.
      if (UndefEltBits.isAllOnes()) {
        if (!AllowWholeUndefs)
          return false;
        UndefElts.setBit(i);
        continue;
      }
 
      // If only some bits are UNDEF then treat them as zero (or bail if not
      // supported).
      if (UndefEltBits.getBoolValue() && !AllowPartialUndefs)
        return false;
 
      EltBits[i] = MaskBits.extractBits(EltSizeInBits, BitOffset);
    }
    return true;
  };
 
  // Collect constant bits and insert into mask/undef bit masks.
  auto CollectConstantBits = [](const Constant *Cst, APInt &Mask, APInt &Undefs,
                                unsigned UndefBitIndex) {
    if (!Cst)
      return false;
    if (isa<UndefValue>(Cst)) {
      Undefs.setBit(UndefBitIndex);
      return true;
    }
    if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
      Mask = APInt::getSplat(CInt->getType()->getPrimitiveSizeInBits(),
                             CInt->getValue());
      return true;
    }
    if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
      Mask = APInt::getSplat(CFP->getType()->getPrimitiveSizeInBits(),
                             CFP->getValueAPF().bitcastToAPInt());
      return true;
    }
    if (auto *CDS = dyn_cast<ConstantDataSequential>(Cst)) {
      Type *Ty = CDS->getType();
      if (Ty->isVectorTy()) {
        Mask = APInt::getZero(Ty->getPrimitiveSizeInBits());
        Type *EltTy = CDS->getElementType();
        bool IsInteger = EltTy->isIntegerTy();
        bool IsFP =
            EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
        if (!IsInteger && !IsFP)
          return false;
        unsigned EltBits = EltTy->getPrimitiveSizeInBits();
        for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I)
          if (IsInteger)
            Mask.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
          else
            Mask.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
                            I * EltBits);
        return true;
      }
    }
    return false;
  };
 
  // Handle UNDEFs.
  if (Op.isUndef()) {
    APInt UndefSrcElts = APInt::getAllOnes(NumElts);
    SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0));
    return CastBitData(UndefSrcElts, SrcEltBits);
  }
 
  // Extract scalar constant bits.
  if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) {
    APInt UndefSrcElts = APInt::getZero(1);
    SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue());
    return CastBitData(UndefSrcElts, SrcEltBits);
  }
  if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
    APInt UndefSrcElts = APInt::getZero(1);
    APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
    SmallVector<APInt, 64> SrcEltBits(1, RawBits);
    return CastBitData(UndefSrcElts, SrcEltBits);
  }
 
  // Extract constant bits from build vector.
  if (auto *BV = dyn_cast<BuildVectorSDNode>(Op)) {
    BitVector Undefs;
    SmallVector<APInt> SrcEltBits;
    unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
    if (BV->getConstantRawBits(true, SrcEltSizeInBits, SrcEltBits, Undefs)) {
      APInt UndefSrcElts = APInt::getZero(SrcEltBits.size());
      for (unsigned I = 0, E = SrcEltBits.size(); I != E; ++I)
        if (Undefs[I])
          UndefSrcElts.setBit(I);
      return CastBitData(UndefSrcElts, SrcEltBits);
    }
  }
 
  // Extract constant bits from constant pool vector.
  if (auto *Cst = getTargetConstantFromNode(Op)) {
    Type *CstTy = Cst->getType();
    unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
    if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0)
      return false;
 
    unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits();
    unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
    if ((SizeInBits % SrcEltSizeInBits) != 0)
      return false;
 
    APInt UndefSrcElts(NumSrcElts, 0);
    SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
    for (unsigned i = 0; i != NumSrcElts; ++i)
      if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i],
                               UndefSrcElts, i))
        return false;
 
    return CastBitData(UndefSrcElts, SrcEltBits);
  }
 
  // Extract constant bits from a broadcasted constant pool scalar.
  if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
      EltSizeInBits <= VT.getScalarSizeInBits()) {
    auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
    if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits())
      return false;
 
    SDValue Ptr = MemIntr->getBasePtr();
    if (const Constant *C = getTargetConstantFromBasePtr(Ptr)) {
      unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
      unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
 
      APInt UndefSrcElts(NumSrcElts, 0);
      SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
      if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
        if (UndefSrcElts[0])
          UndefSrcElts.setBits(0, NumSrcElts);
        if (SrcEltBits[0].getBitWidth() != SrcEltSizeInBits)
          SrcEltBits[0] = SrcEltBits[0].trunc(SrcEltSizeInBits);
        SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
        return CastBitData(UndefSrcElts, SrcEltBits);
      }
    }
  }
 
  // Extract constant bits from a subvector broadcast.
  if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
    auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
    SDValue Ptr = MemIntr->getBasePtr();
    // The source constant may be larger than the subvector broadcast,
    // ensure we extract the correct subvector constants.
    if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) {
      Type *CstTy = Cst->getType();
      unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
      unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits();
      if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 ||
          (SizeInBits % SubVecSizeInBits) != 0)
        return false;
      unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
      unsigned NumSubElts = SubVecSizeInBits / CstEltSizeInBits;
      unsigned NumSubVecs = SizeInBits / SubVecSizeInBits;
      APInt UndefSubElts(NumSubElts, 0);
      SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs,
                                        APInt(CstEltSizeInBits, 0));
      for (unsigned i = 0; i != NumSubElts; ++i) {
        if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i],
                                 UndefSubElts, i))
          return false;
        for (unsigned j = 1; j != NumSubVecs; ++j)
          SubEltBits[i + (j * NumSubElts)] = SubEltBits[i];
      }
      UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(),
                                     UndefSubElts);
      return CastBitData(UndefSubElts, SubEltBits);
    }
  }
 
  // Extract a rematerialized scalar constant insertion.
  if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
      Op.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
      isa<ConstantSDNode>(Op.getOperand(0).getOperand(0))) {
    unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
    unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
 
    APInt UndefSrcElts(NumSrcElts, 0);
    SmallVector<APInt, 64> SrcEltBits;
    const APInt &C = Op.getOperand(0).getConstantOperandAPInt(0);
    SrcEltBits.push_back(C.zextOrTrunc(SrcEltSizeInBits));
    SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0));
    return CastBitData(UndefSrcElts, SrcEltBits);
  }
 
  // Insert constant bits from a base and sub vector sources.
  if (Op.getOpcode() == ISD::INSERT_SUBVECTOR) {
    // If bitcasts to larger elements we might lose track of undefs - don't
    // allow any to be safe.
    unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
    bool AllowUndefs = EltSizeInBits >= SrcEltSizeInBits;
 
    APInt UndefSrcElts, UndefSubElts;
    SmallVector<APInt, 32> EltSrcBits, EltSubBits;
    if (getTargetConstantBitsFromNode(Op.getOperand(1), SrcEltSizeInBits,
                                      UndefSubElts, EltSubBits,
                                      AllowWholeUndefs && AllowUndefs,
                                      AllowPartialUndefs && AllowUndefs) &&
        getTargetConstantBitsFromNode(Op.getOperand(0), SrcEltSizeInBits,
                                      UndefSrcElts, EltSrcBits,
                                      AllowWholeUndefs && AllowUndefs,
                                      AllowPartialUndefs && AllowUndefs)) {
      unsigned BaseIdx = Op.getConstantOperandVal(2);
      UndefSrcElts.insertBits(UndefSubElts, BaseIdx);
      for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i)
        EltSrcBits[BaseIdx + i] = EltSubBits[i];
      return CastBitData(UndefSrcElts, EltSrcBits);
    }
  }
 
  // Extract constant bits from a subvector's source.
  if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
      getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits, UndefElts,
                                    EltBits, AllowWholeUndefs,
                                    AllowPartialUndefs)) {
    EVT SrcVT = Op.getOperand(0).getValueType();
    unsigned NumSrcElts = SrcVT.getSizeInBits() / EltSizeInBits;
    unsigned NumSubElts = VT.getSizeInBits() / EltSizeInBits;
    unsigned BaseOfs = Op.getConstantOperandVal(1) * VT.getScalarSizeInBits();
    unsigned BaseIdx = BaseOfs / EltSizeInBits;
    assert((SrcVT.getSizeInBits() % EltSizeInBits) == 0 &&
           (VT.getSizeInBits() % EltSizeInBits) == 0 &&
           (BaseOfs % EltSizeInBits) == 0 && "Bad subvector index");
 
    UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
    if ((BaseIdx + NumSubElts) != NumSrcElts)
      EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
    if (BaseIdx != 0)
      EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
    return true;
  }
 
  // Extract constant bits from shuffle node sources.
  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Op)) {
    // TODO - support shuffle through bitcasts.
    if (EltSizeInBits != VT.getScalarSizeInBits())
      return false;
 
    ArrayRef<int> Mask = SVN->getMask();
    if ((!AllowWholeUndefs || !AllowPartialUndefs) &&
        llvm::any_of(Mask, [](int M) { return M < 0; }))
      return false;
 
    APInt UndefElts0, UndefElts1;
    SmallVector<APInt, 32> EltBits0, EltBits1;
    if (isAnyInRange(Mask, 0, NumElts) &&
        !getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
                                       UndefElts0, EltBits0, AllowWholeUndefs,
                                       AllowPartialUndefs))
      return false;
    if (isAnyInRange(Mask, NumElts, 2 * NumElts) &&
        !getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
                                       UndefElts1, EltBits1, AllowWholeUndefs,
                                       AllowPartialUndefs))
      return false;
 
    UndefElts = APInt::getZero(NumElts);
    for (int i = 0; i != (int)NumElts; ++i) {
      int M = Mask[i];
      if (M < 0) {
        UndefElts.setBit(i);
        EltBits.push_back(APInt::getZero(EltSizeInBits));
      } else if (M < (int)NumElts) {
        if (UndefElts0[M])
          UndefElts.setBit(i);
        EltBits.push_back(EltBits0[M]);
      } else {
        if (UndefElts1[M - NumElts])
          UndefElts.setBit(i);
        EltBits.push_back(EltBits1[M - NumElts]);
      }
    }
    return true;
  }
 
  return false;
}
 
namespace llvm {
namespace X86 {
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
  APInt UndefElts;
  SmallVector<APInt, 16> EltBits;
  if (getTargetConstantBitsFromNode(
          Op, Op.getScalarValueSizeInBits(), UndefElts, EltBits,
          /*AllowWholeUndefs*/ true, AllowPartialUndefs)) {
    int SplatIndex = -1;
    for (int i = 0, e = EltBits.size(); i != e; ++i) {
      if (UndefElts[i])
        continue;
      if (0 <= SplatIndex && EltBits[i] != EltBits[SplatIndex]) {
        SplatIndex = -1;
        break;
      }
      SplatIndex = i;
    }
    if (0 <= SplatIndex) {
      SplatVal = EltBits[SplatIndex];
      return true;
    }
  }
 
  return false;
}
 
int getRoundingModeX86(unsigned RM) {
  switch (static_cast<::llvm::RoundingMode>(RM)) {
    // clang-format off
  case ::llvm::RoundingMode::NearestTiesToEven: return X86::rmToNearest;
  case ::llvm::RoundingMode::TowardNegative:    return X86::rmDownward;
  case ::llvm::RoundingMode::TowardPositive:    return X86::rmUpward;
  case ::llvm::RoundingMode::TowardZero:        return X86::rmTowardZero;
  default:                                      return X86::rmInvalid;
    // clang-format on
  }
}
 
} // namespace X86
} // namespace llvm
 
static bool getTargetShuffleMaskIndices(SDValue MaskNode,
                                        unsigned MaskEltSizeInBits,
                                        SmallVectorImpl<uint64_t> &RawMask,
                                        APInt &UndefElts) {
  // Extract the raw target constant bits.
  SmallVector<APInt, 64> EltBits;
  if (!getTargetConstantBitsFromNode(MaskNode, MaskEltSizeInBits, UndefElts,
                                     EltBits, /* AllowWholeUndefs */ true,
                                     /* AllowPartialUndefs */ false))
    return false;
 
  // Insert the extracted elements into the mask.
  for (const APInt &Elt : EltBits)
    RawMask.push_back(Elt.getZExtValue());
 
  return true;
}
 
static bool isConstantPowerOf2(SDValue V, unsigned EltSizeInBIts,
                               bool AllowUndefs) {
  APInt UndefElts;
  SmallVector<APInt, 64> EltBits;
  if (!getTargetConstantBitsFromNode(V, EltSizeInBIts, UndefElts, EltBits,
                                     /*AllowWholeUndefs*/ AllowUndefs,
                                     /*AllowPartialUndefs*/ false))
    return false;
 
  bool IsPow2OrUndef = true;
  for (unsigned I = 0, E = EltBits.size(); I != E; ++I)
    IsPow2OrUndef &= UndefElts[I] || EltBits[I].isPowerOf2();
  return IsPow2OrUndef;
}
 
// Helper to attempt to return a cheaper, bit-inverted version of \p V.
static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
  // TODO: don't always ignore oneuse constraints.
  V = peekThroughBitcasts(V);
  EVT VT = V.getValueType();
 
  // Match not(xor X, -1) -> X.
  if (V.getOpcode() == ISD::XOR &&
      (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) ||
       isAllOnesConstant(V.getOperand(1))))
    return V.getOperand(0);
 
  // Match not(extract_subvector(not(X)) -> extract_subvector(X).
  if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
      (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
    if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
      Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), VT, Not,
                         V.getOperand(1));
    }
  }
 
  // Match not(pcmpgt(C, X)) -> pcmpgt(X, C - 1).
  if (V.getOpcode() == X86ISD::PCMPGT &&
      !ISD::isBuildVectorAllZeros(V.getOperand(0).getNode()) &&
      !ISD::isBuildVectorAllOnes(V.getOperand(0).getNode()) &&
      V.getOperand(0).hasOneUse()) {
    APInt UndefElts;
    SmallVector<APInt> EltBits;
    if (getTargetConstantBitsFromNode(V.getOperand(0),
                                      V.getScalarValueSizeInBits(), UndefElts,
                                      EltBits) &&
        !ISD::isBuildVectorOfConstantSDNodes(V.getOperand(1).getNode())) {
      // Don't fold min_signed_value -> (min_signed_value - 1)
      bool MinSigned = false;
      for (APInt &Elt : EltBits) {
        MinSigned |= Elt.isMinSignedValue();
        Elt -= 1;
      }
      if (!MinSigned) {
        SDLoc DL(V);
        MVT VT = V.getSimpleValueType();
        return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1),
                           getConstVector(EltBits, UndefElts, VT, DAG, DL));
      }
    }
  }
 
  // Match not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
  SmallVector<SDValue, 2> CatOps;
  if (collectConcatOps(V.getNode(), CatOps, DAG)) {
    for (SDValue &CatOp : CatOps) {
      SDValue NotCat = IsNOT(CatOp, DAG);
      if (!NotCat)
        return SDValue();
      CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
    }
    return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), VT, CatOps);
  }
 
  // Match not(or(not(X),not(Y))) -> and(X, Y).
  if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
      V.getOperand(0).hasOneUse() && V.getOperand(1).hasOneUse()) {
    // TODO: Handle cases with single NOT operand -> ANDNP
    if (SDValue Op1 = IsNOT(V.getOperand(1), DAG))
      if (SDValue Op0 = IsNOT(V.getOperand(0), DAG))
        return DAG.getNode(ISD::AND, SDLoc(V), VT, DAG.getBitcast(VT, Op0),
                           DAG.getBitcast(VT, Op1));
  }
 
  return SDValue();
}
 
/// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
/// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
/// Note: This ignores saturation, so inputs must be checked first.
static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
                                  bool Unary, unsigned NumStages = 1) {
  assert(Mask.empty() && "Expected an empty shuffle mask vector");
  unsigned NumElts = VT.getVectorNumElements();
  unsigned NumLanes = VT.getSizeInBits() / 128;
  unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
  unsigned Offset = Unary ? 0 : NumElts;
  unsigned Repetitions = 1u << (NumStages - 1);
  unsigned Increment = 1u << NumStages;
  assert((NumEltsPerLane >> NumStages) > 0 && "Illegal packing compaction");
 
  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
    for (unsigned Stage = 0; Stage != Repetitions; ++Stage) {
      for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
        Mask.push_back(Elt + (Lane * NumEltsPerLane));
      for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
        Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
    }
  }
}
 
// Split the demanded elts of a PACKSS/PACKUS node between its operands.
static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
                                APInt &DemandedLHS, APInt &DemandedRHS) {
  int NumLanes = VT.getSizeInBits() / 128;
  int NumElts = DemandedElts.getBitWidth();
  int NumInnerElts = NumElts / 2;
  int NumEltsPerLane = NumElts / NumLanes;
  int NumInnerEltsPerLane = NumInnerElts / NumLanes;
 
  DemandedLHS = APInt::getZero(NumInnerElts);
  DemandedRHS = APInt::getZero(NumInnerElts);
 
  // Map DemandedElts to the packed operands.
  for (int Lane = 0; Lane != NumLanes; ++Lane) {
    for (int Elt = 0; Elt != NumInnerEltsPerLane; ++Elt) {
      int OuterIdx = (Lane * NumEltsPerLane) + Elt;
      int InnerIdx = (Lane * NumInnerEltsPerLane) + Elt;
      if (DemandedElts[OuterIdx])
        DemandedLHS.setBit(InnerIdx);
      if (DemandedElts[OuterIdx + NumInnerEltsPerLane])
        DemandedRHS.setBit(InnerIdx);
    }
  }
}
 
// Split the demanded elts of a HADD/HSUB node between its operands.
static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
                                 APInt &DemandedLHS, APInt &DemandedRHS) {
  getHorizDemandedEltsForFirstOperand(VT.getSizeInBits(), DemandedElts,
                                      DemandedLHS, DemandedRHS);
  DemandedLHS |= DemandedLHS << 1;
  DemandedRHS |= DemandedRHS << 1;
}
 
/// Calculates the shuffle mask corresponding to the target-specific opcode.
/// If the mask could be calculated, returns it in \p Mask, returns the shuffle
/// operands in \p Ops, and returns true.
/// Sets \p IsUnary to true if only one source is used. Note that this will set
/// IsUnary for shuffles which use a single input multiple times, and in those
/// cases it will adjust the mask to only have indices within that single input.
/// It is an error to call this with non-empty Mask/Ops vectors.
static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
                                 SmallVectorImpl<SDValue> &Ops,
                                 SmallVectorImpl<int> &Mask, bool &IsUnary) {
  if (!isTargetShuffle(N.getOpcode()))
    return false;
 
  MVT VT = N.getSimpleValueType();
  unsigned NumElems = VT.getVectorNumElements();
  unsigned MaskEltSize = VT.getScalarSizeInBits();
  SmallVector<uint64_t, 32> RawMask;
  APInt RawUndefs;
  uint64_t ImmN;
 
  assert(Mask.empty() && "getTargetShuffleMask expects an empty Mask vector");
  assert(Ops.empty() && "getTargetShuffleMask expects an empty Ops vector");
 
  IsUnary = false;
  bool IsFakeUnary = false;
  switch (N.getOpcode()) {
  case X86ISD::BLENDI:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeBLENDMask(NumElems, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::SHUFP:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeSHUFPMask(NumElems, MaskEltSize, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::INSERTPS:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeINSERTPSMask(ImmN, Mask, /*SrcIsMem=*/false);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::EXTRQI:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    if (isa<ConstantSDNode>(N.getOperand(1)) &&
        isa<ConstantSDNode>(N.getOperand(2))) {
      int BitLen = N.getConstantOperandVal(1);
      int BitIdx = N.getConstantOperandVal(2);
      DecodeEXTRQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
      IsUnary = true;
    }
    break;
  case X86ISD::INSERTQI:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    if (isa<ConstantSDNode>(N.getOperand(2)) &&
        isa<ConstantSDNode>(N.getOperand(3))) {
      int BitLen = N.getConstantOperandVal(2);
      int BitIdx = N.getConstantOperandVal(3);
      DecodeINSERTQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
      IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    }
    break;
  case X86ISD::UNPCKH:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    DecodeUNPCKHMask(NumElems, MaskEltSize, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::UNPCKL:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    DecodeUNPCKLMask(NumElems, MaskEltSize, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::MOVHLPS:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    DecodeMOVHLPSMask(NumElems, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::MOVLHPS:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    DecodeMOVLHPSMask(NumElems, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::VALIGN:
    assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
           "Only 32-bit and 64-bit elements are supported!");
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeVALIGNMask(NumElems, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    Ops.push_back(N.getOperand(1));
    Ops.push_back(N.getOperand(0));
    break;
  case X86ISD::PALIGNR:
    assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePALIGNRMask(NumElems, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    Ops.push_back(N.getOperand(1));
    Ops.push_back(N.getOperand(0));
    break;
  case X86ISD::VSHLDQ:
    assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePSLLDQMask(NumElems, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::VSRLDQ:
    assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePSRLDQMask(NumElems, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::PSHUFD:
  case X86ISD::VPERMILPI:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePSHUFMask(NumElems, MaskEltSize, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::PSHUFHW:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePSHUFHWMask(NumElems, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::PSHUFLW:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodePSHUFLWMask(NumElems, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::VZEXT_MOVL:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    DecodeZeroMoveLowMask(NumElems, Mask);
    IsUnary = true;
    break;
  case X86ISD::VBROADCAST:
    // We only decode broadcasts of same-sized vectors, peeking through to
    // extracted subvectors is likely to cause hasOneUse issues with
    // SimplifyDemandedBits etc.
    if (N.getOperand(0).getValueType() == VT) {
      DecodeVectorBroadcast(NumElems, Mask);
      IsUnary = true;
      break;
    }
    return false;
  case X86ISD::VPERMILPV: {
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    IsUnary = true;
    SDValue MaskNode = N.getOperand(1);
    if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
                                    RawUndefs)) {
      DecodeVPERMILPMask(NumElems, MaskEltSize, RawMask, RawUndefs, Mask);
      break;
    }
    return false;
  }
  case X86ISD::PSHUFB: {
    assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    IsUnary = true;
    SDValue MaskNode = N.getOperand(1);
    if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
      DecodePSHUFBMask(RawMask, RawUndefs, Mask);
      break;
    }
    return false;
  }
  case X86ISD::VPERMI:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeVPERMMask(NumElems, ImmN, Mask);
    IsUnary = true;
    break;
  case X86ISD::MOVSS:
  case X86ISD::MOVSD:
  case X86ISD::MOVSH:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    DecodeScalarMoveMask(NumElems, /* IsLoad */ false, Mask);
    break;
  case X86ISD::VPERM2X128:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    DecodeVPERM2X128Mask(NumElems, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::SHUF128:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
    decodeVSHUF64x2FamilyMask(NumElems, MaskEltSize, ImmN, Mask);
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    break;
  case X86ISD::MOVSLDUP:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    DecodeMOVSLDUPMask(NumElems, Mask);
    IsUnary = true;
    break;
  case X86ISD::MOVSHDUP:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    DecodeMOVSHDUPMask(NumElems, Mask);
    IsUnary = true;
    break;
  case X86ISD::MOVDDUP:
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    DecodeMOVDDUPMask(NumElems, Mask);
    IsUnary = true;
    break;
  case X86ISD::VPERMIL2: {
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    SDValue MaskNode = N.getOperand(2);
    SDValue CtrlNode = N.getOperand(3);
    if (ConstantSDNode *CtrlOp = dyn_cast<ConstantSDNode>(CtrlNode)) {
      unsigned CtrlImm = CtrlOp->getZExtValue();
      if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
                                      RawUndefs)) {
        DecodeVPERMIL2PMask(NumElems, MaskEltSize, CtrlImm, RawMask, RawUndefs,
                            Mask);
        break;
      }
    }
    return false;
  }
  case X86ISD::VPPERM: {
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
    SDValue MaskNode = N.getOperand(2);
    if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
      DecodeVPPERMMask(RawMask, RawUndefs, Mask);
      break;
    }
    return false;
  }
  case X86ISD::VPERMV: {
    assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
    IsUnary = true;
    // Unlike most shuffle nodes, VPERMV's mask operand is operand 0.
    Ops.push_back(N.getOperand(1));
    SDValue MaskNode = N.getOperand(0);
    if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
                                    RawUndefs)) {
      DecodeVPERMVMask(RawMask, RawUndefs, Mask);
      break;
    }
    return false;
  }
  case X86ISD::VPERMV3: {
    assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
    assert(N.getOperand(2).getValueType() == VT && "Unexpected value type");
    IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(2);
    // Unlike most shuffle nodes, VPERMV3's mask operand is the middle one.
    Ops.push_back(N.getOperand(0));
    Ops.push_back(N.getOperand(2));
    SDValue MaskNode = N.getOperand(1);
    if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
                                    RawUndefs)) {
      DecodeVPERMV3Mask(RawMask, RawUndefs, Mask);
      break;
    }
    return false;
  }
  case X86ISD::COMPRESS: {
    SDValue CmpVec = N.getOperand(0);
    SDValue PassThru = N.getOperand(1);
    SDValue CmpMask = N.getOperand(2);
    APInt UndefElts;
    SmallVector<APInt> EltBits;
    if (!getTargetConstantBitsFromNode(CmpMask, 1, UndefElts, EltBits))
      return false;
    assert(UndefElts.getBitWidth() == NumElems && EltBits.size() == NumElems &&
           "Illegal compression mask");
    for (unsigned I = 0; I != NumElems; ++I) {
      if (!EltBits[I].isZero())
        Mask.push_back(I);
    }
    while (Mask.size() != NumElems) {
      Mask.push_back(NumElems + Mask.size());
    }
    Ops.push_back(CmpVec);
    Ops.push_back(PassThru);
    return true;
  }
  case X86ISD::EXPAND: {
    SDValue ExpVec = N.getOperand(0);
    SDValue PassThru = N.getOperand(1);
    SDValue ExpMask = N.getOperand(2);
    APInt UndefElts;
    SmallVector<APInt> EltBits;
    if (!getTargetConstantBitsFromNode(ExpMask, 1, UndefElts, EltBits))
      return false;
    assert(UndefElts.getBitWidth() == NumElems && EltBits.size() == NumElems &&
           "Illegal expansion mask");
    unsigned ExpIndex = 0;
    for (unsigned I = 0; I != NumElems; ++I) {
      if (EltBits[I].isZero())
        Mask.push_back(I + NumElems);
      else
        Mask.push_back(ExpIndex++);
    }
    Ops.push_back(ExpVec);
    Ops.push_back(PassThru);
    return true;
  }
  default:
    llvm_unreachable("unknown target shuffle node");
  }
 
  // Empty mask indicates the decode failed.
  if (Mask.empty())
    return false;
 
  // Check if we're getting a shuffle mask with zero'd elements.
  if (!AllowSentinelZero && isAnyZero(Mask))
    return false;
 
  // If we have a fake unary shuffle, the shuffle mask is spread across two
  // inputs that are actually the same node. Re-map the mask to always point
  // into the first input.
  if (IsFakeUnary)
    for (int &M : Mask)
      if (M >= (int)Mask.size())
        M -= Mask.size();
 
  // If we didn't already add operands in the opcode-specific code, default to
  // adding 1 or 2 operands starting at 0.
  if (Ops.empty()) {
    Ops.push_back(N.getOperand(0));
    if (!IsUnary || IsFakeUnary)
      Ops.push_back(N.getOperand(1));
  }
 
  return true;
}
 
// Wrapper for getTargetShuffleMask with InUnary;
static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
                                 SmallVectorImpl<SDValue> &Ops,
                                 SmallVectorImpl<int> &Mask) {
  bool IsUnary;
  return getTargetShuffleMask(N, AllowSentinelZero, Ops, Mask, IsUnary);
}
 
/// Compute whether each element of a shuffle is zeroable.
///
/// A "zeroable" vector shuffle element is one which can be lowered to zero.
/// Either it is an undef element in the shuffle mask, the element of the input
/// referenced is undef, or the element of the input referenced is known to be
/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
/// as many lanes with this technique as possible to simplify the remaining
/// shuffle.
static void computeZeroableShuffleElements(ArrayRef<int> Mask,
                                           SDValue V1, SDValue V2,
                                           APInt &KnownUndef, APInt &KnownZero) {
  int Size = Mask.size();
  KnownUndef = KnownZero = APInt::getZero(Size);
 
  V1 = peekThroughBitcasts(V1);
  V2 = peekThroughBitcasts(V2);
 
  bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
  bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
 
  int VectorSizeInBits = V1.getValueSizeInBits();
  int ScalarSizeInBits = VectorSizeInBits / Size;
  assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
 
  for (int i = 0; i < Size; ++i) {
    int M = Mask[i];
    // Handle the easy cases.
    if (M < 0) {
      KnownUndef.setBit(i);
      continue;
    }
    if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
      KnownZero.setBit(i);
      continue;
    }
 
    // Determine shuffle input and normalize the mask.
    SDValue V = M < Size ? V1 : V2;
    M %= Size;
 
    // Currently we can only search BUILD_VECTOR for UNDEF/ZERO elements.
    if (V.getOpcode() != ISD::BUILD_VECTOR)
      continue;
 
    // If the BUILD_VECTOR has fewer elements then the bitcasted portion of
    // the (larger) source element must be UNDEF/ZERO.
    if ((Size % V.getNumOperands()) == 0) {
      int Scale = Size / V->getNumOperands();
      SDValue Op = V.getOperand(M / Scale);
      if (Op.isUndef())
        KnownUndef.setBit(i);
      if (X86::isZeroNode(Op))
        KnownZero.setBit(i);
      else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
        APInt Val = Cst->getAPIntValue();
        Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
        if (Val == 0)
          KnownZero.setBit(i);
      } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
        APInt Val = Cst->getValueAPF().bitcastToAPInt();
        Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
        if (Val == 0)
          KnownZero.setBit(i);
      }
      continue;
    }
 
    // If the BUILD_VECTOR has more elements then all the (smaller) source
    // elements must be UNDEF or ZERO.
    if ((V.getNumOperands() % Size) == 0) {
      int Scale = V->getNumOperands() / Size;
      bool AllUndef = true;
      bool AllZero = true;
      for (int j = 0; j < Scale; ++j) {
        SDValue Op = V.getOperand((M * Scale) + j);
        AllUndef &= Op.isUndef();
        AllZero &= X86::isZeroNode(Op);
      }
      if (AllUndef)
        KnownUndef.setBit(i);
      if (AllZero)
        KnownZero.setBit(i);
      continue;
    }
  }
}
 
/// Decode a target shuffle mask and inputs and see if any values are
/// known to be undef or zero from their inputs.
/// Returns true if the target shuffle mask was decoded.
/// FIXME: Merge this with computeZeroableShuffleElements?
static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
                                         SmallVectorImpl<SDValue> &Ops,
                                         APInt &KnownUndef, APInt &KnownZero) {
  bool IsUnary;
  if (!isTargetShuffle(N.getOpcode()))
    return false;
 
  MVT VT = N.getSimpleValueType();
  if (!getTargetShuffleMask(N, true, Ops, Mask, IsUnary))
    return false;
 
  int Size = Mask.size();
  SDValue V1 = Ops[0];
  SDValue V2 = IsUnary ? V1 : Ops[1];
  KnownUndef = KnownZero = APInt::getZero(Size);
 
  V1 = peekThroughBitcasts(V1);
  V2 = peekThroughBitcasts(V2);
 
  assert((VT.getSizeInBits() % Size) == 0 &&
         "Illegal split of shuffle value type");
  unsigned EltSizeInBits = VT.getSizeInBits() / Size;
 
  // Extract known constant input data.
  APInt UndefSrcElts[2];
  SmallVector<APInt, 32> SrcEltBits[2];
  bool IsSrcConstant[2] = {
      getTargetConstantBitsFromNode(V1, EltSizeInBits, UndefSrcElts[0],
                                    SrcEltBits[0], /*AllowWholeUndefs*/ true,
                                    /*AllowPartialUndefs*/ false),
      getTargetConstantBitsFromNode(V2, EltSizeInBits, UndefSrcElts[1],
                                    SrcEltBits[1], /*AllowWholeUndefs*/ true,
                                    /*AllowPartialUndefs*/ false)};
 
  for (int i = 0; i < Size; ++i) {
    int M = Mask[i];
 
    // Already decoded as SM_SentinelZero / SM_SentinelUndef.
    if (M < 0) {
      assert(isUndefOrZero(M) && "Unknown shuffle sentinel value!");
      if (SM_SentinelUndef == M)
        KnownUndef.setBit(i);
      if (SM_SentinelZero == M)
        KnownZero.setBit(i);
      continue;
    }
 
    // Determine shuffle input and normalize the mask.
    unsigned SrcIdx = M / Size;
    SDValue V = M < Size ? V1 : V2;
    M %= Size;
 
    // We are referencing an UNDEF input.
    if (V.isUndef()) {
      KnownUndef.setBit(i);
      continue;
    }
 
    // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF.
    // TODO: We currently only set UNDEF for integer types - floats use the same
    // registers as vectors and many of the scalar folded loads rely on the
    // SCALAR_TO_VECTOR pattern.
    if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
        (Size % V.getValueType().getVectorNumElements()) == 0) {
      int Scale = Size / V.getValueType().getVectorNumElements();
      int Idx = M / Scale;
      if (Idx != 0 && !VT.isFloatingPoint())
        KnownUndef.setBit(i);
      else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
        KnownZero.setBit(i);
      continue;
    }
 
    // INSERT_SUBVECTOR - to widen vectors we often insert them into UNDEF
    // base vectors.
    if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
      SDValue Vec = V.getOperand(0);
      int NumVecElts = Vec.getValueType().getVectorNumElements();
      if (Vec.isUndef() && Size == NumVecElts) {
        int Idx = V.getConstantOperandVal(2);
        int NumSubElts = V.getOperand(1).getValueType().getVectorNumElements();
        if (M < Idx || (Idx + NumSubElts) <= M)
          KnownUndef.setBit(i);
      }
      continue;
    }
 
    // Attempt to extract from the source's constant bits.
    if (IsSrcConstant[SrcIdx]) {
      if (UndefSrcElts[SrcIdx][M])
        KnownUndef.setBit(i);
      else if (SrcEltBits[SrcIdx][M] == 0)
        KnownZero.setBit(i);
    }
  }
 
  assert(VT.getVectorNumElements() == (unsigned)Size &&
         "Different mask size from vector size!");
  return true;
}
 
// Replace target shuffle mask elements with known undef/zero sentinels.
static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
                                              const APInt &KnownUndef,
                                              const APInt &KnownZero,
                                              bool ResolveKnownZeros= true) {
  unsigned NumElts = Mask.size();
  assert(KnownUndef.getBitWidth() == NumElts &&
         KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
 
  for (unsigned i = 0; i != NumElts; ++i) {
    if (KnownUndef[i])
      Mask[i] = SM_SentinelUndef;
    else if (ResolveKnownZeros && KnownZero[i])
      Mask[i] = SM_SentinelZero;
  }
}
 
// Extract target shuffle mask sentinel elements to known undef/zero bitmasks.
static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl<int> &Mask,
                                              APInt &KnownUndef,
                                              APInt &KnownZero) {
  unsigned NumElts = Mask.size();
  KnownUndef = KnownZero = APInt::getZero(NumElts);
 
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = Mask[i];
    if (SM_SentinelUndef == M)
      KnownUndef.setBit(i);
    if (SM_SentinelZero == M)
      KnownZero.setBit(i);
  }
}
 
// Attempt to create a shuffle mask from a VSELECT/BLENDV condition mask.
static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
                                         SDValue Cond, bool IsBLENDV = false) {
  EVT CondVT = Cond.getValueType();
  unsigned EltSizeInBits = CondVT.getScalarSizeInBits();
  unsigned NumElts = CondVT.getVectorNumElements();
 
  APInt UndefElts;
  SmallVector<APInt, 32> EltBits;
  if (!getTargetConstantBitsFromNode(Cond, EltSizeInBits, UndefElts, EltBits,
                                     /*AllowWholeUndefs*/ true,
                                     /*AllowPartialUndefs*/ false))
    return false;
 
  Mask.resize(NumElts, SM_SentinelUndef);
 
  for (int i = 0; i != (int)NumElts; ++i) {
    Mask[i] = i;
    // Arbitrarily choose from the 2nd operand if the select condition element
    // is undef.
    // TODO: Can we do better by matching patterns such as even/odd?
    if (UndefElts[i] || (!IsBLENDV && EltBits[i].isZero()) ||
        (IsBLENDV && EltBits[i].isNonNegative()))
      Mask[i] += NumElts;
  }
 
  return true;
}
 
// Forward declaration (for getFauxShuffleMask recursive check).
static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
                                   SmallVectorImpl<SDValue> &Inputs,
                                   SmallVectorImpl<int> &Mask,
                                   const SelectionDAG &DAG, unsigned Depth,
                                   bool ResolveKnownElts);
 
// Attempt to decode ops that could be represented as a shuffle mask.
// The decoded shuffle mask may contain a different number of elements to the
// destination value type.
// TODO: Merge into getTargetShuffleInputs()
static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
                               SmallVectorImpl<int> &Mask,
                               SmallVectorImpl<SDValue> &Ops,
                               const SelectionDAG &DAG, unsigned Depth,
                               bool ResolveKnownElts) {
  Mask.clear();
  Ops.clear();
 
  MVT VT = N.getSimpleValueType();
  unsigned NumElts = VT.getVectorNumElements();
  unsigned NumSizeInBits = VT.getSizeInBits();
  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
  if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
    return false;
  assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size");
  unsigned NumSizeInBytes = NumSizeInBits / 8;
  unsigned NumBytesPerElt = NumBitsPerElt / 8;
 
  unsigned Opcode = N.getOpcode();
  switch (Opcode) {
  case ISD::VECTOR_SHUFFLE: {
    // Don't treat ISD::VECTOR_SHUFFLE as a target shuffle so decode it here.
    ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask();
    if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) {
      Mask.append(ShuffleMask.begin(), ShuffleMask.end());
      Ops.push_back(N.getOperand(0));
      Ops.push_back(N.getOperand(1));
      return true;
    }
    return false;
  }
  case ISD::AND:
  case X86ISD::ANDNP: {
    // Attempt to decode as a per-byte mask.
    APInt UndefElts;
    SmallVector<APInt, 32> EltBits;
    SDValue N0 = N.getOperand(0);
    SDValue N1 = N.getOperand(1);
    bool IsAndN = (X86ISD::ANDNP == Opcode);
    uint64_t ZeroMask = IsAndN ? 255 : 0;
    if (!getTargetConstantBitsFromNode(IsAndN ? N0 : N1, 8, UndefElts, EltBits,
                                       /*AllowWholeUndefs*/ false,
                                       /*AllowPartialUndefs*/ false))
      return false;
    // We can't assume an undef src element gives an undef dst - the other src
    // might be zero.
    assert(UndefElts.isZero() && "Unexpected UNDEF element in AND/ANDNP mask");
    for (int i = 0, e = (int)EltBits.size(); i != e; ++i) {
      const APInt &ByteBits = EltBits[i];
      if (ByteBits != 0 && ByteBits != 255)
        return false;
      Mask.push_back(ByteBits == ZeroMask ? SM_SentinelZero : i);
    }
    Ops.push_back(IsAndN ? N1 : N0);
    return true;
  }
  case ISD::OR: {
    // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
    // is a valid shuffle index.
    SDValue N0 = peekThroughBitcasts(N.getOperand(0));
    SDValue N1 = peekThroughBitcasts(N.getOperand(1));
    if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
      return false;
 
    SmallVector<int, 64> SrcMask0, SrcMask1;
    SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
    APInt Demand0 = APInt::getAllOnes(N0.getValueType().getVectorNumElements());
    APInt Demand1 = APInt::getAllOnes(N1.getValueType().getVectorNumElements());
    if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG,
                                Depth + 1, true) ||
        !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG,
                                Depth + 1, true))
      return false;
 
    size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
    SmallVector<int, 64> Mask0, Mask1;
    narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
    narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
    for (int i = 0; i != (int)MaskSize; ++i) {
      // NOTE: Don't handle demanded SM_SentinelUndef, as we can end up in
      // infinite loops converting between OR and BLEND shuffles due to
      // canWidenShuffleElements merging away undef elements, meaning we
      // fail to recognise the OR as the undef element isn't known zero.
      if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
        Mask.push_back(SM_SentinelZero);
      else if (Mask1[i] == SM_SentinelZero)
        Mask.push_back(i);
      else if (Mask0[i] == SM_SentinelZero)
        Mask.push_back(i + MaskSize);
      else if (MaskSize == NumElts && !DemandedElts[i])
        Mask.push_back(SM_SentinelUndef);
      else
        return false;
    }
    Ops.push_back(N.getOperand(0));
    Ops.push_back(N.getOperand(1));
    return true;
  }
  case ISD::CONCAT_VECTORS: {
    // Limit this to vXi64 vector cases to make the most of cross lane shuffles.
    unsigned NumSubElts = N.getOperand(0).getValueType().getVectorNumElements();
    if (NumBitsPerElt == 64) {
      for (unsigned I = 0, E = N.getNumOperands(); I != E; ++I) {
        for (unsigned M = 0; M != NumSubElts; ++M)
          Mask.push_back((I * NumElts) + M);
        Ops.push_back(N.getOperand(I));
      }
      return true;
    }
    return false;
  }
  case ISD::INSERT_SUBVECTOR: {
    SDValue Src = N.getOperand(0);
    SDValue Sub = N.getOperand(1);
    EVT SubVT = Sub.getValueType();
    unsigned NumSubElts = SubVT.getVectorNumElements();
    uint64_t InsertIdx = N.getConstantOperandVal(2);
    // Subvector isn't demanded - just return the base vector.
    if (DemandedElts.extractBits(NumSubElts, InsertIdx) == 0) {
      Mask.resize(NumElts);
      std::iota(Mask.begin(), Mask.end(), 0);
      Ops.push_back(Src);
      return true;
    }
    // Handle CONCAT(SUB0, SUB1).
    // Limit to vXi64/splat cases to make the most of cross lane shuffles.
    if (Depth > 0 && InsertIdx == NumSubElts && NumElts == (2 * NumSubElts) &&
        Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
        Src.getOperand(0).isUndef() &&
        Src.getOperand(1).getValueType() == SubVT &&
        Src.getConstantOperandVal(2) == 0 &&
        (NumBitsPerElt == 64 || Src.getOperand(1) == Sub) &&
        SDNode::areOnlyUsersOf({N.getNode(), Src.getNode()}, Sub.getNode())) {
      Mask.resize(NumElts);
      std::iota(Mask.begin(), Mask.begin() + NumSubElts, 0);
      std::iota(Mask.begin() + NumSubElts, Mask.end(), NumElts);
      Ops.push_back(Src.getOperand(1));
      Ops.push_back(Sub);
      return true;
    }
    // Handle INSERT_SUBVECTOR(UNDEF, SUB, IDX) iff IDX != 0
    if (InsertIdx != 0 && Src.isUndef() &&
        peekThroughBitcasts(Sub).getOpcode() != ISD::EXTRACT_SUBVECTOR) {
      Mask.assign(NumElts, SM_SentinelUndef);
      std::iota(Mask.begin() + InsertIdx, Mask.begin() + InsertIdx + NumSubElts,
                0);
      Ops.push_back(Sub);
      return true;
    }
    if (!N->isOnlyUserOf(Sub.getNode()))
      return false;
 
    SmallVector<int, 64> SubMask;
    SmallVector<SDValue, 2> SubInputs;
    SDValue SubSrc = peekThroughOneUseBitcasts(Sub);
    EVT SubSrcVT = SubSrc.getValueType();
    if (!SubSrcVT.isVector())
      return false;
 
    // Handle INSERT_SUBVECTOR(SRC0, EXTRACT_SUBVECTOR(SRC1)).
    if (SubSrc.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
        SubSrc.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
      uint64_t ExtractIdx = SubSrc.getConstantOperandVal(1);
      SDValue SubSrcSrc = SubSrc.getOperand(0);
      unsigned NumSubSrcSrcElts =
          SubSrcSrc.getValueType().getVectorNumElements();
      unsigned MaxElts = std::max(NumElts, NumSubSrcSrcElts);
      assert((MaxElts % NumElts) == 0 && (MaxElts % NumSubSrcSrcElts) == 0 &&
             "Subvector valuetype mismatch");
      InsertIdx *= (MaxElts / NumElts);
      ExtractIdx *= (MaxElts / NumSubSrcSrcElts);
      NumSubElts *= (MaxElts / NumElts);
      bool SrcIsUndef = Src.isUndef();
      for (int i = 0; i != (int)MaxElts; ++i)
        Mask.push_back(SrcIsUndef ? SM_SentinelUndef : i);
      for (int i = 0; i != (int)NumSubElts; ++i)
        Mask[InsertIdx + i] = (SrcIsUndef ? 0 : MaxElts) + ExtractIdx + i;
      if (!SrcIsUndef)
        Ops.push_back(Src);
      Ops.push_back(SubSrcSrc);
      return true;
    }
 
    // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
    APInt SubDemand = APInt::getAllOnes(SubSrcVT.getVectorNumElements());
    if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG,
                                Depth + 1, ResolveKnownElts))
      return false;
 
    // Subvector shuffle inputs must not be larger than the subvector.
    if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
          return SubVT.getFixedSizeInBits() <
                 SubInput.getValueSizeInBits().getFixedValue();
        }))
      return false;
 
    if (SubMask.size() != NumSubElts) {
      assert(((SubMask.size() % NumSubElts) == 0 ||
              (NumSubElts % SubMask.size()) == 0) &&
             "Illegal submask scale");
      if ((NumSubElts % SubMask.size()) == 0) {
        int Scale = NumSubElts / SubMask.size();
        SmallVector<int, 64> ScaledSubMask;
        narrowShuffleMaskElts(Scale, SubMask, ScaledSubMask);
        SubMask = ScaledSubMask;
      } else {
        int Scale = SubMask.size() / NumSubElts;
        NumSubElts = SubMask.size();
        NumElts *= Scale;
        InsertIdx *= Scale;
      }
    }
    Ops.push_back(Src);
    Ops.append(SubInputs.begin(), SubInputs.end());
    if (ISD::isBuildVectorAllZeros(Src.getNode()))
      Mask.append(NumElts, SM_SentinelZero);
    else
      for (int i = 0; i != (int)NumElts; ++i)
        Mask.push_back(i);
    for (int i = 0; i != (int)NumSubElts; ++i) {
      int M = SubMask[i];
      if (0 <= M) {
        int InputIdx = M / NumSubElts;
        M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
      }
      Mask[i + InsertIdx] = M;
    }
    return true;
  }
  case X86ISD::PINSRB:
  case X86ISD::PINSRW:
  case ISD::SCALAR_TO_VECTOR:
  case ISD::INSERT_VECTOR_ELT: {
    // Match against a insert_vector_elt/scalar_to_vector of an extract from a
    // vector, for matching src/dst vector types.
    SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
 
    unsigned DstIdx = 0;
    if (Opcode != ISD::SCALAR_TO_VECTOR) {
      // Check we have an in-range constant insertion index.
      if (!isa<ConstantSDNode>(N.getOperand(2)) ||
          N.getConstantOperandAPInt(2).uge(NumElts))
        return false;
      DstIdx = N.getConstantOperandVal(2);
 
      // Attempt to recognise an INSERT*(VEC, 0, DstIdx) shuffle pattern.
      if (X86::isZeroNode(Scl)) {
        Ops.push_back(N.getOperand(0));
        for (unsigned i = 0; i != NumElts; ++i)
          Mask.push_back(i == DstIdx ? SM_SentinelZero : (int)i);
        return true;
      }
    }
 
    // Peek through trunc/aext/zext/bitcast.
    // TODO: aext shouldn't require SM_SentinelZero padding.
    // TODO: handle shift of scalars.
    unsigned MinBitsPerElt = Scl.getScalarValueSizeInBits();
    while (Scl.getOpcode() == ISD::TRUNCATE ||
           Scl.getOpcode() == ISD::ANY_EXTEND ||
           Scl.getOpcode() == ISD::ZERO_EXTEND ||
           (Scl.getOpcode() == ISD::BITCAST &&
            Scl.getScalarValueSizeInBits() ==
                Scl.getOperand(0).getScalarValueSizeInBits())) {
      Scl = Scl.getOperand(0);
      MinBitsPerElt =
          std::min<unsigned>(MinBitsPerElt, Scl.getScalarValueSizeInBits());
    }
    if ((MinBitsPerElt % 8) != 0)
      return false;
 
    // Attempt to find the source vector the scalar was extracted from.
    SDValue SrcExtract;
    if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
         Scl.getOpcode() == X86ISD::PEXTRW ||
         Scl.getOpcode() == X86ISD::PEXTRB) &&
        Scl.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
      SrcExtract = Scl;
    }
    if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
      return false;
 
    SDValue SrcVec = SrcExtract.getOperand(0);
    EVT SrcVT = SrcVec.getValueType();
    if (!SrcVT.getScalarType().isByteSized())
      return false;
    unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
    unsigned SrcByte = SrcIdx * (SrcVT.getScalarSizeInBits() / 8);
    unsigned DstByte = DstIdx * NumBytesPerElt;
    MinBitsPerElt =
        std::min<unsigned>(MinBitsPerElt, SrcVT.getScalarSizeInBits());
 
    // Create 'identity' byte level shuffle mask and then add inserted bytes.
    if (Opcode == ISD::SCALAR_TO_VECTOR) {
      Ops.push_back(SrcVec);
      Mask.append(NumSizeInBytes, SM_SentinelUndef);
    } else {
      Ops.push_back(SrcVec);
      Ops.push_back(N.getOperand(0));
      for (int i = 0; i != (int)NumSizeInBytes; ++i)
        Mask.push_back(NumSizeInBytes + i);
    }
 
    unsigned MinBytesPerElts = MinBitsPerElt / 8;
    MinBytesPerElts = std::min(MinBytesPerElts, NumBytesPerElt);
    for (unsigned i = 0; i != MinBytesPerElts; ++i)
      Mask[DstByte + i] = SrcByte + i;
    for (unsigned i = MinBytesPerElts; i < NumBytesPerElt; ++i)
      Mask[DstByte + i] = SM_SentinelZero;
    return true;
  }
  case X86ISD::PACKSS:
  case X86ISD::PACKUS: {
    SDValue N0 = N.getOperand(0);
    SDValue N1 = N.getOperand(1);
    assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
           N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
           "Unexpected input value type");
 
    APInt EltsLHS, EltsRHS;
    getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
 
    // If we know input saturation won't happen (or we don't care for particular
    // lanes), we can treat this as a truncation shuffle.
    bool Offset0 = false, Offset1 = false;
    if (Opcode == X86ISD::PACKSS) {
      if ((!(N0.isUndef() || EltsLHS.isZero()) &&
           DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
          (!(N1.isUndef() || EltsRHS.isZero()) &&
           DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
        return false;
      // We can't easily fold ASHR into a shuffle, but if it was feeding a
      // PACKSS then it was likely being used for sign-extension for a
      // truncation, so just peek through and adjust the mask accordingly.
      if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) &&
          N0.getConstantOperandAPInt(1) == NumBitsPerElt) {
        Offset0 = true;
        N0 = N0.getOperand(0);
      }
      if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) &&
          N1.getConstantOperandAPInt(1) == NumBitsPerElt) {
        Offset1 = true;
        N1 = N1.getOperand(0);
      }
    } else {
      APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
      if ((!(N0.isUndef() || EltsLHS.isZero()) &&
           !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
          (!(N1.isUndef() || EltsRHS.isZero()) &&
           !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
        return false;
    }
 
    bool IsUnary = (N0 == N1);
 
    Ops.push_back(N0);
    if (!IsUnary)
      Ops.push_back(N1);
 
    createPackShuffleMask(VT, Mask, IsUnary);
 
    if (Offset0 || Offset1) {
      for (int &M : Mask)
        if ((Offset0 && isInRange(M, 0, NumElts)) ||
            (Offset1 && isInRange(M, NumElts, 2 * NumElts)))
          ++M;
    }
    return true;
  }
  case ISD::VSELECT:
  case X86ISD::BLENDV: {
    SDValue Cond = N.getOperand(0);
    if (createShuffleMaskFromVSELECT(Mask, Cond, Opcode == X86ISD::BLENDV)) {
      Ops.push_back(N.getOperand(1));
      Ops.push_back(N.getOperand(2));
      return true;
    }
    return false;
  }
  case X86ISD::VTRUNC: {
    SDValue Src = N.getOperand(0);
    EVT SrcVT = Src.getValueType();
    if (SrcVT.getSizeInBits() != NumSizeInBits)
      return false;
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
    unsigned Scale = NumBitsPerSrcElt / NumBitsPerElt;
    assert((NumBitsPerSrcElt % NumBitsPerElt) == 0 && "Illegal truncation");
    for (unsigned i = 0; i != NumSrcElts; ++i)
      Mask.push_back(i * Scale);
    Mask.append(NumElts - NumSrcElts, SM_SentinelZero);
    Ops.push_back(Src);
    return true;
  }
  case ISD::SHL:
  case ISD::SRL: {
    APInt UndefElts;
    SmallVector<APInt, 32> EltBits;
    if (!getTargetConstantBitsFromNode(N.getOperand(1), NumBitsPerElt,
                                       UndefElts, EltBits,
                                       /*AllowWholeUndefs*/ true,
                                       /*AllowPartialUndefs*/ false))
      return false;
 
    // We can only decode 'whole byte' bit shifts as shuffles.
    for (unsigned I = 0; I != NumElts; ++I)
      if (DemandedElts[I] && !UndefElts[I] &&
          (EltBits[I].urem(8) != 0 || EltBits[I].uge(NumBitsPerElt)))
        return false;
 
    Mask.append(NumSizeInBytes, SM_SentinelUndef);
    Ops.push_back(N.getOperand(0));
 
    for (unsigned I = 0; I != NumElts; ++I) {
      if (!DemandedElts[I] || UndefElts[I])
        continue;
      unsigned ByteShift = EltBits[I].getZExtValue() / 8;
      unsigned Lo = I * NumBytesPerElt;
      unsigned Hi = Lo + NumBytesPerElt;
      // Clear mask to all zeros and insert the shifted byte indices.
      std::fill(Mask.begin() + Lo, Mask.begin() + Hi, SM_SentinelZero);
      if (ISD::SHL == Opcode)
        std::iota(Mask.begin() + Lo + ByteShift, Mask.begin() + Hi, Lo);
      else
        std::iota(Mask.begin() + Lo, Mask.begin() + Hi - ByteShift,
                  Lo + ByteShift);
    }
    return true;
  }
  case X86ISD::VSHLI:
  case X86ISD::VSRLI: {
    uint64_t ShiftVal = N.getConstantOperandVal(1);
    // Out of range bit shifts are guaranteed to be zero.
    if (NumBitsPerElt <= ShiftVal) {
      Mask.append(NumElts, SM_SentinelZero);
      return true;
    }
 
    // We can only decode 'whole byte' bit shifts as shuffles.
    if ((ShiftVal % 8) != 0)
      break;
 
    uint64_t ByteShift = ShiftVal / 8;
    Ops.push_back(N.getOperand(0));
 
    // Clear mask to all zeros and insert the shifted byte indices.
    Mask.append(NumSizeInBytes, SM_SentinelZero);
 
    if (X86ISD::VSHLI == Opcode) {
      for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
        for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
          Mask[i + j] = i + j - ByteShift;
    } else {
      for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
        for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
          Mask[i + j - ByteShift] = i + j;
    }
    return true;
  }
  case ISD::ROTL:
  case ISD::ROTR: {
    APInt UndefElts;
    SmallVector<APInt, 32> EltBits;
    if (!getTargetConstantBitsFromNode(N.getOperand(1), NumBitsPerElt,
                                       UndefElts, EltBits,
                                       /*AllowWholeUndefs*/ true,
                                       /*AllowPartialUndefs*/ false))
      return false;
 
    // We can only decode 'whole byte' bit rotates as shuffles.
    for (unsigned I = 0; I != NumElts; ++I)
      if (DemandedElts[I] && !UndefElts[I] &&
          (EltBits[I].urem(NumBitsPerElt) % 8) != 0)
        return false;
 
    Ops.push_back(N.getOperand(0));
    for (unsigned I = 0; I != NumElts; ++I) {
      if (!DemandedElts[I] || UndefElts[I]) {
        Mask.append(NumBytesPerElt, SM_SentinelUndef);
        continue;
      }
      int Offset = EltBits[I].urem(NumBitsPerElt) / 8;
      Offset = (ISD::ROTL == Opcode ? NumBytesPerElt - Offset : Offset);
      int BaseIdx = I * NumBytesPerElt;
      for (int J = 0; J != (int)NumBytesPerElt; ++J) {
        Mask.push_back(BaseIdx + ((Offset + J) % NumBytesPerElt));
      }
    }
    return true;
  }
  case X86ISD::VROTLI:
  case X86ISD::VROTRI: {
    // We can only decode 'whole byte' bit rotates as shuffles.
    uint64_t RotateVal = N.getConstantOperandAPInt(1).urem(NumBitsPerElt);
    if ((RotateVal % 8) != 0)
      return false;
    Ops.push_back(N.getOperand(0));
    int Offset = RotateVal / 8;
    Offset = (X86ISD::VROTLI == Opcode ? NumBytesPerElt - Offset : Offset);
    for (int i = 0; i != (int)NumElts; ++i) {
      int BaseIdx = i * NumBytesPerElt;
      for (int j = 0; j != (int)NumBytesPerElt; ++j) {
        Mask.push_back(BaseIdx + ((Offset + j) % NumBytesPerElt));
      }
    }
    return true;
  }
  case X86ISD::VSHLD:
  case X86ISD::VSHRD: {
    // We can only decode 'whole byte' bit funnel shifts as shuffles.
    uint64_t ShiftVal = N.getConstantOperandAPInt(2).urem(NumBitsPerElt);
    int Offset = ShiftVal / 8;
    if ((ShiftVal % 8) != 0 || Offset == 0)
      return false;
    Ops.push_back(N.getOperand(X86ISD::VSHRD == Opcode ? 1 : 0));
    Ops.push_back(N.getOperand(X86ISD::VSHRD == Opcode ? 0 : 1));
    Offset = X86ISD::VSHRD == Opcode ? (NumBytesPerElt - Offset) : Offset;
    for (int I = 0; I != (int)NumElts; ++I) {
      int BaseIdx = (I * NumBytesPerElt) - Offset;
      for (int J = 0; J != (int)NumBytesPerElt; ++J) {
        int MaskIdx = BaseIdx + J;
        MaskIdx += J < Offset ? (NumSizeInBytes + NumBytesPerElt) : 0;
        Mask.push_back(MaskIdx);
      }
    }
    return true;
  }
  case X86ISD::VBROADCAST: {
    SDValue Src = N.getOperand(0);
    if (!Src.getSimpleValueType().isVector()) {
      if (Src.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
          !isNullConstant(Src.getOperand(1)) ||
          Src.getOperand(0).getValueType().getScalarType() !=
              VT.getScalarType())
        return false;
      Src = Src.getOperand(0);
    }
    Ops.push_back(Src);
    Mask.append(NumElts, 0);
    return true;
  }
  case ISD::SIGN_EXTEND_VECTOR_INREG: {
    SDValue Src = N.getOperand(0);
    EVT SrcVT = Src.getValueType();
    unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
 
    // Extended source must be a simple vector.
    if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
        (NumBitsPerSrcElt % 8) != 0)
      return false;
 
    // We can only handle all-signbits extensions.
    APInt DemandedSrcElts =
        DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
    if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt)
      return false;
 
    assert((NumBitsPerElt % NumBitsPerSrcElt) == 0 && "Unexpected extension");
    unsigned Scale = NumBitsPerElt / NumBitsPerSrcElt;
    for (unsigned I = 0; I != NumElts; ++I)
      Mask.append(Scale, I);
    Ops.push_back(Src);
    return true;
  }
  case ISD::ZERO_EXTEND:
  case ISD::ANY_EXTEND:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
  case ISD::ANY_EXTEND_VECTOR_INREG: {
    SDValue Src = N.getOperand(0);
    EVT SrcVT = Src.getValueType();
 
    // Extended source must be a simple vector.
    if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
        (SrcVT.getScalarSizeInBits() % 8) != 0)
      return false;
 
    bool IsAnyExtend =
        (ISD::ANY_EXTEND == Opcode || ISD::ANY_EXTEND_VECTOR_INREG == Opcode);
    DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts,
                         IsAnyExtend, Mask);
    Ops.push_back(Src);
    return true;
  }
  }
 
  return false;
}
 
/// Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
                                              SmallVectorImpl<int> &Mask) {
  int MaskWidth = Mask.size();
  SmallVector<SDValue, 16> UsedInputs;
  for (int i = 0, e = Inputs.size(); i < e; ++i) {
    int lo = UsedInputs.size() * MaskWidth;
    int hi = lo + MaskWidth;
 
    // Strip UNDEF input usage.
    if (Inputs[i].isUndef())
      for (int &M : Mask)
        if ((lo <= M) && (M < hi))
          M = SM_SentinelUndef;
 
    // Check for unused inputs.
    if (none_of(Mask, [lo, hi](int i) { return (lo <= i) && (i < hi); })) {
      for (int &M : Mask)
        if (lo <= M)
          M -= MaskWidth;
      continue;
    }
 
    // Check for repeated inputs.
    bool IsRepeat = false;
    for (int j = 0, ue = UsedInputs.size(); j != ue; ++j) {
      if (peekThroughBitcasts(UsedInputs[j]) != peekThroughBitcasts(Inputs[i]))
        continue;
      for (int &M : Mask)
        if (lo <= M)
          M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth);
      IsRepeat = true;
      break;
    }
    if (IsRepeat)
      continue;
 
    UsedInputs.push_back(Inputs[i]);
  }
  Inputs = std::move(UsedInputs);
}
 
/// Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs
/// and then sets the SM_SentinelUndef and SM_SentinelZero values.
/// Returns true if the target shuffle mask was decoded.
static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
                                   SmallVectorImpl<SDValue> &Inputs,
                                   SmallVectorImpl<int> &Mask,
                                   APInt &KnownUndef, APInt &KnownZero,
                                   const SelectionDAG &DAG, unsigned Depth,
                                   bool ResolveKnownElts) {
  if (Depth >= SelectionDAG::MaxRecursionDepth)
    return false; // Limit search depth.
 
  EVT VT = Op.getValueType();
  if (!VT.isSimple() || !VT.isVector())
    return false;
 
  if (getTargetShuffleAndZeroables(Op, Mask, Inputs, KnownUndef, KnownZero)) {
    if (ResolveKnownElts)
      resolveTargetShuffleFromZeroables(Mask, KnownUndef, KnownZero);
    return true;
  }
  if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
                         ResolveKnownElts)) {
    resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
    return true;
  }
  return false;
}
 
static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
                                   SmallVectorImpl<SDValue> &Inputs,
                                   SmallVectorImpl<int> &Mask,
                                   const SelectionDAG &DAG, unsigned Depth,
                                   bool ResolveKnownElts) {
  APInt KnownUndef, KnownZero;
  return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, KnownUndef,
                                KnownZero, DAG, Depth, ResolveKnownElts);
}
 
static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
                                   SmallVectorImpl<int> &Mask,
                                   const SelectionDAG &DAG, unsigned Depth = 0,
                                   bool ResolveKnownElts = true) {
  EVT VT = Op.getValueType();
  if (!VT.isSimple() || !VT.isVector())
    return false;
 
  unsigned NumElts = Op.getValueType().getVectorNumElements();
  APInt DemandedElts = APInt::getAllOnes(NumElts);
  return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth,
                                ResolveKnownElts);
}
 
// Attempt to create a scalar/subvector broadcast from the base MemSDNode.
static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT,
                                 EVT MemVT, MemSDNode *Mem, unsigned Offset,
                                 SelectionDAG &DAG) {
  assert((Opcode == X86ISD::VBROADCAST_LOAD ||
          Opcode == X86ISD::SUBV_BROADCAST_LOAD) &&
         "Unknown broadcast load type");
 
  // Ensure this is a simple (non-atomic, non-voltile), temporal read memop.
  if (!Mem || !Mem->readMem() || !Mem->isSimple() || Mem->isNonTemporal())
    return SDValue();
 
  SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
                                         TypeSize::getFixed(Offset), DL);
  SDVTList Tys = DAG.getVTList(VT, MVT::Other);
  SDValue Ops[] = {Mem->getChain(), Ptr};
  SDValue BcstLd = DAG.getMemIntrinsicNode(
      Opcode, DL, Tys, Ops, MemVT,
      DAG.getMachineFunction().getMachineMemOperand(
          Mem->getMemOperand(), Offset, MemVT.getStoreSize()));
  DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
  return BcstLd;
}
 
/// Returns the scalar element that will make up the i'th
/// element of the result of the vector shuffle.
static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
                                   SelectionDAG &DAG, unsigned Depth) {
  if (Depth >= SelectionDAG::MaxRecursionDepth)
    return SDValue(); // Limit search depth.
 
  EVT VT = Op.getValueType();
  unsigned Opcode = Op.getOpcode();
  unsigned NumElems = VT.getVectorNumElements();
 
  // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
  if (auto *SV = dyn_cast<ShuffleVectorSDNode>(Op)) {
    int Elt = SV->getMaskElt(Index);
 
    if (Elt < 0)
      return DAG.getUNDEF(VT.getVectorElementType());
 
    SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
    return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
  }
 
  // Recurse into target specific vector shuffles to find scalars.
  if (isTargetShuffle(Opcode)) {
    MVT ShufVT = VT.getSimpleVT();
    MVT ShufSVT = ShufVT.getVectorElementType();
    int NumElems = (int)ShufVT.getVectorNumElements();
    SmallVector<int, 16> ShuffleMask;
    SmallVector<SDValue, 16> ShuffleOps;
    if (!getTargetShuffleMask(Op, true, ShuffleOps, ShuffleMask))
      return SDValue();
 
    int Elt = ShuffleMask[Index];
    if (Elt == SM_SentinelZero)
      return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT)
                                 : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT);
    if (Elt == SM_SentinelUndef)
      return DAG.getUNDEF(ShufSVT);
 
    assert(0 <= Elt && Elt < (2 * NumElems) && "Shuffle index out of range");
    SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
    return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
  }
 
  // Recurse into insert_subvector base/sub vector to find scalars.
  if (Opcode == ISD::INSERT_SUBVECTOR) {
    SDValue Vec = Op.getOperand(0);
    SDValue Sub = Op.getOperand(1);
    uint64_t SubIdx = Op.getConstantOperandVal(2);
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
 
    if (SubIdx <= Index && Index < (SubIdx + NumSubElts))
      return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1);
    return getShuffleScalarElt(Vec, Index, DAG, Depth + 1);
  }
 
  // Recurse into concat_vectors sub vector to find scalars.
  if (Opcode == ISD::CONCAT_VECTORS) {
    EVT SubVT = Op.getOperand(0).getValueType();
    unsigned NumSubElts = SubVT.getVectorNumElements();
    uint64_t SubIdx = Index / NumSubElts;
    uint64_t SubElt = Index % NumSubElts;
    return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1);
  }
 
  // Recurse into extract_subvector src vector to find scalars.
  if (Opcode == ISD::EXTRACT_SUBVECTOR) {
    SDValue Src = Op.getOperand(0);
    uint64_t SrcIdx = Op.getConstantOperandVal(1);
    return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1);
  }
 
  // We only peek through bitcasts of the same vector width.
  if (Opcode == ISD::BITCAST) {
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems)
      return getShuffleScalarElt(Src, Index, DAG, Depth + 1);
    return SDValue();
  }
 
  // Actual nodes that may contain scalar elements
 
  // For insert_vector_elt - either return the index matching scalar or recurse
  // into the base vector.
  if (Opcode == ISD::INSERT_VECTOR_ELT &&
      isa<ConstantSDNode>(Op.getOperand(2))) {
    if (Op.getConstantOperandAPInt(2) == Index)
      return Op.getOperand(1);
    return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1);
  }
 
  if (Opcode == ISD::SCALAR_TO_VECTOR)
    return (Index == 0) ? Op.getOperand(0)
                        : DAG.getUNDEF(VT.getVectorElementType());
 
  if (Opcode == ISD::BUILD_VECTOR)
    return Op.getOperand(Index);
 
  return SDValue();
}
 
// Use PINSRB/PINSRW/PINSRD to create a build vector.
static SDValue LowerBuildVectorAsInsert(SDValue Op, const SDLoc &DL,
                                        const APInt &NonZeroMask,
                                        unsigned NumNonZero, unsigned NumZero,
                                        SelectionDAG &DAG,
                                        const X86Subtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  unsigned NumElts = VT.getVectorNumElements();
  assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||
          ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&
         "Illegal vector insertion");
 
  SDValue V;
  bool First = true;
 
  for (unsigned i = 0; i < NumElts; ++i) {
    bool IsNonZero = NonZeroMask[i];
    if (!IsNonZero)
      continue;
 
    // If the build vector contains zeros or our first insertion is not the
    // first index then insert into zero vector to break any register
    // dependency else use SCALAR_TO_VECTOR.
    if (First) {
      First = false;
      if (NumZero || 0 != i)
        V = getZeroVector(VT, Subtarget, DAG, DL);
      else {
        assert(0 == i && "Expected insertion into zero-index");
        V = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
        V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V);
        V = DAG.getBitcast(VT, V);
        continue;
      }
    }
    V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, V, Op.getOperand(i),
                    DAG.getVectorIdxConstant(i, DL));
  }
 
  return V;
}
 
/// Custom lower build_vector of v16i8.
static SDValue LowerBuildVectorv16i8(SDValue Op, const SDLoc &DL,