X86ISelLowering.cpp File Reference

#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>
#include "X86GenAsmMatcher.inc"

Go to the source code of this file.

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.
namespace	llvm::X86
	Define some predicates that are used for node matching.

Macros
#define	DEBUG_TYPE   "x86-isel"
#define	GET_REGISTER_MATCHER
#define	GET_EGPR_IF_ENABLED(OPC)

Enumerations
enum	BitTestKind : unsigned {   UndefBit , ConstantBit , NotConstantBit , ShiftBit ,   NotShiftBit }
enum	{ llvm::X86::MaxShuffleCombineDepth = 8 }
enum class	ShrinkMode { MULS8 , MULU8 , MULS16 , MULU16 }
	Different mul shrinking modes. More...

Functions
static bool	mayFoldIntoVector (SDValue Op, const SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	isLogicOp (unsigned Opcode)
static bool	isTargetShuffle (unsigned Opcode)
static bool	isTargetShuffleVariableMask (unsigned Opcode)
static bool	isX86CCSigned (X86::CondCode X86CC)
	Return true if the condition is an signed comparison operation.
static X86::CondCode	TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
static X86::CondCode	TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
	Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
static bool	hasFPCMov (unsigned X86CC)
	Is there a floating point cmov for the specific X86 condition code?
static bool	useVPTERNLOG (const X86Subtarget &Subtarget, MVT VT)
static bool	isUndefOrEqual (int Val, int CmpVal)
	Val is the undef sentinel value or equal to the specified value.
static bool	isUndefOrEqual (ArrayRef< int > Mask, int CmpVal)
	Return true if every element in Mask is the undef sentinel value or equal to the specified value.
static bool	isUndefOrEqualInRange (ArrayRef< int > Mask, int CmpVal, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.
static bool	isUndefOrZero (int Val)
	Val is either the undef or zero sentinel value.
static bool	isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
static bool	isUndefLowerHalf (ArrayRef< int > Mask)
	Return true if the mask creates a vector whose lower half is undefined.
static bool	isUndefUpperHalf (ArrayRef< int > Mask)
	Return true if the mask creates a vector whose upper half is undefined.
static bool	isInRange (int Val, int Low, int Hi)
	Return true if Val falls within the specified range (L, H].
static bool	isAnyInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if the value of any element in Mask falls within the specified range (L, H].
static bool	isAnyZero (ArrayRef< int > Mask)
	Return true if the value of any element in Mask is the zero sentinel value.
static bool	isUndefOrInRange (int Val, int Low, int Hi)
	Return true if Val is undef or if its value falls within the specified range (L, H].
static bool	isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
static bool	isUndefOrZeroOrInRange (int Val, int Low, int Hi)
	Return true if Val is undef, zero or if its value falls within the specified range (L, H].
static bool	isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
static bool	isBlendOrUndef (ArrayRef< int > Mask)
	Return true if every element in Mask, is an in-place blend/select mask or is undef.
static bool	isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
	Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
static bool	isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
static bool	isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
static bool	isCompletePermute (ArrayRef< int > Mask)
	Return true if every element of a single input is referenced by the shuffle mask.
static bool	canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
	Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
static bool	canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, bool V2IsZero, SmallVectorImpl< int > &WidenedMask)
static bool	canWidenShuffleElements (ArrayRef< int > Mask)
static bool	scaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask)
static bool	canScaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts)
static void	growShuffleMask (ArrayRef< int > SrcMask, SmallVectorImpl< int > &DstMask, unsigned SrcSizeInBits, unsigned DstSizeInBits)
static SDValue	getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
static SDValue	getConstVector (ArrayRef< APInt > Bits, const APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getConstVector (ArrayRef< APInt > Bits, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Returns a vector of specified type with all zero elements.
static SDValue	getSplitVectorSrc (SDValue LHS, SDValue RHS, bool AllowCommute)
static SDValue	extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
static SDValue	extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to grab 128-bits from a vector > 128 bits.
static SDValue	extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to grab 256-bits from a 512-bit vector.
static SDValue	insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
static SDValue	insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to put 128-bits into a vector > 128 bits.
static SDValue	widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
static SDValue	widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits)
	Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
static MVT	widenMaskVectorType (MVT VT, const X86Subtarget &Subtarget)
	Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
static SDValue	widenMaskVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
static bool	collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG)
static SDValue	isUpperSubvectorUndef (SDValue V, const SDLoc &DL, SelectionDAG &DAG)
static bool	isFreeToSplitVector (SDValue V, SelectionDAG &DAG)
static std::pair< SDValue, SDValue >	splitVector (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	splitVectorOp (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break an operation into 2 half sized ops and then concatenate the results.
static SDValue	splitVectorIntUnary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break an unary integer operation into 2 half sized ops and then concatenate the result back.
static SDValue	splitVectorIntBinary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break a binary integer operation into 2 half sized ops and then concatenate the result back.
template<typename F>
SDValue	SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true, bool AllowAVX512=true)
static SDValue	getAVX512Node (unsigned Opcode, const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Insert i1-subvector to i1-vector.
static SDValue	concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl)
	Returns a vector of specified type with all bits set.
static unsigned	getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable)
static SDValue	getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
	Handle vector element shifts where the shift amount is a constant.
static SDValue	getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle vector element shifts by a splat shift amount.
static SDValue	getEXTEND_VECTOR_INREG (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
static SDValue	getBitSelect (const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG &DAG)
static SDValue	getVectorShuffle (SelectionDAG &DAG, EVT VT, const SDLoc &dl, SDValue V1, SDValue V2, ArrayRef< int > Mask)
static SDValue	getUnpackl (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
	Returns a vector_shuffle node for an unpackl operation.
static SDValue	getUnpackh (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
	Returns a vector_shuffle node for an unpackh operation.
static SDValue	getPack (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf=false)
	Returns a node that packs the LHS + RHS nodes together at half width.
static SDValue	getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Return a vector_shuffle of the specified vector of zero or undef vector.
static ConstantPoolSDNode *	getTargetConstantPoolFromBasePtr (SDValue Ptr)
static const Constant *	getTargetConstantFromBasePtr (SDValue Ptr)
static const Constant *	getTargetConstantFromNode (LoadSDNode *Load)
static const Constant *	getTargetConstantFromNode (SDValue Op)
static bool	getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=false)
bool	llvm::X86::isConstantSplat (SDValue Op, APInt &SplatVal, bool AllowPartialUndefs=true)
	If Op is a constant whose elements are all the same constant or undefined, return true and return the constant value in SplatVal.
int	llvm::X86::getRoundingModeX86 (unsigned RM)
	Convert LLVM rounding mode to X86 rounding mode.
static bool	getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
static bool	isConstantPowerOf2 (SDValue V, unsigned EltSizeInBIts, bool AllowUndefs)
static SDValue	IsNOT (SDValue V, SelectionDAG &DAG)
static void	createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1)
	Create a shuffle mask that matches the PACKSS/PACKUS truncation.
static void	getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
static void	getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
static bool	getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
	Calculates the shuffle mask corresponding to the target-specific opcode.
static bool	getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask)
static void	computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
	Compute whether each element of a shuffle is zeroable.
static bool	getTargetShuffleAndZeroables (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero)
	Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.
static void	resolveTargetShuffleFromZeroables (SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true)
static void	resolveZeroablesFromTargetShuffle (const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero)
static bool	createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond, bool IsBLENDV=false)
static bool	getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
static bool	getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
static void	resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
	Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
static bool	getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
	Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.
static bool	getTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth=0, bool ResolveKnownElts=true)
static SDValue	getBROADCAST_LOAD (unsigned Opcode, const SDLoc &DL, EVT VT, EVT MemVT, MemSDNode *Mem, unsigned Offset, SelectionDAG &DAG)
static SDValue	getShuffleScalarElt (SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth)
	Returns the scalar element that will make up the i'th element of the result of the vector shuffle.
static SDValue	LowerBuildVectorAsInsert (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBuildVectorv16i8 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v16i8.
static SDValue	LowerBuildVectorv8i16 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v8i16.
static SDValue	LowerBuildVectorv4x32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v4i32 or v4f32.
static SDValue	getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
	Return a vector logical shift node.
static SDValue	LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
static bool	findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
static SDValue	EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize, unsigned Depth=0)
	Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
static SDValue	combineToConsecutiveLoads (EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
static Constant *	getConstantVector (MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
static Constant *	getConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C)
static bool	isFoldableUseOfShuffle (SDNode *N)
static bool	isFoldableAsShuffle (SDValue V)
static bool	isMaskableNode (SDValue V, const X86Subtarget &Subtarget)
static SDValue	lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.
static int	getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
	For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
static SDValue	buildFromShuffleMostly (SDValue Op, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	LowerBUILD_VECTORvXbf16 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBUILD_VECTORvXi1 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	isHorizOp (unsigned Opcode)
static bool	isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses, bool AllowSubAddOrAddSubContract)
	Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).
static SDValue	LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerBuildVectorToBitOp (BuildVectorSDNode *Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
static bool	isShuffleFoldableLoad (SDValue V)
	Helper to test for a load that can be folded with x86 shuffles.
static SDValue	lowerBuildVectorAsBlend (BuildVectorSDNode *BVOp, SDLoc const &DL, X86Subtarget const &Subtarget, SelectionDAG &DAG)
	Attempt to lower a BUILD_VECTOR of scalar values to a shuffle of splats representing a blend.
static SDValue	widenBuildVector (BuildVectorSDNode *BVOp, SDLoc const &DL, X86Subtarget const &Subtarget, SelectionDAG &DAG)
	Widen a BUILD_VECTOR if the scalar operands are freely mergeable.
static SDValue	materializeVectorConstant (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Create a vector constant without a load.
static SDValue	createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.
static SDValue	LowerBUILD_VECTORAsVariablePermute (SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerAVXCONCAT_VECTORS (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerCONCAT_VECTORSvXi1 (SDValue Op, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	IsElementEquivalent (int MaskSize, SDValue Op, SDValue ExpectedOp, int Idx, int ExpectedIdx)
	Checks whether the vector elements referenced by two shuffle masks are equivalent.
static bool	isNoopShuffleMask (ArrayRef< int > Mask)
	Tiny helper function to identify a no-op mask.
static bool	isLaneCrossingShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
	Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.
static bool	is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
	Test whether there are elements crossing 128-bit lanes in this shuffle mask.
static bool	isMultiLaneShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
	Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.
static bool	isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each sub-lane.
static bool	is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each 128-bit lane.
static bool	is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask)
static bool	is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each 256-bit lane.
static bool	isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, unsigned EltSizeInBits, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a target shuffle mask is equivalent within each sub-lane.
static bool	isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a target shuffle mask is equivalent within each sub-lane.
static bool	isShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
	Checks whether a shuffle mask is equivalent to an explicit list of arguments.
static bool	isTargetShuffleEquivalent (MVT VT, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, const SelectionDAG &DAG, SDValue V1=SDValue(), SDValue V2=SDValue())
	Checks whether a target shuffle mask is equivalent to an explicit pattern.
static bool	isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT, const SelectionDAG &DAG)
static bool	is128BitUnpackShuffleMask (ArrayRef< int > Mask, const SelectionDAG &DAG)
static bool	hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask)
	Return true if a shuffle mask chooses elements identically in its top and bottom halves.
static unsigned	getV4X86ShuffleImm (ArrayRef< int > Mask)
	Get a 4-lane 8-bit shuffle immediate for a mask.
static SDValue	getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
static unsigned	getSHUFPDImm (ArrayRef< int > Mask)
static SDValue	getSHUFPDImmForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
static bool	isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
static SDValue	combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth)
	Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
static SDValue	lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a shuffle with a single PSHUFB of V1 or V2.
static SDValue	getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.
static SDValue	lowerShuffleWithEXPAND (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	matchShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleWithUNPCK256 (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.
static bool	matchShuffleAsVTRUNC (MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget)
static SDValue	getAVX512TruncNode (const SDLoc &DL, MVT DstVT, SDValue Src, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool ZeroUppers)
static SDValue	lowerShuffleWithVPMOV (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVTRUNC (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static int	canLowerByDroppingElements (ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput)
	Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.
static bool	matchShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1)
static SDValue	lowerShuffleWithPACK (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
	Try to emit a bitmask instruction for a shuffle.
static SDValue	lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Try to emit a blend instruction for a shuffle using bit math.
static SDValue	getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.
static bool	matchShuffleAsBlend (MVT VT, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
static SDValue	lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to emit a blend instruction for a shuffle.
static SDValue	lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
	Try to lower as a blend of elements from two inputs followed by a single-input permutation.
static SDValue	lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
static SDValue	lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
static SDValue	lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.
static bool	isBroadcastShuffleMask (ArrayRef< int > Mask)
static bool	isNoopOrBroadcastShuffleMask (ArrayRef< int > Mask)
static bool	isSingleElementRepeatedMask (ArrayRef< int > Mask)
	Check if the Mask consists of the same element repeated multiple times.
static SDValue	lowerShuffleAsDecomposedShuffleMerge (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic routine to decompose a shuffle and blend into independent blends and permutes.
static int	matchShuffleAsBitRotate (MVT &RotateVT, int EltSizeInBits, const X86Subtarget &Subtarget, ArrayRef< int > Mask)
static SDValue	lowerShuffleAsBitRotate (const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower shuffle using X86ISD::VROTLI rotations.
static int	matchShuffleAsElementRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
	Try to match a vector shuffle as an element rotation.
static int	matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
	Try to lower a vector shuffle as a byte rotation.
static SDValue	lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVALIGN (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a dword/qword rotation.
static SDValue	lowerShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a byte shift sequence.
static int	matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
	Try to lower a vector shuffle as a bit shift (shifts in zeros).
static SDValue	lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool BitwiseOnly)
static int	matchShuffleAsVSHLD (MVT &ShiftVT, SDValue &V1, SDValue &V2, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
	Try to match a vector shuffle as a X86ISD::VSHLD funnel shift.
static bool	matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
static bool	matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
static SDValue	lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
	Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
static SDValue	lowerShuffleAsSpecificExtension (const SDLoc &DL, MVT VT, int Scale, int Offset, unsigned ExtOpc, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower a vector shuffle as an any/signed/zero extension.
static SDValue	lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a zero extension on any microarch.
static SDValue	getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
	Try to get a scalar value for a specific element of a vector.
template<typename T>
static bool	isSoftF16 (T VT, const X86Subtarget &Subtarget)
template<typename T>
static bool	isBF16orSoftF16 (T VT, const X86Subtarget &Subtarget)
static SDValue	lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower insertion of a single element into a zero vector.
static SDValue	lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.
static bool	isSingleSHUFPSMask (ArrayRef< int > Mask)
	Test whether this can be lowered with a single SHUFPS instruction.
static bool	isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
	Test whether the specified input (0 or 1) is in-place blended by the given mask.
static bool	isShuffleMaskInputBroadcastable (int Input, ArrayRef< int > Mask, int BroadcastableElement=0)
	Test whether the specified input (0 or 1) is a broadcast/splat blended by the given mask.
static SDValue	lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
	If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering.
static SDValue	lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower broadcast of a single element.
static bool	matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
static SDValue	lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit floating point shuffles.
static SDValue	lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit integer shuffles.
static SDValue	lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
	Lower a vector shuffle using the SHUFPS instruction.
static SDValue	lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 4-lane 32-bit floating point shuffles.
static SDValue	lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 4-lane i32 vector shuffles.
static SDValue	lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
static SDValue	lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
	Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
static SDValue	lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic lowering of 8-lane i16 shuffles.
static SDValue	lowerV8F16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 8-lane 16-bit floating point shuffles.
static SDValue	lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > OriginalMask, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic lowering of v16i8 shuffles.
static SDValue	lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Dispatching routine to lower various 128-bit x86 vector shuffles.
static SDValue	splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool SimpleOnly)
	Generic routine to split vector shuffle into half-sized shuffles.
static SDValue	lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Either split a vector in halves or decompose the shuffles and the blend/unpack.
static SDValue	lowerShuffleAsLanePermuteAndSHUFP (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.
static void	computeInLaneShuffleMask (const ArrayRef< int > &Mask, int LaneSize, SmallVector< int > &InLaneMask)
	Helper to get compute inlane shuffle mask for a complete shuffle mask.
static SDValue	lowerShuffleAsLanePermuteAndShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.
static SDValue	lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering 2-lane 128-bit shuffles.
static SDValue	lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
static bool	getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
	If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.
static SDValue	getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false)
	Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.
static SDValue	lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
static SDValue	lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
static bool	matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable)
static SDValue	lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVTRUNCAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
static SDValue	lowerShufflePairAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 4-lane 64-bit floating point shuffles.
static SDValue	lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 4-lane 64-bit integer shuffles.
static SDValue	lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 32-bit floating point shuffles.
static SDValue	lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 32-bit integer shuffles.
static SDValue	lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 16-bit integer shuffles.
static SDValue	lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 32-lane 8-bit integer shuffles.
static SDValue	lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	High-level routine to lower various 256-bit x86 vector shuffles.
static SDValue	lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a 128-bit shuffles.
static SDValue	lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 64-bit floating point shuffles.
static SDValue	lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 32-bit floating point shuffles.
static SDValue	lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 64-bit integer shuffles.
static SDValue	lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 32-bit integer shuffles.
static SDValue	lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 32-lane 16-bit integer shuffles.
static SDValue	lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 64-lane 8-bit integer shuffles.
static SDValue	lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	High-level routine to lower various 512-bit x86 vector shuffles.
static SDValue	lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static int	match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
static SDValue	lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
	Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
static bool	canCombineAsMaskOperation (SDValue V, const X86Subtarget &Subtarget)
static SDValue	canonicalizeShuffleMaskWithHorizOp (MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerVECTOR_SHUFFLE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Top-level lowering for x86 vector shuffles.
static SDValue	lowerVECTOR_COMPRESS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a VSELECT instruction to a vector shuffle.
static SDValue	LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
static SDValue	ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Extract one bit from mask vector, like v16i1 or v8i1.
static APInt	getExtractedDemandedElts (SDNode *N)
static SDValue	InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Insert one bit to mask vector, like v16i1 or v8i1.
static SDValue	LowerFLDEXP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	GetTLSADDR (SelectionDAG &DAG, GlobalAddressSDNode *GA, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LoadGlobalBaseReg=false, bool LocalDynamic=false)
static SDValue	LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSGeneralDynamicModelX32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool Is64Bit, bool Is64BitLP64)
static SDValue	LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
static SDValue	LowerShiftParts (SDValue Op, SelectionDAG &DAG)
	Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
static SDValue	LowerI64IntToFP_AVX512DQ (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerI64IntToFP16 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
static SDValue	vectorizeExtractedCast (SDValue Cast, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction.
static SDValue	lowerFPToIntToFP (SDValue CastToFP, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops.
static SDValue	lowerINT_TO_FP_vXi64 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	promoteXINT_TO_FP (SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
static bool	isLegalConversion (MVT VT, MVT FloatVT, bool IsSigned, const X86Subtarget &Subtarget)
static bool	shouldUseHorizontalOp (bool IsSingleSource, const SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Horizontal vector math instructions may be slower than normal math with shuffles.
static SDValue	LowerUINT_TO_FP_i64 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	64-bit unsigned integer to double expansion.
static SDValue	LowerUINT_TO_FP_i32 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	32-bit unsigned integer to float expansion.
static SDValue	lowerUINT_TO_FP_v2i32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerUINT_TO_FP_vXi32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerUINT_TO_FP_vec (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerAVXExtend (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	LowerZERO_EXTEND_Mask (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
static SDValue	truncateVectorWithPACKUS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
static SDValue	truncateVectorWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Truncate using inreg sign extension and X86ISD::PACKSS.
static SDValue	matchTruncateWithPACK (unsigned &PackOpcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDNodeFlags Flags=SDNodeFlags())
	Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension.
static SDValue	LowerTruncateVecPackWithSignBits (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDNodeFlags Flags=SDNodeFlags())
	This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
static SDValue	LowerTruncateVecPack (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
static SDValue	LowerTruncateVecI1 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	expandFP_TO_UINT_SSE (MVT VT, SDValue Src, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerFP16_TO_FP (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFP_TO_FP16 (SDValue Op, SelectionDAG &DAG)
static SDValue	lowerAddSubToHorizontalOp (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
static SDValue	LowerFROUND (SDValue Op, SelectionDAG &DAG)
	ISD::FROUND is defined to round to nearest with ties rounding away from 0.
static SDValue	LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
	The only differences between FABS and FNEG are the mask and the logic op.
static SDValue	LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
static SDValue	getBT (SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG)
	Helper for attempting to create a X86ISD::BT node.
static SDValue	getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
	Helper for creating a X86ISD::SETCC node.
static bool	isOrXorXorTree (SDValue X, bool Root=true)
	Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.
template<typename F>
static SDValue	emitOrXorXorTree (SDValue X, const SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV)
	Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.
static SDValue	combineVectorSizedSetCCEquality (EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.
static bool	matchScalarReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr)
	Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns.
static SDValue	LowerVectorAllEqual (const SDLoc &DL, SDValue LHS, SDValue RHS, ISD::CondCode CC, const APInt &OriginalMask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
static SDValue	MatchVectorAllEqualTest (SDValue OrigLHS, SDValue OrigRHS, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
static bool	hasNonFlagsUse (SDValue Op)
	return true if Op has a use that doesn't just read flags.
static bool	isProfitableToUseFlagOp (SDValue Op)
static SDValue	EmitTest (SDValue Op, X86::CondCode X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
static SDValue	EmitCmp (SDValue Op0, SDValue Op1, X86::CondCode X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.
static SDValue	LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, X86::CondCode &X86CC)
	Result of 'and' is compared against zero.
static bool	cheapX86FSETCC_SSE (ISD::CondCode SetCCOpcode)
static unsigned	translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling)
	Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
static SDValue	splitVSETCC (EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SelectionDAG &DAG, const SDLoc &dl)
	Break a VSETCC 256/512-bit vector into two new 128/256 ones and then concatenate the result back.
static SDValue	LowerIntVSETCC_AVX512 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc, bool NSW)
	Given a buildvector constant, return a new vector constant with each element incremented or decremented.
static SDValue	LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	As another special case, use PSUBUS[BW] when it's profitable.
static SDValue	LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	EmitAVX512Test (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
static std::pair< SDValue, SDValue >	getX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)
static bool	isX86LogicalCmp (SDValue Op)
	Return true if opcode is a X86 logical comparison.
static bool	isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
static SDValue	LowerSELECTWithCmpZero (SDValue CmpVal, SDValue LHS, SDValue RHS, unsigned X86CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	canEmitConjunctionForCCMP (SelectionDAG &DAG, SDValue Val, bool &CanNegate, bool &MustBeFirst, bool &PreferFirst, bool WillNegate, unsigned Depth=0)
	Returns true if Val is a tree of integer AND/OR/SETCC operations that can be expressed as a CCMP conjunction.
static SDValue	emitConjunctionForCCMPRec (SDValue Val, X86::CondCode &OutCC, bool Negate, SDValue CCOp, X86::CondCode Predicate, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Emit a conjunction or disjunction tree as a CMP followed by a chain of CCMP operations.
static SDValue	emitConjunctionForCCMP (SDValue Val, X86::CondCode &OutCC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerSIGN_EXTEND_Mask (SDValue Op, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG)
	Change a vector store into a pair of half-size vector stores.
static SDValue	scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
	Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
static SDValue	LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
	Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
static SDValue	LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG, unsigned UpperEltOpSrc=0)
	Creates an SDNode for a predicated scalar operation.
static int	getSEHRegistrationNodeSize (const Function *Fn)
static SDValue	recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
	When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
static SDValue	getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
	Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.
static void	getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
	Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
static SDValue	LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
static SDValue	MarkEHGuard (SDValue Op, SelectionDAG &DAG)
static SDValue	EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
	Emit Truncating Store with signed or unsigned saturation.
static SDValue	EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
	Emit Masked Truncating Store with signed or unsigned saturation.
static SDValue	LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
static SDValue	createSetFPEnvNodes (SDValue Ptr, SDValue Chain, const SDLoc &DL, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG, const X86Subtarget &Subtarget)
uint64_t	getGFNICtrlImm (unsigned Opcode, unsigned Amt=0)
SDValue	getGFNICtrlMask (unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, unsigned Amt=0)
static APInt	getGFNIByteAffine (const APInt &ByteToAffine, const APInt &Matrix64, const APInt &Addend8)
static SDValue	LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector CTLZ using native supported vector CTLZ instruction.
static SDValue	LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	peekThroughDemandedElts (SDValue V, const APInt &DemandedElts, SelectionDAG &DAG)
static SDValue	LowerVECREDUCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool AllowScalarization)
static SDValue	lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerAVG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMINMAX (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMINMAX_REDUCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerFMINIMUM_FMAXIMUM (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerABD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowervXi8MulWithUNPCK (SDValue A, SDValue B, const SDLoc &dl, MVT VT, bool IsSigned, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue *Low=nullptr)
static SDValue	LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMULO (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	supportedVectorShiftWithImm (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static bool	supportedVectorShiftWithBaseAmnt (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static bool	supportedVectorVarShift (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static SDValue	LowerShiftByScalarImmediate (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerShiftByScalarVariable (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static std::pair< const Value *, BitTestKind >	FindSingleBitChange (const Value *V)
static X86::CondCode	matchSignedNewValueCC (const Instruction *I)
static X86::CondCode	matchAddCC (const AtomicRMWInst *AI, const Instruction *I)
static X86::CondCode	matchSubCC (const AtomicRMWInst *AI, const Instruction *I)
static X86::CondCode	matchOrCC (const AtomicRMWInst *AI, const Instruction *I)
static X86::CondCode	matchAndCC (const AtomicRMWInst *AI, const Instruction *I)
static X86::CondCode	matchXorCC (const AtomicRMWInst *AI, const Instruction *I)
static X86::CondCode	getCmpArithCC (const AtomicRMWInst *AI)
static bool	shouldExpandCmpArithRMWInIR (const AtomicRMWInst *AI)
static SDValue	emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, const SDLoc &DL)
	Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.
static SDValue	LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	getMOVMSK (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Compute the horizontal sum of bytes in V for the elements of VT.
static SDValue	LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerVectorCTPOP (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTPOP (SDValue N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCLMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerPARITY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower atomic_load_ops into LOCK-prefixed operations.
static SDValue	LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG)
static SDValue	ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
	Widen a vector input to a vector of NVT.
static SDValue	LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerCVTPS2PH (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerPREFETCH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static StringRef	getInstrStrFromOpNo (const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo)
static SDValue	getFlagsOfCmpZeroFori1 (SelectionDAG &DAG, const SDLoc &DL, SDValue Mask)
static MachineBasicBlock *	emitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
	Utility function to emit xbegin specifying the start of an RTM region.
static bool	checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static bool	isCMOVPseudo (MachineInstr &MI)
static MachineInstrBuilder	createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
static unsigned	getSUBriOpcode (bool IsLP64)
static unsigned	getOpcodeForIndirectThunk (unsigned RPOpc)
static const char *	getIndirectThunkSymbol (const X86Subtarget &Subtarget, Register Reg)
static void	computeKnownBitsForPSADBW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static void	computeKnownBitsForPMADDWD (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static void	computeKnownBitsForPMADDUBSW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static KnownBits	computeKnownBitsForHorizontalOperation (const SDValue Op, const APInt &DemandedElts, unsigned Depth, const SelectionDAG &DAG, const function_ref< KnownBits(const KnownBits &, const KnownBits &)> KnownBitsFunc)
static SDValue	narrowLoadToVZLoad (LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG)
static bool	matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue V1, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
static bool	matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
static bool	matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
static bool	matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
static SDValue	combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, unsigned RootOpcode, MVT RootVT, ArrayRef< int > BaseMask, int Depth, ArrayRef< const SDNode * > SrcNodes, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineX86ShuffleChain (ArrayRef< SDValue > Inputs, unsigned RootOpc, MVT RootVT, ArrayRef< int > BaseMask, int Depth, ArrayRef< const SDNode * > SrcNodes, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
	Combine an arbitrary chain of shuffles into a single instruction if possible.
static SDValue	combineX86ShufflesConstants (MVT VT, ArrayRef< SDValue > Ops, ArrayRef< int > Mask, ArrayRef< const SDNode * > SrcNodes, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, unsigned RootOpc, MVT RootVT, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
	Fully generic combining of x86 shuffle instructions.
static SDValue	combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Helper entry wrapper to combineX86ShufflesRecursively.
static SmallVector< int, 4 >	getPSHUFShuffleMask (SDValue N)
	Get the PSHUF-style mask from PSHUF node.
static APInt	getBLENDIBlendMask (SDValue V)
	Get the expanded blend mask from a BLENDI node.
static SDValue	combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
	Search for a combinable shuffle across a chain ending in pshufd.
static SDValue	combineCommutableSHUFP (SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	combineBlendOfPermutes (MVT VT, SDValue N0, SDValue N1, ArrayRef< int > BlendMask, const APInt &DemandedElts, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
static bool	isUnaryOp (unsigned Opcode)
static SDValue	canonicalizeShuffleWithOp (SDValue N, SelectionDAG &DAG, const SDLoc &DL)
static SDValue	canonicalizeLaneShuffleWithRepeatedOps (SDValue V, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
	Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
static SDValue	combineTargetShuffle (SDValue N, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Try to combine x86 target specific shuffles.
static bool	isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even)
	Checks if the shuffle mask takes subsequent elements alternately from two vectors.
static bool	isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd, bool &HasAllowContract)
	Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.
static SDValue	combineShuffleToFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
static SDValue	combineShuffleToAddSubOrFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
static SDValue	narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
	If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.
static SDValue	combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	checkBitcastSrcVectorSize (SDValue Src, unsigned Size, bool AllowTruncate, unsigned Depth)
static unsigned	getAltBitOpcode (unsigned Opcode)
static SDValue	adjustBitcastSrcVectorSSE1 (SelectionDAG &DAG, SDValue Src, const SDLoc &DL)
static SDValue	signExtendBitcastSrcVector (SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
static SDValue	combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG)
static SDValue	combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBitcastToBoolVector (EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth=0)
static SDValue	combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	detectExtMul (SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, SDValue &Op1)
static SDValue	createVPDPBUSD (SelectionDAG &DAG, SDValue LHS, SDValue RHS, unsigned &LogBias, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	createPSADBW (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineVECREDUCE_LOGIC (SDNode *Reduce, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVPDPBUSDPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineExtractFromVectorLoad (SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget, TargetLowering::DAGCombinerInfo &DCI)
	Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op.
static SDValue	combineArithReduction (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.
static SDValue	combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
static SDValue	combineVECREDUCE_MUL (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineToExtendBoolVectorInReg (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
	If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
static SDValue	combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
static SDValue	combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.
static SDValue	combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	commuteSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on SELECT and VSELECT nodes.
static SDValue	combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
static SDValue	checkSignTestSetCCCombine (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG)
static SDValue	checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
static bool	checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
	Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
static SDValue	combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG)
static SDValue	combinePTESTCC (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.
static SDValue	combineSetCCMOVMSK (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.
static SDValue	combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
static bool	canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
static SDValue	reduceVMULWidth (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
static SDValue	combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
static SDValue	combineMulToPMADDWD (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMulToPMULDQ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMulToPMADD52 (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineIntDivRem (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineShiftToPMULH (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineShiftLeft (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineHorizOpWithShuffle (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorHADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
static SDValue	combineAndNotIntoANDNP (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
static SDValue	combineAndShuffleNot (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.
static SDValue	PromoteMaskArithmetic (SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth)
static SDValue	PromoteMaskArithmetic (SDValue N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static unsigned	convertIntLogicToFPLogicOpcode (unsigned Opcode)
static SDValue	convertIntLogicToFPLogic (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.
static SDValue	combineBitOpWithMOVMSK (unsigned Opc, const SDLoc &DL, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineBitOpWithShift (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineBitOpWithPACK (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineAndMaskToShift (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
static SDValue	getIndexFromUnindexedLoad (LoadSDNode *Ld)
static bool	hasBZHI (const X86Subtarget &Subtarget, MVT VT)
static SDValue	combineAndNotOrIntoAndNotAnd (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z)) This undoes the inverse fold performed in InstCombine.
static SDValue	combineMaskBitOp (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineScalarAndWithMaskSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	getBMIMatchingOp (unsigned Opc, SelectionDAG &DAG, SDValue OpMustEq, SDValue Op, unsigned Depth)
static SDValue	combineBMILogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineAndXorSubWithBMI (SDNode *And, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Fold AND(Y, XOR(X, NEG(X))) -> ANDN(Y, BLSMSK(X)) if BMI is available.
static SDValue	combineX86SubCmpForFlags (SDNode *N, SDValue Flag, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineAndOrForCcmpCtest (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	canonicalizeBitSelect (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
static SDValue	combineLogicBlendIntoPBLENDV (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerX86CmpEqZeroToCtlzSrl (SDValue Op, SelectionDAG &DAG)
static SDValue	combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineAddOrSubToADCOrSBB (bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG, bool ZeroSecondOpOnly=false)
	If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
static SDValue	combineAddOrSubToADCOrSBB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
static SDValue	combineOrWithGF2P8AFFINEQB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
	GF2P8AFFINEQB computes each output bit from one row of the 8x8 affine matrix, then xors the corresponding immediate bit.
static SDValue	combineOrXorWithSETCC (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	foldXorTruncShiftIntoCmp (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
static SDValue	foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
static SDValue	detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
	Detect patterns of truncation with unsigned saturation:
static SDValue	detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false)
	Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).
static SDValue	combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineConstantPoolLoads (SDNode *N, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineAtomicLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	canonicalizeBoolMask (unsigned Opcode, EVT VT, SDValue Mask, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static int	getOneTrueElt (SDValue V)
	If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
static bool	getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, Align &Alignment, unsigned &Offset)
	Given a masked memory load/store operation, return true if it has one mask bit set.
static SDValue	reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
static SDValue	combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If exactly one element of the mask is set for a masked store, it is a vector extract, truncate (iff truncating store) and scalar store.
static SDValue	combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	narrowBitOpRMW (StoreSDNode *St, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineScalarMinMaxAbsStore (StoreSDNode *St, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Fold store(abs/min/max(load…)) of scalar i16/i32/i64 into SIMD PABS/PMIN/PMAX to keep memory-bound sort2/abs in the XMM domain.
static SDValue	combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVEXTRACT_STORE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	isHorizontalBinOp (unsigned HOpcode, SDValue &LHS, SDValue &RHS, const SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative, SmallVectorImpl< int > &PostShuffleMask, bool ForceHorizOp)
	Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
static SDValue	combineToHorizontalAddSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFMulcFCMulc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFaddCFmul (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on floating-point adds/subs.
static SDValue	combineLRINT_LLRINT (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combinei64TruncSrlConstant (SDValue N, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
static SDValue	combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
	Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
static SDValue	combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
static SDValue	combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVTRUNC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineVTRUNCSAT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	isFNEG (SelectionDAG &DAG, SDNode *N, unsigned Depth=0)
	Returns the negated value if the node N flips sign of FP value.
static unsigned	negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc, bool NegRes)
static SDValue	combineFneg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on floating point negations.
static SDValue	combineX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	foldXor1SetCC (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
static SDValue	combineXorWithGF2P8AFFINEQB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
static SDValue	combineXorWithTwoGF2P8AFFINEQB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
static SDValue	combineXorSubCTLZ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineBITREVERSE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineAVG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	isNullFPScalarOrVectorConst (SDValue V)
static SDValue	getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
static SDValue	combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FAND nodes.
static SDValue	combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FANDN nodes.
static SDValue	combineFOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
static SDValue	combineFMinFMax (SDNode *N, SelectionDAG &DAG)
	Do target-specific dag combines on PSEUDO_FMIN/PSEUDO_FMAX nodes.
static SDValue	combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::ANDNP nodes.
static SDValue	peekThroughBitPosExtTrunc (SDValue V, unsigned BW)
static SDValue	combineBTToBitOpFlag (SDNode *N, SelectionDAG &DAG)
static SDValue	combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineCVTPH2PS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineSextInRegCmov (SDNode *N, SelectionDAG &DAG)
static SDValue	combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
static SDValue	combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG)
static SDValue	combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	getInvertedVectorForFMA (SDValue V, SelectionDAG &DAG)
static SDValue	combineFMA (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineFMADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	widenBuildVec (SDNode *Extend, SelectionDAG &DAG)
static SDValue	combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	truncateAVX512SetCCNoBWI (EVT VT, EVT OpVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.
static SDValue	combineAVX512SetCCToKMOV (EVT VT, SDValue Op0, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineTESTP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineX86GatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	rebuildGatherScatter (MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Mask, SDValue Scale, SelectionDAG &DAG)
static SDValue	combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Optimize branch condition evaluation.
static SDValue	combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
static SDValue	combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG)
	If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.
static SDValue	combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineFPToSInt (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFP_TO_xINT_SAT (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineRotate (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineFunnelShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	needCarryOrOverflowFlag (SDValue Flags)
static bool	onlyZeroFlagUsed (SDValue Flags)
static SDValue	combineCMP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static void	matchGenericOp (unsigned Opc, SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, bool Negate=false)
	If a matching generic opcode node exists, replace its uses with the value of the target-specific node N.
static SDValue	combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineX86XOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineSBB (SDNode *N, SelectionDAG &DAG)
static SDValue	combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	matchPMADDWD (SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
static SDValue	matchPMADDWD_2 (SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
static SDValue	combineAddOfPMADDWD (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT)
static SDValue	pushAddIntoCmovOfConsts (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	CMOV of constants requires materializing constant operands in registers.
static SDValue	matchVPMADD52 (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
static SDValue	combineAdd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineSubABS (EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineSubSetcc (SDNode *N, SelectionDAG &DAG)
static SDValue	combineX86CloadCstore (SDNode *N, SelectionDAG &DAG)
static SDValue	combineSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static std::optional< unsigned >	CastIntSETCCtoFP (MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, unsigned NumSignificantBitsRHS)
static SDValue	combineCONCAT_VECTORS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineINSERT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	narrowExtractedVectorSelect (SDNode *Ext, const SDLoc &DL, SelectionDAG &DAG)
	If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.
static SDValue	combineEXTRACT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineSCALAR_TO_VECTOR (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVPMADD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineVPMADD52LH (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineEXTEND_VECTOR_INREG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineKSHIFT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineAndOnGF2P8AFFINEQBOperand (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
static SDValue	combineNestedGF2P8AFFINEQB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
static SDValue	combineXorOnGF2P8AFFINEQBOperand (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, EVT VT)
static SDValue	combineGF2P8AFFINEQB (SDNode *N, SelectionDAG &DAG)
static SDValue	combineFP16_TO_FP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFP_EXTEND (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineBROADCAST_LOAD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineFP_ROUND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMOVDQ2Q (SDNode *N, SelectionDAG &DAG)
static SDValue	combineBMI (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combinePCLMULQDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	FixupMMXIntrinsicTypes (SDNode *N, SelectionDAG &DAG)
static SDValue	combineINTRINSIC_WO_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineINTRINSIC_W_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineINTRINSIC_VOID (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static X86::CondCode	parseConstraintCode (llvm::StringRef Constraint)
static bool	isGRClass (const TargetRegisterClass &RC)
	Check if RC is a general purpose register class.
static bool	isFRClass (const TargetRegisterClass &RC)
	Check if RC is a vector register class.
static bool	isVKClass (const TargetRegisterClass &RC)
	Check if RC is a mask register class.
static bool	useEGPRInlineAsm (const X86Subtarget &Subtarget)

Variables
static cl::opt< int >	ExperimentalPrefInnermostLoopAlignment ("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden)
static cl::opt< int >	BrMergingBaseCostThresh ("x86-br-merging-base-cost", cl::init(2), cl::desc("Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden)
static cl::opt< int >	BrMergingCcmpBias ("x86-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions."), cl::Hidden)
static cl::opt< bool >	WidenShift ("x86-widen-shift", cl::init(true), cl::desc("Replace narrow shifts with wider shifts."), cl::Hidden)
static cl::opt< int >	BrMergingLikelyBias ("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden)
static cl::opt< int >	BrMergingUnlikelyBias ("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden)
static cl::opt< bool >	MulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
const unsigned	X87StateSize = 28
const unsigned	FPStateSize = 32
const unsigned	FPStateSizeInBits = FPStateSize * 8

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 70 of file X86ISelLowering.cpp.

GET_EGPR_IF_ENABLED

#define GET_EGPR_IF_ENABLED

(

OPC

)

Value:

(Subtarget.hasEGPR() ? OPC##_EVEX : OPC)

GET_REGISTER_MATCHER

#define GET_REGISTER_MATCHER

Definition at line 28842 of file X86ISelLowering.cpp.

Enumeration Type Documentation

BitTestKind

enum BitTestKind : unsigned

Enumerator
UndefBit
ConstantBit
NotConstantBit
ShiftBit
NotShiftBit

Definition at line 32597 of file X86ISelLowering.cpp.

ShrinkMode

enum class ShrinkMode

strong

Different mul shrinking modes.

Enumerator
MULS8
MULU8
MULS16
MULU16

Definition at line 50283 of file X86ISelLowering.cpp.

Function Documentation

adjustBitcastSrcVectorSSE1()

SDValue adjustBitcastSrcVectorSSE1 ( SelectionDAG & DAG, SDValue Src, const SDLoc & DL )

static

Definition at line 46404 of file X86ISelLowering.cpp.

References adjustBitcastSrcVectorSSE1(), llvm::ISD::AND, llvm::ISD::BITCAST, llvm::cast(), DL, getAltBitOpcode(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isNormalLoad(), llvm::ISD::OR, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETLT, and llvm::ISD::XOR.

Referenced by adjustBitcastSrcVectorSSE1(), and combineBitcastvxi1().

buildFromShuffleMostly()

SDValue buildFromShuffleMostly ( SDValue Op, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 8615 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetLoweringInfo(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_VECTOR_ELT, llvm::isa(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), Opc, llvm::ISD::POISON, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::ISD::UNDEF.

canCombineAsMaskOperation()

bool canCombineAsMaskOperation ( SDValue V, const X86Subtarget & Subtarget )

static

Definition at line 18538 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, llvm::MVT::getScalarType(), llvm::X86Subtarget::hasAVX512(), llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::PSEUDO_FMAX, llvm::ISD::PSEUDO_FMIN, llvm::ISD::SHL, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::ISD::XOR.

Referenced by commuteSelect(), and lowerVECTOR_SHUFFLE().

canEmitConjunctionForCCMP()

bool canEmitConjunctionForCCMP ( SelectionDAG & DAG, SDValue Val, bool & CanNegate, bool & MustBeFirst, bool & PreferFirst, bool WillNegate, unsigned Depth = 0 )

static

Returns true if Val is a tree of integer AND/OR/SETCC operations that can be expressed as a CCMP conjunction.

This mirrors AArch64's canEmitConjunction and tracks the negation bookkeeping required to lower arbitrarily nested AND/OR trees (e.g. AND(cc, OR(cc, cc))) correctly.

Parameters

CanNegate	Set to true if the whole sub-tree can be negated just by inverting the conditions on the SETCC leaves.
MustBeFirst	Set to true if this sub-tree needs to be negated but cannot be negated naturally, so it must be emitted first.
PreferFirst	Set to true if processing this sub-tree first may result in more efficient code (e.g. because the flags of a corresponding SUB node can be reused).
WillNegate	True when the result of this sub-expression must be negated (i.e. the outer expression is an OR).

Definition at line 25549 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), canEmitConjunctionForCCMP(), llvm::Depth, llvm::SelectionDAG::doesNodeExist(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::EVT::isInteger(), llvm::O0, llvm::O1, llvm::ISD::OR, llvm::ISD::SETCC, and llvm::ISD::SUB.

Referenced by canEmitConjunctionForCCMP(), emitConjunctionForCCMP(), and emitConjunctionForCCMPRec().

canLowerByDroppingElements()

int canLowerByDroppingElements ( ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput )

static

Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

(even) N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

(odd) N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns

N above, or the number of times even/odd elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 11091 of file X86ISelLowering.cpp.

References assert(), llvm::isPowerOf2_32(), N, llvm::Offset, and uint64_t.

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

canonicalizeBitSelect()

SDValue canonicalizeBitSelect ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 53240 of file X86ISelLowering.cpp.

References A(), llvm::ISD::AND, assert(), B(), C(), DL, getAVX512Node(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), Imm, llvm::MVT::isVector(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), useVPTERNLOG(), X, and Y.

Referenced by combineOr().

canonicalizeBoolMask()

SDValue canonicalizeBoolMask ( unsigned Opcode, EVT VT, SDValue Mask, const SDLoc & DL, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 54479 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::EVT::bitsGT(), llvm::EVT::changeVectorElementTypeToInteger(), combineToExtendBoolVectorInReg(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAVX512(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::TRUNCATE.

Referenced by combineGatherScatter(), combineMaskedLoad(), and combineMaskedStore().

canonicalizeLaneShuffleWithRepeatedOps()

SDValue canonicalizeLaneShuffleWithRepeatedOps ( SDValue V, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).

Definition at line 43095 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::isUndef(), LHS, llvm::X86::mayFoldLoad(), llvm::peekThroughBitcasts(), RHS, SDValue(), and uint64_t.

Referenced by combineTargetShuffle().

canonicalizeShuffleMaskWithCommute()

bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int > Mask )

static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 18476 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

canonicalizeShuffleMaskWithHorizOp()

SDValue canonicalizeShuffleMaskWithHorizOp ( MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 41752 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), llvm::any_of(), assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::Hi, isInRange(), llvm::SDNode::isOnlyUserOf(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrZero(), isUndefOrZeroOrInRange(), LHS, llvm::Lo, llvm::M0(), llvm::M1(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, scaleShuffleElements(), SDValue(), shouldUseHorizontalOp(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::Undef, llvm::V1, and widenSubVector().

Referenced by combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

canonicalizeShuffleWithOp()

SDValue canonicalizeShuffleWithOp ( SDValue N, SelectionDAG & DAG, const SDLoc & DL )

static

Definition at line 42903 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), isLogicOp(), llvm::SelectionDAG::isSplatValue(), isTargetShuffle(), isUnaryOp(), LHS, N, Opc, llvm::peekThroughOneUseBitcasts(), RHS, SDValue(), and llvm::ISD::SINT_TO_FP.

Referenced by combineShuffle().

canReduceVMulWidth()

bool canReduceVMulWidth ( SDNode * N, SelectionDAG & DAG, ShrinkMode & Mode )

static

Definition at line 50285 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::EVT::getScalarSizeInBits(), Mode, MULS16, MULS8, MULU16, MULU8, N, and llvm::SelectionDAG::SignBitIsZero().

Referenced by matchPMADDWD(), and reduceVMULWidth().

canScaleShuffleElements()

bool canScaleShuffleElements ( ArrayRef< int > Mask, unsigned NumDstElts )

static

Definition at line 4224 of file X86ISelLowering.cpp.

References scaleShuffleElements().

Referenced by combineBlendOfPermutes(), and combineSetCCMOVMSK().

canWidenShuffleElements() [1/3]

bool canWidenShuffleElements ( ArrayRef< int > Mask )

static

Definition at line 4189 of file X86ISelLowering.cpp.

References canWidenShuffleElements().

canWidenShuffleElements() [2/3]

bool canWidenShuffleElements ( ArrayRef< int > Mask, const APInt & Zeroable, bool V2IsZero, SmallVectorImpl< int > & WidenedMask )

static

Definition at line 4173 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), llvm::APInt::isZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

canWidenShuffleElements() [3/3]

bool canWidenShuffleElements ( ArrayRef< int > Mask, SmallVectorImpl< int > & WidenedMask )

static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 4123 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::M0(), llvm::M1(), Size, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canWidenShuffleElements(), canWidenShuffleElements(), combineBlendOfPermutes(), combineExtractWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), lowerVECTOR_SHUFFLE(), and scaleShuffleElements().

CastIntSETCCtoFP()

std::optional< unsigned > CastIntSETCCtoFP ( MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, unsigned NumSignificantBitsRHS )

static

Definition at line 60553 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getFltSemantics(), llvm::MVT::getScalarType(), llvm::APFloatBase::semanticsPrecision(), llvm::ISD::SETEQ, llvm::ISD::SETGT, and llvm::ISD::SINT_TO_FP.

Referenced by combineConcatVectorOps().

cheapX86FSETCC_SSE()

bool cheapX86FSETCC_SSE ( ISD::CondCode SetCCOpcode )

static

Definition at line 24365 of file X86ISelLowering.cpp.

References llvm::ISD::SETONE, and llvm::ISD::SETUEQ.

Referenced by convertIntLogicToFPLogic(), and LowerVSETCC().

checkAndUpdateEFLAGSKill()

bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator SelectItr, MachineBasicBlock * BB, const TargetRegisterInfo * TRI )

static

Definition at line 37034 of file X86ISelLowering.cpp.

References llvm::isPhysRegUsedAfter(), and TRI.

checkBitcastSrcVectorSize()

bool checkBitcastSrcVectorSize ( SDValue Src, unsigned Size, bool AllowTruncate, unsigned Depth )

static

Definition at line 46354 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, checkBitcastSrcVectorSize(), llvm::Depth, llvm::ISD::FREEZE, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, Size, llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, and llvm::ISD::XOR.

checkBoolTestAndOrSetCCCombine()

bool checkBoolTestAndOrSetCCCombine ( SDValue Cond, X86::CondCode & CC0, X86::CondCode & CC1, SDValue & Flags, bool & isAnd )

static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 49468 of file X86ISelLowering.cpp.

References llvm::ISD::AND, Cond, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), and llvm::ISD::OR.

Referenced by combineCMov().

checkBoolTestSetCCCombine()

SDValue checkBoolTestSetCCCombine ( SDValue Cmp, X86::CondCode & CC )

static

Definition at line 49350 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), C(), llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::dyn_cast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::ConstantSDNode::getZExtValue(), llvm::isOneConstant(), SDValue(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSetCCEFLAGS().

checkSignTestSetCCCombine()

SDValue checkSignTestSetCCCombine ( SDValue Cmp, X86::CondCode & CC, SelectionDAG & DAG )

static

Definition at line 49273 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::cast(), llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getValidShiftAmount(), llvm::isNullConstant(), llvm::APInt::lshrInPlace(), SDValue(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::SelectionDAG::SignBitIsZero(), and llvm::ISD::SRA.

Referenced by combineSetCCEFLAGS().

collectConcatOps()

bool collectConcatOps ( SDNode * N, SmallVectorImpl< SDValue > & Ops, SelectionDAG & DAG )

static

Definition at line 4524 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), collectConcatOps(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::Lo, N, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::Sub, and uint64_t.

Referenced by collectConcatOps(), combineBitcastvxi1(), combineINSERT_SUBVECTOR(), combineSetCCMOVMSK(), combineTargetShuffle(), isFNEG(), isFreeToSplitVector(), IsNOT(), isUpperSubvectorUndef(), and splitVector().

combineADC()

SDValue combineADC ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 59725 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, combineCarryThroughADD(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_B, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), LHS, N, needCarryOrOverflowFlag(), RHS, llvm::APInt::sadd_ov(), SDValue(), and llvm::APInt::uadd_ov().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAdd()

SDValue combineAdd ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 60195 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), C(), combineAddOfPMADDWD(), combineAddOrSubToADCOrSBB(), combineToHorizontalAddSub(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::SDNode::getVTList(), getZeroVector(), llvm::SDNode::hasAnyUseOfValue(), llvm::X86Subtarget::hasAVX(), llvm::SDNode::hasOneUse(), llvm::isa(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86::isZeroNode(), llvm::PatternMatch::m_Add(), llvm::SDPatternMatch::m_AllOf(), llvm::PatternMatch::m_c_BinOp(), llvm::SDPatternMatch::m_ConstInt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::SDPatternMatch::m_Srl(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::m_ZExt(), matchPMADDWD(), matchPMADDWD_2(), matchVPMADD52(), N, pushAddIntoCmovOfConsts(), llvm::SDPatternMatch::sd_match(), Select, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, supportedVectorShiftWithImm(), std::swap(), llvm::APInt::ult(), llvm::SelectionDAG::willNotOverflowAdd(), X, and Y.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAddOfPMADDWD()

SDValue combineAddOfPMADDWD ( SelectionDAG & DAG, SDValue N0, SDValue N1, const SDLoc & DL, EVT VT )

static

Definition at line 60048 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), LHS, llvm::SelectionDAG::MaskedVectorIsZero(), RHS, and SDValue().

Referenced by combineAdd().

combineAddOrSubToADCOrSBB() [1/2]

SDValue combineAddOrSubToADCOrSBB ( bool IsSub, const SDLoc & DL, EVT VT, SDValue X, SDValue Y, SelectionDAG & DAG, bool ZeroSecondOpOnly = false )

static

If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.

This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}. Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.

Definition at line 53475 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::dyn_cast(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86::isZeroNode(), LowerAndToBT(), SDValue(), llvm::ISD::SETNE, X, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by combineAdd(), combineAddOrSubToADCOrSBB(), combineOr(), combineOrXorWithSETCC(), combineSub(), and combineX86AddSub().

combineAddOrSubToADCOrSBB() [2/2]

SDValue combineAddOrSubToADCOrSBB ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.

This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}.

Definition at line 53684 of file X86ISelLowering.cpp.

References combineAddOrSubToADCOrSBB(), DL, llvm::SelectionDAG::getNegative(), N, llvm::ISD::OR, SDValue(), llvm::ISD::SUB, X, and Y.

combineAnd()

SDValue combineAnd ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 52943 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::all_of(), llvm::ISD::AND, assert(), llvm::EVT::changeVectorElementType(), llvm::APInt::clearAllBits(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndNotOrIntoAndNotAnd(), combineAndOrForCcmpCtest(), combineAndShuffleNot(), combineAndXorSubWithBMI(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineCompareEqual(), combineMaskBitOp(), combineScalarAndWithMaskSetcc(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, llvm::APInt::exactLogBase2(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), getBitWidth(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, llvm::EVT::is512BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::isConstOrConstSplat(), llvm::isOneConstant(), llvm::SDNode::isOnlyUserOf(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isZero(), LHS, llvm::SDPatternMatch::m_AllOf(), llvm::PatternMatch::m_And(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_SExt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Value(), llvm::SelectionDAG::MaskedValueIsZero(), matchScalarReduction(), llvm::X86::MaxShuffleCombineDepth, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::APInt::setBit(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), X, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAndLoadToBZHI()

SDValue combineAndLoadToBZHI ( SDNode * Node, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 52558 of file X86ISelLowering.cpp.

References llvm::AllOnes, llvm::ISD::AND, llvm::cast(), llvm::dyn_cast(), GEP, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), getIndexFromUnindexedLoad(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), hasBZHI(), llvm::isa(), SDValue(), llvm::ISD::SRL, llvm::ISD::SUB, llvm::Sub, llvm::ISD::TRUNCATE, and uint64_t.

Referenced by combineAnd().

combineAndMaskToShift()

SDValue combineAndMaskToShift ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

If this is a zero/all-bits result that is bitwise-anded with a low bits mask.

(Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.

Definition at line 52408 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), getTargetVShiftByConstNode(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesOrAllOnesSplat(), llvm::isBitwiseNot(), llvm::X86::isConstantSplat(), llvm::EVT::isInteger(), llvm::APInt::isMask(), llvm::EVT::isSimple(), N, llvm::peekThroughBitcasts(), SDValue(), llvm::ISD::SRA, llvm::ISD::SRL, supportedVectorShiftWithImm(), X, and Y.

Referenced by combineAnd().

combineAndNotIntoANDNP()

SDValue combineAndNotIntoANDNP ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).

Definition at line 52015 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), IsNOT(), N, SDValue(), X, and Y.

Referenced by combineAnd().

combineAndNotOrIntoAndNotAnd()

SDValue combineAndNotOrIntoAndNotAnd ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z)) This undoes the inverse fold performed in InstCombine.

Definition at line 52491 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::hasAndNot(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::PatternMatch::m_And(), llvm::MIPatternMatch::m_Not(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Or(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), X, and Y.

Referenced by combineAnd().

combineAndnp()

SDValue combineAndnp ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on X86ISD::ANDNP nodes.

Definition at line 57276 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, llvm::ISD::BITCAST, llvm::APInt::clearAllBits(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), I, llvm::MVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), isZero(), N, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::APInt::setBit(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAndOnGF2P8AFFINEQBOperand()

SDValue combineAndOnGF2P8AFFINEQBOperand ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 62873 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), Imm, llvm::X86::isConstantSplat(), llvm::SelectionDAG::isSplatValue(), llvm::PatternMatch::m_And(), llvm::SDPatternMatch::m_ConstInt(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SDPatternMatch::sd_match(), SDValue(), X, and Y.

Referenced by combineGF2P8AFFINEQB().

combineAndOrForCcmpCtest()

SDValue combineAndOrForCcmpCtest ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & ST )

static

Definition at line 52863 of file X86ISelLowering.cpp.

References llvm::X86::COND_NP, llvm::X86::COND_P, DL, llvm::SDNode::getAsAPIntVal(), llvm::X86::getCCMPCondFlagsFromCondCode(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::SelectionDAG::getTargetConstant(), llvm::isNullConstant(), N, Opc, llvm::ISD::OR, SDValue(), llvm::Sub, and std::swap().

Referenced by combineAnd(), and combineOr().

combineAndShuffleNot()

SDValue combineAndShuffleNot ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.

Definition at line 52047 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isa(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), N, llvm::peekThroughOneUseBitcasts(), SDValue(), splitVector(), llvm::X86Subtarget::useAVX512Regs(), X, and Y.

Referenced by combineAnd().

combineAndXorSubWithBMI()

SDValue combineAndXorSubWithBMI ( SDNode * And, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Fold AND(Y, XOR(X, NEG(X))) -> ANDN(Y, BLSMSK(X)) if BMI is available.

Definition at line 52777 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_Deferred(), llvm::MIPatternMatch::m_Neg(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Xor(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SUB, X, llvm::ISD::XOR, and Y.

Referenced by combineAnd().

combineArithReduction()

SDValue combineArithReduction ( SDNode * ExtElt, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.

Definition at line 47911 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::ISD::INSERT_VECTOR_ELT, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::Lo, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), Opc, SDValue(), shouldUseHorizontalOp(), SplitOpsAndApply(), splitVector(), llvm::ISD::TRUNCATE, llvm::APInt::ule(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineExtractVectorElt().

combineAtomicLoad()

SDValue combineAtomicLoad ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 54350 of file X86ISelLowering.cpp.

References llvm::ISD::ATOMIC_LOAD, llvm::BitWidth, llvm::cast(), DL, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getMergeValues(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAVG()

SDValue combineAVG ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 56978 of file X86ISelLowering.cpp.

References llvm::ISD::AVGCEILS, llvm::ISD::AVGCEILU, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::APInt::getSignMask(), llvm::EVT::isVector(), N, SDValue(), and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAVX512SetCCToKMOV()

SDValue combineAVX512SetCCToKMOV ( EVT VT, SDValue Op0, ISD::CondCode CC, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 58236 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::APInt::countTrailingZeros(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::extractBits(), llvm::SelectionDAG::getAnyExtOrTrunc(), getBitWidth(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::X86Subtarget::hasAVX512(), I, llvm::APInt::isAllOnes(), llvm::EVT::isVector(), llvm::Load, llvm::Masked, N, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SRL.

Referenced by combineSetCC().

combineBasicSADPattern()

SDValue combineBasicSADPattern ( SDNode * Extract, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 47474 of file X86ISelLowering.cpp.

References llvm::ISD::ABDU, llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, createPSADBW(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::SDPatternMatch::m_Abs(), llvm::SDPatternMatch::m_AllOf(), llvm::SDPatternMatch::m_AnyOf(), llvm::PatternMatch::m_c_BinOp(), llvm::PatternMatch::m_Deferred(), llvm::SDPatternMatch::m_SpecificVectorElementVT(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_UMax(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExt(), llvm::SelectionDAG::matchBinOpReduction(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by combineExtractVectorElt().

combineBitcast()

SDValue combineBitcast ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 46851 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, C(), llvm::cast(), llvm::EVT::changeVectorElementTypeToInteger(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countl_zero(), createMMXBuildVector(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, F64, llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::APFloatBase::IEEEdouble(), llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::is128BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::isNullConstantOrUndef(), llvm::isPowerOf2_32(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::EVT::isVectorOf(), N, llvm::SDNode::ops(), llvm::ISD::OR, llvm::peekThroughTruncates(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SETLT, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBitcastToBoolVector()

SDValue combineBitcastToBoolVector ( EVT VT, SDValue V, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget, unsigned Depth = 0 )

static

Definition at line 46762 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, C(), llvm::cast(), combineBitcastToBoolVector(), llvm::ISD::Constant, llvm::Depth, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, Opc, llvm::ISD::OR, SDValue(), llvm::ISD::SHL, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineBitcast(), and combineBitcastToBoolVector().

combineBitcastvxi1()

SDValue combineBitcastvxi1 ( SelectionDAG & DAG, EVT VT, SDValue Src, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 46469 of file X86ISelLowering.cpp.

References adjustBitcastSrcVectorSSE1(), llvm::all_of(), llvm::ISD::ANY_EXTEND, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), checkBitcastSrcVectorSize(), collectConcatOps(), combineBitcastvxi1(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), getMOVMSK(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isSimple(), SDValue(), llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::ISD::SIGN_EXTEND, signExtendBitcastSrcVector(), llvm::MVT::SimpleTy, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::TRUNCATE, and widenSubVector().

Referenced by combineAnd(), combineBitcast(), combineBitcastvxi1(), combineExtractVectorElt(), combineOr(), and combineVECREDUCE_LOGIC().

combineBitOpWithMOVMSK()

SDValue combineBitOpWithMOVMSK ( unsigned Opc, const SDLoc & DL, SDValue N0, SDValue N1, SelectionDAG & DAG )

static

Definition at line 52301 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), convertIntLogicToFPLogicOpcode(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isFloatingPoint(), Opc, llvm::ISD::OR, SDValue(), and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBitOpWithPACK()

SDValue combineBitOpWithPACK ( unsigned Opc, const SDLoc & DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG & DAG )

static

Definition at line 52370 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), LHS, Opc, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), RHS, SDValue(), and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBitOpWithShift()

SDValue combineBitOpWithShift ( unsigned Opc, const SDLoc & DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG & DAG )

static

Definition at line 52332 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), Opc, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), SDValue(), and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBITREVERSE()

SDValue combineBITREVERSE ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 56950 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVectorOf(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBlendOfPermutes()

SDValue combineBlendOfPermutes ( MVT VT, SDValue N0, SDValue N1, ArrayRef< int > BlendMask, const APInt & DemandedElts, SelectionDAG & DAG, const X86Subtarget & Subtarget, const SDLoc & DL )

static

Definition at line 42798 of file X86ISelLowering.cpp.

References assert(), canScaleShuffleElements(), canWidenShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::getShuffleDemandedElts(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), I, llvm::APInt::intersects(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isBlendOrUndef(), isLaneCrossingShuffleMask(), isUndefOrEqual(), isUndefOrInRange(), llvm::APInt::isZero(), llvm::peekThroughOneUseBitcasts(), scaleShuffleElements(), SDValue(), and llvm::SM_SentinelUndef.

Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

combineBMI()

SDValue combineBMI ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63316 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBMILogicOp()

SDValue combineBMILogicOp ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 52757 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), getBMIMatchingOp(), llvm::EVT::isScalarInteger(), N, SDValue(), and llvm::ISD::XOR.

Referenced by combineAnd(), and combineXor().

combineBrCond()

SDValue combineBrCond ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Optimize branch condition evaluation.

Definition at line 58947 of file X86ISelLowering.cpp.

References combineSetCCEFLAGS(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBROADCAST_LOAD()

SDValue combineBROADCAST_LOAD ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63160 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, SDValue(), and llvm::SDNode::users().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBT()

SDValue combineBT ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 57518 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, combineBTToBitOpFlag(), llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::Log2_32(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBTToBitOpFlag()

SDValue combineBTToBitOpFlag ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 57456 of file X86ISelLowering.cpp.

References assert(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::PatternMatch::m_And(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Or(), llvm::SDPatternMatch::m_Rotl(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Xor(), N, peekThroughBitPosExtTrunc(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SDPatternMatch::sd_match(), SDValue(), and llvm::Value::users().

Referenced by combineBT().

combineCarryThroughADD()

SDValue combineCarryThroughADD ( SDValue EFLAGS, SelectionDAG & DAG )

static

Definition at line 49509 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86::COND_A, llvm::X86::COND_B, llvm::X86::COND_E, DL, getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SDNode::hasOneUse(), llvm::isa(), llvm::isAllOnesConstant(), llvm::EVT::isInteger(), llvm::isOneConstant(), SDValue(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, uint64_t, and llvm::ISD::ZERO_EXTEND.

combineCastedMaskArithmetic()

SDValue combineCastedMaskArithmetic ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 46636 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, combinevXi1ConstantToInteger(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::hasAVX512(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isScalarInteger(), llvm::EVT::isVectorOf(), LHS, llvm::PatternMatch::m_BitCast(), llvm::PatternMatch::m_BitwiseLogic(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Value(), N, RHS, llvm::SDPatternMatch::sd_match(), and SDValue().

Referenced by combineBitcast().

combineCMov()

SDValue combineCMov ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].

Definition at line 50026 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::Add, assert(), llvm::X86Subtarget::canUseCMOV(), checkBoolTestAndOrSetCCCombine(), combineSetCCEFLAGS(), Cond, llvm::X86::COND_AE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_POISON, DL, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasBitScanPassThrough(), hasFPCMov(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::ISD::MUL, N, SDValue(), llvm::ISD::SHL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::ult(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCMP()

SDValue combineCMP ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 59440 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::BitWidth, combineX86SubCmpForFlags(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::MaskedValueIsZero(), N, needCarryOrOverflowFlag(), onlyZeroFlagUsed(), llvm::ISD::OR, SDValue(), llvm::ISD::SHL, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::APInt::ult(), widenMaskVector(), llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCommutableSHUFP()

SDValue combineCommutableSHUFP ( SDValue N, MVT VT, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 42743 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), Imm, llvm::X86::mayFoldLoad(), N, llvm::peekThroughOneUseBitcasts(), llvm::rotl(), and SDValue().

Referenced by combineTargetShuffle().

combineCompareEqual()

SDValue combineCompareEqual ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.

Likewise for OR -> CMPNEQSS.

Definition at line 51907 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, isAndOrOfSetCCs(), N, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineAnd(), and combineOr().

combineCONCAT_VECTORS()

SDValue combineCONCAT_VECTORS ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 61830 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), C(), combineConcatVectorOps(), llvm::dyn_cast(), E(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), I, llvm::ISD::isBitwiseLogicOp(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::peekThroughBitcasts(), SDValue(), and llvm::ISD::SETCC.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineConcatVectorOps()

SDValue combineConcatVectorOps ( const SDLoc & DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG & DAG, const X86Subtarget & Subtarget, unsigned Depth )

static

Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.

ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.

Definition at line 60575 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::all_equal(), llvm::all_of(), llvm::X86TargetLowering::allowsMemoryAccess(), llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::BITCAST, C(), llvm::cast(), CastIntSETCCtoFP(), llvm::EVT::changeVectorElementType(), combineConcatVectorOps(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_POISON, llvm::ISD::CTPOP, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_POISON, llvm::DecodeVPERMMask(), llvm::Depth, DL, llvm::SelectionDAG::doesNodeExist(), llvm::dyn_cast(), E(), EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::Fast, llvm::ISD::FCEIL, llvm::ISD::FDIV, llvm::ISD::FFLOOR, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::ISD::FNEARBYINT, llvm::ISD::FNEG, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::ISD::FSQRT, llvm::ISD::FSUB, llvm::ISD::FTRUNC, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), getBLENDIBlendMask(), getBROADCAST_LOAD(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getEntryNode(), getEXTEND_VECTOR_INREG(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), getMask(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86Subtarget::getPreferVectorWidth(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), I, Imm, llvm::APInt::insertBits(), is128BitLaneCrossingShuffleMask(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::ISD::isNormalLoad(), llvm::EVT::isSimple(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), llvm::EVT::isVector(), LHS, lowerShuffleWithPERMV(), llvm::SDPatternMatch::m_SMaxLike(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::SelectionDAG::MaxRecursionDepth, llvm::X86::mayFoldLoad(), llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::ISD::MUL, llvm::none_of(), Opc, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::ISD::PSEUDO_FMAX, llvm::ISD::PSEUDO_FMIN, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SCALAR_TO_VECTOR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::seq(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SETOEQ, llvm::ISD::SETOGT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::SINT_TO_FP, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SUB, llvm::Sub, translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::X86Subtarget::useAVX512Regs(), llvm::X86Subtarget::useBWIRegs(), llvm::ISD::VECTOR_SHUFFLE, llvm::ISD::VSELECT, widenSubVector(), X, llvm::ISD::XOR, llvm::ISD::ZERO_EXTEND, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::APInt::zext().

Referenced by combineCONCAT_VECTORS(), combineConcatVectorOps(), combineINSERT_SUBVECTOR(), combineOr(), combineTargetShuffle(), and lowerShuffleAsVTRUNC().

combineConstantPoolLoads()

SDValue combineConstantPoolLoads ( SDNode * N, const SDLoc & dl, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 54274 of file X86ISelLowering.cpp.

References llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), E(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::EVT::getFixedSizeInBits(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), llvm::Value::getType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::isNormalLoad(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::ISD::NON_EXTLOAD, SDValue(), and llvm::SDNode::users().

Referenced by combineLoad().

combineCVTP2I_CVTTP2I()

SDValue combineCVTP2I_CVTTP2I ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 57238 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCVTPH2PS()

SDValue combineCVTPH2PS ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 57537 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::DELETED_NODE, llvm::SelectionDAG::getBitcast(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineEXTEND_VECTOR_INREG()

SDValue combineEXTEND_VECTOR_INREG ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 62733 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::EVT::changeVectorElementType(), combineX86ShufflesRecursively(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtLoad(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE41(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isLoadLegal(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Load, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineEXTRACT_SUBVECTOR()

SDValue combineEXTRACT_SUBVECTOR ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 62152 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_UINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsElementEquivalent(), llvm::ISD::isExtOpcode(), llvm::ISD::isExtVecInRegOpcode(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), scaleShuffleElements(), SDValue(), llvm::seq(), llvm::ISD::SINT_TO_FP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, splitVectorIntBinary(), llvm::Sub, llvm::ISD::TRUNCATE, uint64_t, llvm::ISD::UINT_TO_FP, llvm::ISD::VSELECT, and widenSubVector().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineExtractFromVectorLoad()

SDValue combineExtractFromVectorLoad ( SDNode * N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const SDLoc & dl, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 47567 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::commonAlignment(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getLoad(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::TargetLowering::getVectorElementPointer(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MachinePointerInfo::getWithOffset(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::Load, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, SDValue(), and uint64_t.

Referenced by combineExtractVectorElt(), and combineExtractWithShuffle().

combineExtractVectorElt()

SDValue combineExtractVectorElt ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 48074 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, combineArithReduction(), combineBasicSADPattern(), combineBitcastvxi1(), combineExtractFromVectorLoad(), combineExtractWithShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineVPDPBUSDPattern(), llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getShiftAmountOperand(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBitwiseLogicOp(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, scalarizeExtEltFP(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::Sub, llvm::ISD::TRUNCATE, uint64_t, and llvm::SDNode::users().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineExtractWithShuffle()

SDValue combineExtractWithShuffle ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 47604 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), assert(), canWidenShuffleElements(), llvm::cast(), combineExtractFromVectorLoad(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalarInteger(), isUndefOrZeroInRange(), llvm::Lo, llvm::Load, N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::APInt::ult(), and llvm::APInt::urem().

Referenced by combineExtractVectorElt().

combineExtSetcc()

SDValue combineExtSetcc ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 57808 of file X86ISelLowering.cpp.

References llvm::cast(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::isInteger(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), N, SDValue(), llvm::ISD::SETCC, Size, llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

combineFaddCFmul()

SDValue combineFaddCFmul ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 55778 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::ISD::FADD, llvm::SelectionDAG::getBitcast(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDNodeFlags::hasAllowContract(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), LHS, N, RHS, and SDValue().

Referenced by combineFaddFsub().

combineFaddFsub()

SDValue combineFaddFsub ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on floating-point adds/subs.

Definition at line 55843 of file X86ISelLowering.cpp.

References combineFaddCFmul(), combineToHorizontalAddSub(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFAnd()

SDValue combineFAnd ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 57055 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), combineX86FPLogicOp(), getNullFPConstForNullVal(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFAndFNotToFAndn()

SDValue combineFAndFNotToFAndn ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 57023 of file X86ISelLowering.cpp.

References C(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), N, and SDValue().

Referenced by combineFAnd().

combineFAndn()

SDValue combineFAndn ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 57072 of file X86ISelLowering.cpp.

References combineX86FPLogicOp(), getNullFPConstForNullVal(), isNullFPScalarOrVectorConst(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMA()

SDValue combineFMA ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 57958 of file X86ISelLowering.cpp.

References A(), assert(), B(), C(), CodeSize, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), getInvertedVectorForFMA(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::isNullConstant(), llvm::TargetLoweringBase::isOperationExpand(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), llvm::TargetLoweringBase::isTypeLegal(), N, negateFMAOpcode(), and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMADDSUB()

SDValue combineFMADDSUB ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 58055 of file X86ISelLowering.cpp.

References CodeSize, llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, negateFMAOpcode(), and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMinFMax()

SDValue combineFMinFMax ( SDNode * N, SelectionDAG & DAG )

static

Do target-specific dag combines on PSEUDO_FMIN/PSEUDO_FMAX nodes.

Definition at line 57106 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm_unreachable, N, llvm::ISD::PSEUDO_FMAX, llvm::ISD::PSEUDO_FMIN, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMinNumFMaxNum()

SDValue combineFMinNumFMaxNum ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 57131 of file X86ISelLowering.cpp.

References DL, llvm::ISD::FMAXNUM, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::SelectionDAG::isKnownNeverNaN(), isSoftF16(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::ISD::PSEUDO_FMAX, llvm::ISD::PSEUDO_FMIN, SDValue(), and llvm::ISD::SETUO.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMulcFCMulc()

SDValue combineFMulcFCMulc ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 55737 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::getBitcast(), llvm::KnownBits::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getNode(), llvm::KnownBits::isConstant(), LHS, N, RHS, std::swap(), and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFneg()

SDValue combineFneg ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on floating point negations.

Definition at line 56536 of file X86ISelLowering.cpp.

References CodeSize, DL, llvm::ISD::FMUL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetLowering::getNegatedExpression(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isFNEG(), llvm::TargetLoweringBase::isTypeLegal(), N, and SDValue().

Referenced by combineFOr(), combineXor(), and llvm::X86TargetLowering::PerformDAGCombine().

combineFOr()

SDValue combineFOr ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 57086 of file X86ISelLowering.cpp.

References assert(), combineFneg(), combineX86FPLogicOp(), isNullFPScalarOrVectorConst(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP16_TO_FP()

SDValue combineFP16_TO_FP ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 63042 of file X86ISelLowering.cpp.

References llvm::X86::CUR_DIRECTION, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_TO_FP16, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVectorIdxConstant(), N, llvm::ISD::SCALAR_TO_VECTOR, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP_EXTEND()

SDValue combineFP_EXTEND ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 63065 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::EVT::changeTypeToInteger(), llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::EVT::isVectorOf(), N, SDValue(), llvm::ISD::SHL, llvm::ISD::STRICT_FP_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP_ROUND()

SDValue combineFP_ROUND ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 63194 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86::CUR_DIRECTION, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FABS, llvm::ISD::FNEG, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), llvm::SDNode::isStrictFPOpcode(), llvm::EVT::isVectorOf(), N, Opc, SDValue(), llvm::ISD::SINT_TO_FP, and llvm::ISD::STRICT_SINT_TO_FP.

combineFP_TO_xINT_SAT()

SDValue combineFP_TO_xINT_SAT ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 59252 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::Full, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::APInt::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isVector(), N, SDValue(), llvm::APInt::sext(), llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMIN, llvm::Undef, and llvm::APInt::zext().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFPToSInt()

SDValue combineFPToSInt ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 59241 of file X86ISelLowering.cpp.

References llvm::ISD::FRINT, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::LRINT, N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFunnelShift()

SDValue combineFunnelShift ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 59337 of file X86ISelLowering.cpp.

References DL, llvm::ISD::FSHR, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), Imm, llvm::X86::isConstantSplat(), llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), N, SDValue(), std::swap(), uint64_t, and llvm::APInt::urem().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineGatherScatter()

SDValue combineGatherScatter ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 58773 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::sampleprof::Base, C(), canonicalizeBoolMask(), llvm::cast(), llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::dyn_cast(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getSplatBuildVector(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getValidMinimumShiftAmount(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), I, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::BuildVectorSDNode::isConstant(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Log2_32(), N, llvm::BitVector::none(), rebuildGatherScatter(), SDValue(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), llvm::Splat, llvm::ISD::SUB, llvm::ISD::TRUNCATE, uint64_t, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineGF2P8AFFINEQB()

SDValue combineGF2P8AFFINEQB ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 63023 of file X86ISelLowering.cpp.

References combineAndOnGF2P8AFFINEQBOperand(), combineNestedGF2P8AFFINEQB(), combineXorOnGF2P8AFFINEQBOperand(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineHorizOpWithShuffle()

SDValue combineHorizOpWithShuffle ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 51304 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), isAnyZero(), llvm::EVT::isFloatingPoint(), isHorizOp(), LHS, llvm::Lo, N, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), SDValue(), llvm::SelectionDAG::SplitVector(), and std::swap().

Referenced by combineVectorHADDSUB(), and combineVectorPack().

combinei64TruncSrlConstant()

SDValue combinei64TruncSrlConstant ( SDValue N, EVT VT, SelectionDAG & DAG, const SDLoc & DL )

static

Definition at line 55896 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::APInt::countr_zero(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getValidShiftAmount(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::isa(), llvm::APInt::lshr(), N, llvm::ISD::OR, SDValue(), llvm::ISD::SRL, llvm::APInt::trunc(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by combineTruncatedArithmetic().

combineINSERT_SUBVECTOR()

SDValue combineINSERT_SUBVECTOR ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 61879 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), collectConcatOps(), combineConcatVectorOps(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShufflesRecursively(), llvm::copy(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), getBROADCAST_LOAD(), llvm::SDValue::getConstantOperandVal(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::MVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::APInt::insertBits(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), isFoldableAsShuffle(), llvm::MVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), isShuffleEquivalent(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::Lo, N, llvm::peekThroughOneUseBitcasts(), SDValue(), llvm::ISD::SHL, llvm::SmallVectorTemplateCommon< T, typename >::size(), uint64_t, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineIntDivRem()

SDValue combineIntDivRem ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 50693 of file X86ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FDIV, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_UINT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::getUnmaskedBinOpOpcode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE2(), llvm::APFloatBase::IEEEsingle(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::APInt::isAllOnes(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::MASKED_SDIV, llvm::ISD::MASKED_SREM, llvm::ISD::MASKED_UDIV, llvm::ISD::MASKED_UREM, llvm::ISD::MUL, N, llvm::NextPowerOf2(), Opc, llvm::ISD::SDIV, SDValue(), llvm::APFloatBase::semanticsPrecision(), llvm::APInt::setBit(), llvm::ISD::SINT_TO_FP, splitVectorIntBinary(), llvm::ISD::SREM, llvm::ISD::SUB, llvm::X86::TO_NEAREST_INT, llvm::APInt::uge(), llvm::ISD::UINT_TO_FP, llvm::ISD::UREM, llvm::X86Subtarget::useAVX512Regs(), widenSubVector(), llvm::SelectionDAG::WidenVector(), X, and Y.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_VOID()

SDValue combineINTRINSIC_VOID ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63433 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, SDValue(), and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_W_CHAIN()

SDValue combineINTRINSIC_W_CHAIN ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63419 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, SDValue(), and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_WO_CHAIN()

SDValue combineINTRINSIC_WO_CHAIN ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63405 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithoutChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, SDValue(), and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineKSHIFT()

SDValue combineKSHIFT ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 62802 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), I, llvm::isa(), llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllZeros(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::APInt::lshr(), N, llvm::peekThroughBitcasts(), RHS, SDValue(), llvm::APInt::shl(), llvm::TargetLowering::SimplifyDemandedVectorElts(), and uint64_t.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineLoad()

SDValue combineLoad ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 54373 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::cast(), combineConstantPoolLoads(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), extractSubVector(), llvm::Fast, llvm::SelectionDAG::getAddrSpaceCast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::TypeSize::getFixed(), llvm::EVT::getFixedSizeInBits(), llvm::SDNode::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::ISD::NON_EXTLOAD, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, SDValue(), llvm::ISD::TokenFactor, and llvm::SDNode::users().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineLogicBlendIntoConditionalNegate()

SDValue combineLogicBlendIntoConditionalNegate ( EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 48614 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), llvm::SDPatternMatch::m_AllOf(), llvm::MIPatternMatch::m_Neg(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SUB, std::swap(), X, llvm::ISD::XOR, and Y.

Referenced by combineLogicBlendIntoPBLENDV(), and combineSelect().

combineLogicBlendIntoPBLENDV()

SDValue combineLogicBlendIntoPBLENDV ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 53321 of file X86ISelLowering.cpp.

References assert(), combineLogicBlendIntoConditionalNegate(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::isInteger(), matchLogicBlend(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), SDValue(), X, and Y.

Referenced by combineOr().

combineLRINT_LLRINT()

SDValue combineLRINT_LLRINT ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 55854 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::TargetLoweringBase::isTypeLegal(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMaskBitOp()

SDValue combineMaskBitOp ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 52515 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::ISD::isBitwiseLogicOp(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_BitwiseLogic(), llvm::MIPatternMatch::m_Not(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_Value(), N, Opc, llvm::SDPatternMatch::sd_match(), SDValue(), and llvm::ISD::TRUNCATE.

Referenced by combineAnd(), combineOr(), and combineXor().

combineMaskedLoad()

SDValue combineMaskedLoad ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 54653 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), canonicalizeBoolMask(), llvm::cast(), combineMaskedLoadConstantMask(), llvm::ISD::DELETED_NODE, DL, llvm::SelectionDAG::getMaskedLoad(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAVX512(), llvm::ISD::MLOAD, N, llvm::ISD::NON_EXTLOAD, reduceMaskedLoadToScalarLoad(), SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyMultipleUseDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMaskedLoadConstantMask()

SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode * ML, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 54605 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMaskedLoad(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), ML, and SDValue().

Referenced by combineMaskedLoad().

combineMaskedStore()

SDValue combineMaskedStore ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 54751 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), canonicalizeBoolMask(), llvm::cast(), llvm::ISD::DELETED_NODE, detectSSatPattern(), detectUSatPattern(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlign(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedStoreSDNode::getOffset(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::APInt::getSignMask(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::Value::hasOneUse(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::EVT::isInteger(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::EVT::isVector(), llvm::ISD::MSTORE, N, Opc, reduceMaskedStoreToScalarStore(), SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMOVDQ2Q()

SDValue combineMOVDQ2Q ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 63295 of file X86ISelLowering.cpp.

References llvm::cast(), llvm::MemSDNode::getBaseAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMOVMSK()

SDValue combineMOVMSK ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 58603 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxPopulation(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::X86Subtarget::hasSSE2(), Imm, llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), IsNOT(), llvm::KnownBits::isZero(), N, llvm::peekThroughOneUseBitcasts(), SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMul()

SDValue combineMul ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 50836 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), C(), combineMulSpecial(), combineMulToPMADD52(), combineMulToPMADDWD(), combineMulToPMULDQ(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::Function::hasMinSize(), INT64_MIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::KnownBits::isConstant(), llvm::EVT::isInteger(), llvm::isPowerOf2_64(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::Log2_64(), MulConstantOptimization, N, Opc, reduceVMULWidth(), SDValue(), llvm::ISD::SHL, llvm::ISD::SUB, std::swap(), and uint64_t.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMulSpecial()

SDValue combineMulSpecial ( uint64_t MulAmt, SDNode * N, SelectionDAG & DAG, EVT VT, const SDLoc & DL )

static

Definition at line 50416 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::countr_zero(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isPowerOf2_64(), llvm::Log2_64(), N, SDValue(), llvm::ISD::SHL, llvm::ISD::SUB, and uint64_t.

Referenced by combineMul().

combineMulToPMADD52()

SDValue combineMulToPMADD52 ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 50657 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), getZeroVector(), llvm::KnownBits::mul(), N, and SDValue().

Referenced by combineMul().

combineMulToPMADDWD()

SDValue combineMulToPMADDWD ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 50505 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::ComputeMaxSignificantBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isPowerOf2_32(), llvm::EVT::isVectorOf(), llvm::SelectionDAG::MaskedValueIsZero(), N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, SplitOpsAndApply(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by combineMul().

combineMulToPMULDQ()

SDValue combineMulToPMULDQ ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 50616 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::isPowerOf2_32(), llvm::EVT::isVectorOf(), llvm::SelectionDAG::MaskedValueIsZero(), N, SDValue(), and SplitOpsAndApply().

Referenced by combineMul().

combineNestedGF2P8AFFINEQB()

SDValue combineNestedGF2P8AFFINEQB ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 62923 of file X86ISelLowering.cpp.

References DL, llvm::APInt::extractBits(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNIByteAffine(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), I, llvm::APInt::insertBits(), llvm::APInt::isSplat(), llvm::APInt::lshr(), llvm::SDPatternMatch::m_ConstInt(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::APInt::trunc(), and X.

Referenced by combineGF2P8AFFINEQB().

combineOr()

SDValue combineOr ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 53807 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), canonicalizeBitSelect(), combineAddOrSubToADCOrSBB(), combineAndOrForCcmpCtest(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineCompareEqual(), combineConcatVectorOps(), combineLogicBlendIntoPBLENDV(), combineMaskBitOp(), combineOrWithGF2P8AFFINEQB(), combineOrXorWithSETCC(), combineX86ShufflesRecursively(), Concat, llvm::ISD::CONCAT_VECTORS, Cond, convertIntLogicToFPLogic(), llvm::dyn_cast(), extractSubVector(), llvm::SelectionDAG::getBitcast(), getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getFreeze(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, llvm::APInt::isAllOnes(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SelectionDAG::isGuaranteedNotToBeUndefOrPoison(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::PatternMatch::m_BinOp(), llvm::SDPatternMatch::m_InsertSubvector(), llvm::PatternMatch::m_Or(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::SelectionDAG::MaskedVectorIsZero(), matchScalarReduction(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::APInt::setBit(), llvm::ISD::SETNE, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, uint64_t, llvm::SelectionDAG::UpdateNodeOperands(), X, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineOrCmpEqZeroToCtlzSrl()

SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 53396 of file X86ISelLowering.cpp.

References llvm::X86::COND_E, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::getTargetLowering(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::X86TargetLowering::isCtlzFast(), llvm::isNullConstant(), LHS, lowerX86CmpEqZeroToCtlzSrl(), N, llvm::ISD::OR, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, SDValue(), std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineZext().

combineOrWithGF2P8AFFINEQB()

SDValue combineOrWithGF2P8AFFINEQB ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

GF2P8AFFINEQB computes each output bit from one row of the 8x8 affine matrix, then xors the corresponding immediate bit.

OR-ing the result with a constant byte splat forces selected output bits to 1, so the original affine result for those bits is irrelevant.

Fold: gf2p8affineqb(X, Matrix, Imm) | Mask into: gf2p8affineqb(X, NewMatrix, Imm | Mask)

Definition at line 53717 of file X86ISelLowering.cpp.

References assert(), DL, E(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::SelectionDAG::haveNoCommonBitsSet(), I, Imm, llvm::X86::isConstantSplat(), LHS, llvm::SDPatternMatch::m_ConstInt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Or(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), Matrix, N, llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::SDPatternMatch::sd_match(), SDValue(), uint64_t, and X.

Referenced by combineOr().

combineOrXorWithSETCC()

SDValue combineOrXorWithSETCC ( unsigned Opc, const SDLoc & DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG & DAG )

static

Definition at line 53771 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), combineAddOrSubToADCOrSBB(), DL, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), isConstantPowerOf2(), Opc, llvm::ISD::OR, SDValue(), llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineOr(), and combineXor().

combinePCLMULQDQ()

SDValue combinePCLMULQDQ ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 63326 of file X86ISelLowering.cpp.

References assert(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), isInRange(), isRepeatedTargetShuffleMask(), llvm::M0(), llvm::M1(), N, llvm::peekThroughBitcasts(), llvm::scaleShuffleMaskElts(), SDValue(), llvm::TargetLowering::SimplifyDemandedVectorElts(), and uint64_t.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinePMULDQ()

SDValue combinePMULDQ ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 62600 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), LHS, N, RHS, SDValue(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinePMULH()

SDValue combinePMULH ( SDValue Src, EVT VT, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 56020 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isVectorOf(), LHS, llvm::SDPatternMatch::m_ConstInt(), llvm::PatternMatch::m_Mul(), llvm::SDPatternMatch::m_Srl(), llvm::PatternMatch::m_Value(), llvm::ISD::MULHS, llvm::ISD::MULHU, Opc, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), uint64_t, llvm::APInt::ult(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

combinePTESTCC()

SDValue combinePTESTCC ( SDValue EFLAGS, X86::CondCode & CC, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.

Definition at line 49568 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::KnownBits::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), llvm::MVT::getDoubleNumVectorElementsVT(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getFreeze(), getMOVMSK(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::EVT::is128BitVector(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::KnownBits::isConstant(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::OR, llvm::peekThroughBitcasts(), SDValue(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and X.

Referenced by combineSetCCEFLAGS().

combineRedundantDWordShuffle()

SDValue combineRedundantDWordShuffle ( SDValue N, MutableArrayRef< int > Mask, const SDLoc & DL, SelectionDAG & DAG )

static

Search for a combinable shuffle across a chain ending in pshufd.

We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.

Definition at line 42613 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm_unreachable, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and SDValue().

Referenced by combineTargetShuffle().

combineRotate()

SDValue combineRotate ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 59310 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), Imm, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::X86::isConstantSplat(), llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), N, llvm::ISD::ROTR, SDValue(), uint64_t, and llvm::APInt::urem().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSBB()

SDValue combineSBB ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 59703 of file X86ISelLowering.cpp.

References combineCarryThroughADD(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isNullConstant(), LHS, N, RHS, SDValue(), and llvm::ISD::SUB.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSCALAR_TO_VECTOR()

SDValue combineSCALAR_TO_VECTOR ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 62449 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::User::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::Known, llvm::SDPatternMatch::m_AllOf(), llvm::SDPatternMatch::m_AnyExt(), llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Value(), N, Opc, llvm::peekThroughOneUseBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, supportedVectorShiftWithImm(), uint64_t, llvm::Value::users(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineScalarAndWithMaskSetcc()

SDValue combineScalarAndWithMaskSetcc ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 52633 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::EVT::isVectorOf(), N, SDValue(), and llvm::ISD::SETCC.

Referenced by combineAnd().

combineScalarMinMaxAbsStore()

SDValue combineScalarMinMaxAbsStore ( StoreSDNode * St, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Fold store(abs/min/max(load…)) of scalar i16/i32/i64 into SIMD PABS/PMIN/PMAX to keep memory-bound sort2/abs in the XMM domain.

Definition at line 54949 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::all_of(), DL, F, llvm::MemSDNode::getBaseAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtractVectorElt(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isAbsOpcode(), llvm::ISD::isMinMaxOpcode(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalStore(), llvm::TargetLoweringBase::isOperationLegal(), Opc, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), and llvm::SDNode::uses().

Referenced by combineStore().

combineSelect()

SDValue combineSelect ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 48702 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, C(), llvm::cast(), llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), combineLogicBlendIntoConditionalNegate(), combineSelectOfTwoConstants(), combineToExtendBoolVectorInReg(), combineVSelectToBLENDV(), combinevXi1ConstantToInteger(), combineX86ShufflesRecursively(), commuteSelect(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createShuffleMaskFromVSELECT(), llvm::ISD::CTLZ, DL, llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), I, llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), isConstantPowerOf2(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isConstOrConstSplat(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownToBeAPowerOfTwo(), llvm::KnownBits::isNegative(), llvm::KnownBits::isNonNegative(), IsNOT(), llvm::isNullConstant(), llvm::isNullOrNullSplat(), llvm::isOneConstant(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), llvm::EVT::isVector(), LHS, llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Specific(), llvm::SDPatternMatch::m_SpecificCondCode(), llvm::PatternMatch::m_SpecificInt(), N, narrowVectorSelect(), llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), RHS, llvm::scaleShuffleMaskElts(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SELECT, Select, llvm::ISD::SETCC, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SRL, supportedVectorVarShift(), llvm::ISD::TRUNCATE, and llvm::ISD::VSELECT.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSelectOfTwoConstants()

SDValue combineSelectOfTwoConstants ( SDNode * N, SelectionDAG & DAG, const SDLoc & DL )

static

Definition at line 48430 of file X86ISelLowering.cpp.

References llvm::APInt::abs(), llvm::ISD::ADD, llvm::cast(), Cond, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::ISD::MUL, N, RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSelect().

combineSetCC()

SDValue combineSetCC ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 58321 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, C(), llvm::cast(), combineAVX512SetCCToKMOV(), combineVectorSizedSetCCEquality(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::dyn_cast(), llvm::ISD::FNEG, llvm::SelectionDAG::getAllOnesConstant(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), getValueType(), llvm::SDValue::getValueType(), getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::KnownBits::intersectWith(), llvm::isa(), llvm::ISD::isAbsOpcode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isInteger(), llvm::APInt::isMinSignedValue(), llvm::isNullConstant(), isNullFPScalarOrVectorConst(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), llvm::EVT::isScalarInteger(), llvm::APInt::isSignBitSet(), llvm::ISD::isSignedIntSetCC(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), llvm::EVT::isVectorOf(), LHS, LowerVSETCC(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Value(), llvm::SelectionDAG::MaskedValueIsZero(), MatchVectorAllEqualTest(), N, llvm::KnownBits::One, llvm::ISD::OR, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, truncateAVX512SetCCNoBWI(), llvm::ISD::UMIN, X, and llvm::KnownBits::Zero.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSetCCAtomicArith()

SDValue combineSetCCAtomicArith ( SDValue Cmp, X86::CondCode & CC, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.

Note that this is only legal for some op/cc combinations.

Definition at line 49154 of file X86ISelLowering.cpp.

References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, llvm::cast(), llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NS, llvm::X86::COND_S, llvm::dyn_cast(), llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), lowerAtomicArithWithLOCK(), Opc, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and SDValue().

Referenced by combineSetCCEFLAGS().

combineSetCCEFLAGS()

SDValue combineSetCCEFLAGS ( SDValue EFLAGS, X86::CondCode & CC, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 50003 of file X86ISelLowering.cpp.

References checkBoolTestSetCCCombine(), checkSignTestSetCCCombine(), combineCarryThroughADD(), combinePTESTCC(), combineSetCCAtomicArith(), combineSetCCMOVMSK(), and llvm::X86::COND_B.

Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().

combineSetCCMOVMSK()

SDValue combineSetCCMOVMSK ( SDValue EFLAGS, X86::CondCode & CC, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 49768 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, assert(), llvm::ISD::BITCAST, canScaleShuffleElements(), llvm::MVT::changeVectorElementTypeToInteger(), collectConcatOps(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::dyn_cast(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), getNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isCompletePermute(), llvm::APInt::isMask(), llvm::SDValue::isUndef(), llvm::APInt::isZero(), LHS, llvm::ISD::OR, llvm::peekThroughBitcasts(), RHS, SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by combineSetCCEFLAGS().

combineSext()

SDValue combineSext ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 57853 of file X86ISelLowering.cpp.

References combineExtSetcc(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isVector(), N, promoteExtBeforeAdd(), PromoteMaskArithmetic(), SDValue(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSextInRegCmov()

SDValue combineSextInRegCmov ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 57578 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::cast(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isa(), N, SDValue(), llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSignExtendInReg().

combineShiftLeft()

SDValue combineShiftLeft ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 51060 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, Cond, llvm::ISD::Constant, DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isInteger(), llvm::EVT::isVector(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Specific(), llvm::SDPatternMatch::m_SpecificCondCode(), llvm::PatternMatch::m_SpecificInt(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, supportedVectorVarShift(), llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftRightArithmetic()

SDValue combineShiftRightArithmetic ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 51128 of file X86ISelLowering.cpp.

References combineShiftToPMULH(), llvm::ISD::Constant, DL, llvm::APInt::eq(), llvm::SDNode::getAsAPIntVal(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::APInt::isNegative(), llvm::EVT::isVector(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, Size, llvm::ISD::SRA, supportedVectorVarShift(), and llvm::APInt::ult().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftRightLogical()

SDValue combineShiftRightLogical ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51196 of file X86ISelLowering.cpp.

References llvm::ISD::AND, combineShiftToPMULH(), Cond, llvm::APInt::countr_one(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), getMOVMSK(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignificantBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::APInt::isMask(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Lo, llvm::APInt::lshr(), llvm::MIPatternMatch::m_Not(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Specific(), llvm::SDPatternMatch::m_SpecificCondCode(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), N, llvm::ISD::OR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, supportedVectorVarShift(), llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VSELECT, and X.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftToPMULH()

SDValue combineShiftToPMULH ( SDNode * N, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 51018 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::isVector(), LHS, llvm::PatternMatch::m_Mul(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::ISD::MULHS, llvm::ISD::MULHU, N, Opc, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

Referenced by combineShiftRightArithmetic(), and combineShiftRightLogical().

combineShuffle()

SDValue combineShuffle ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 44313 of file X86ISelLowering.cpp.

References canonicalizeShuffleWithOp(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineToConsecutiveLoads(), combineX86ShufflesRecursively(), llvm::dyn_cast(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), N, narrowShuffle(), SDValue(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShuffleToAddSubOrFMAddSub()

SDValue combineShuffleToAddSubOrFMAddSub ( SDNode * N, const SDLoc & DL, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.

Definition at line 44239 of file X86ISelLowering.cpp.

References combineShuffleToFMAddSub(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), isFMAddSubOrFMSubAdd(), N, Opc, and SDValue().

Referenced by combineShuffle().

combineShuffleToFMAddSub()

SDValue combineShuffleToFMAddSub ( SDNode * N, const SDLoc & DL, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.

Definition at line 44197 of file X86ISelLowering.cpp.

References DL, llvm::ISD::FMA, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDValue::hasOneUse(), isAddSubOrSubAddMask(), isAnyZero(), llvm::TargetLoweringBase::isTypeLegal(), N, SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and std::swap().

Referenced by combineShuffleToAddSubOrFMAddSub().

combineSignExtendInReg()

SDValue combineSignExtendInReg ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 57639 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::cast(), combineSextInRegCmov(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::ISD::LOAD, N, PromoteMaskArithmetic(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSIntToFP()

SDValue combineSIntToFP ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 59120 of file X86ISelLowering.cpp.

References assert(), llvm::BitWidth, llvm::X86TargetLowering::BuildFILD(), llvm::cast(), llvm::EVT::changeVectorElementType(), combineToFPTruncExtElt(), combineVectorCompareAndMaskUnaryOp(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::MemSDNode::getBaseAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), llvm::EVT::isVector(), llvm::ISD::LOAD, N, P, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineStore()

SDValue combineStore ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 55034 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::MVT::changeTypeToInteger(), combineScalarMinMaxAbsStore(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, detectSSatPattern(), detectUSatPattern(), llvm::dyn_cast(), EmitTruncSStore(), llvm::ISD::EXTRACT_VECTOR_ELT, F, llvm::ISD::FABS, llvm::Fast, llvm::ISD::FNEG, llvm::MemSDNode::getAAInfo(), llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlign(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::MemSDNode::getBaseAlign(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::TypeSize::getFixed(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), GetOppositeBranchCondition(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getTruncStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalStore(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::Lo, llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::m_VT(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), mayFoldIntoVector(), N, narrowBitOpRMW(), Opc, llvm::SDNode::ops(), llvm::peekThroughOneUseBitcasts(), llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::ISD::SCALAR_TO_VECTOR, scalarizeVectorStore(), llvm::SDPatternMatch::sd_match(), SDValue(), splitVectorStore(), llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::Align::value(), and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSub()

SDValue combineSub ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 60444 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), combineAddOrSubToADCOrSBB(), combineSubABS(), combineSubSetcc(), combineToHorizontalAddSub(), combineXorSubCTLZ(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDNode::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::isNullConstant(), llvm::X86::isZeroNode(), N, llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSubABS()

SDValue combineSubABS ( EVT VT, const SDLoc & DL, SDValue N0, SDValue N1, SelectionDAG & DAG )

static

Definition at line 60317 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), Cond, llvm::X86::COND_B, llvm::X86::COND_L, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), SDValue(), llvm::ISD::SUB, and X.

Referenced by combineSub().

combineSubSetcc()

SDValue combineSubSetcc ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 60366 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SDValue::hasOneUse(), N, SDValue(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSub().

combineTargetShuffle()

SDValue combineTargetShuffle ( SDValue N, const SDLoc & DL, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Try to combine x86 target specific shuffles.

Definition at line 43152 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), assert(), llvm::ISD::BITCAST, C(), canonicalizeLaneShuffleWithRepeatedOps(), llvm::cast(), collectConcatOps(), combineCommutableSHUFP(), combineConcatVectorOps(), combineRedundantDWordShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), Concat, llvm::ISD::CONCAT_VECTORS, concatSubVectors(), D(), llvm::DecodeVPERM2X128Mask(), DL, llvm::dyn_cast(), llvm::ArrayRef< T >::equals(), extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::ISD::FDIV, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::MemSDNode::getBaseAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), getBLENDIBlendMask(), getBROADCAST_LOAD(), llvm::MemSDNode::getChain(), llvm::SDValue::getConstantOperandVal(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getEntryNode(), llvm::TypeSize::getFixed(), llvm::MVT::getFixedSizeInBits(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), llvm::Hi, Imm, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isEXTLoad(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::MemSDNode::isSimple(), llvm::SelectionDAG::isSplatValue(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), isUndefOrInRange(), llvm::ISD::isUNINDEXEDLoad(), llvm::VectorType::isValidElementType(), llvm::EVT::isVector(), LHS, llvm::Lo, llvm::Load, lowerShuffleWithPERMV(), llvm::SDPatternMatch::m_InsertSubvector(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::SelectionDAG::MaskedVectorIsZero(), llvm::X86::MaxShuffleCombineDepth, llvm::X86::mayFoldLoad(), llvm::MachineMemOperand::MOLoad, N, narrowLoadToVZLoad(), llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::APIntOps::ScaleBitMask(), scaleShuffleElements(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, llvm::ISD::SRL, llvm::Sub, std::swap(), llvm::ISD::TRUNCATE, uint64_t, llvm::X86Subtarget::useAVX512Regs(), llvm::Value::users(), llvm::V1, and widenSubVector().

Referenced by combineShuffle().

combineTESTP()

SDValue combineTESTP ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 58717 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineToConsecutiveLoads()

SDValue combineToConsecutiveLoads ( EVT VT, SDValue Op, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget, bool IsAfterLegalize )

static

Definition at line 8226 of file X86ISelLowering.cpp.

References assert(), DL, EltsFromConsecutiveLoads(), getShuffleScalarElt(), llvm::EVT::getVectorNumElements(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by combineShuffle().

combineToExtendBoolVectorInReg()

SDValue combineToExtendBoolVectorInReg ( unsigned Opcode, const SDLoc & DL, EVT VT, SDValue N0, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 48304 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplat(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is128BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::Offset, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

combineToExtendCMOV()

SDValue combineToExtendCMOV ( SDNode * Extend, SelectionDAG & DAG )

static

Definition at line 57761 of file X86ISelLowering.cpp.

References DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isa(), SDValue(), and llvm::ISD::SIGN_EXTEND.

Referenced by combineSext(), and combineZext().

combineToFPTruncExtElt()

SDValue combineToFPTruncExtElt ( SDNode * N, SelectionDAG & DAG )

static

If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.

This tries to keep the sequence on XMM registers rather than moving between vector and GPRs.

Definition at line 59024 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, SDValue(), and llvm::ISD::TRUNCATE.

Referenced by combineSIntToFP().

combineToHorizontalAddSub()

SDValue combineToHorizontalAddSub ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 55641 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, assert(), DL, extract128BitVector(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::EVT::is256BitVector(), isHorizontalBinOp(), isLaneCrossingShuffleMask(), isUndefUpperHalf(), LHS, N, RHS, llvm::ISD::SADDSAT, SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), SplitOpsAndApply(), llvm::ISD::SSUBSAT, llvm::ISD::SUB, llvm::SmallVectorImpl< T >::truncate(), and llvm::ISD::VECTOR_SHUFFLE.

Referenced by combineAdd(), combineFaddFsub(), combineSub(), and llvm::X86TargetLowering::PerformDAGCombine().

combineTruncate()

SDValue combineTruncate ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 56229 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::cast(), combinePMULH(), combineTruncatedArithmetic(), combineTruncateWithSat(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinTrailingZeros(), detectPMADDUBSW(), DL, llvm::SDNode::getFlags(), llvm::SelectionDAG::getLoad(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::isByteSized(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegal(), llvm::isPowerOf2_64(), llvm::EVT::isScalarInteger(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::none_of(), llvm::SDNodeFlags::NoUnsignedWrap, SDValue(), llvm::ISD::SRL, llvm::APInt::ule(), and llvm::SDNode::uses().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineTruncatedArithmetic()

SDValue combineTruncatedArithmetic ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget, const SDLoc & DL )

static

Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.

e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) ) TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove anything that is guaranteed to be transformed by DAGCombiner.

Definition at line 55938 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), combinei64TruncSrlConstant(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsFreeTruncation, llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::OR, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

combineTruncateWithSat()

SDValue combineTruncateWithSat ( SDValue In, EVT VT, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 54174 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementType(), llvm::ISD::CONCAT_VECTORS, detectSSatPattern(), detectUSatPattern(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), SDValue(), truncateVectorWithPACK(), and llvm::X86Subtarget::useAVX512Regs().

Referenced by combineTruncate().

combineUIntToFP()

SDValue combineUIntToFP ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 59056 of file X86ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), N, P, SDValue(), llvm::SelectionDAG::SignBitIsZero(), llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVECREDUCE_LOGIC()

SDValue combineVECREDUCE_LOGIC ( SDNode * Reduce, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 47263 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::cast(), combineBitcastvxi1(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getFreeze(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), getMOVMSK(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::ISD::getVecReduceBaseOpcode(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::Lo, LowerVectorAllEqual(), llvm::ISD::OR, llvm::ISD::PARITY, RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVECREDUCE_MUL()

SDValue combineVECREDUCE_MUL ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 48270 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Hi, llvm::isPowerOf2_32(), llvm::Lo, llvm::ISD::MUL, N, SDValue(), and llvm::ISD::VECREDUCE_MUL.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorCompare()

SDValue combineVectorCompare ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 60506 of file X86ISelLowering.cpp.

References assert(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), I, LHS, N, Results, RHS, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorCompareAndMaskUnaryOp()

SDValue combineVectorCompareAndMaskUnaryOp ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 58966 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstant(), llvm::EVT::isVector(), N, and SDValue().

Referenced by combineSIntToFP().

combineVectorHADDSUB()

SDValue combineVectorHADDSUB ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51596 of file X86ISelLowering.cpp.

References assert(), combineHorizOpWithShuffle(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorVT(), llvm::SDValue::isUndef(), LHS, N, RHS, SDValue(), and shouldUseHorizontalOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorInsert()

SDValue combineVectorInsert ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51830 of file X86ISelLowering.cpp.

References assert(), combineX86ShufflesRecursively(), DL, llvm::dyn_cast(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getInsertVectorElt(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::Hi, llvm::ISD::INSERT_VECTOR_ELT, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::Lo, llvm::PatternMatch::m_InsertElt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), N, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorPack()

SDValue combineVectorPack ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51450 of file X86ISelLowering.cpp.

References assert(), combineHorizOpWithShuffle(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), getConstVector(), getEXTEND_VECTOR_INREG(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), IsNOT(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), N, SDValue(), llvm::APInt::setBit(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TRUNCATE, llvm::APInt::truncSSat(), llvm::APInt::truncSSatU(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorShiftImm()

SDValue combineVectorShiftImm ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51677 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::APInt::ashrInPlace(), assert(), C(), combineX86ShufflesRecursively(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getV4X86ShuffleImm(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDNode::isOnlyUserOf(), llvm::SDValue::isUndef(), LHS, llvm::APInt::lshrInPlace(), N, llvm::peekThroughOneUseBitcasts(), RHS, SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorTemplateCommon< T, typename >::size(), uint64_t, and X.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorShiftVar()

SDValue combineVectorShiftVar ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 51644 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::EVT::getSimpleVT(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorSizedSetCCEquality()

SDValue combineVectorSizedSetCCEquality ( EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.

Definition at line 23380 of file X86ISelLowering.cpp.

References assert(), llvm::X86::COND_E, llvm::X86::COND_NE, DL, emitOrXorXorTree(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getScalarValueSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::X86Subtarget::hasAVX(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), isOrXorXorTree(), llvm::EVT::isScalarInteger(), mayFoldIntoVector(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useAVX512Regs(), X, llvm::ISD::XOR, Y, and llvm::ISD::ZERO_EXTEND.

combineVEXTRACT_STORE()

SDValue combineVEXTRACT_STORE ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 55415 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::cast(), llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVPDPBUSDPattern()

SDValue combineVPDPBUSDPattern ( SDNode * Extract, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 47403 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, createVPDPBUSD(), detectExtMul(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), LHS, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, RHS, and SDValue().

Referenced by combineExtractVectorElt().

combineVPMADD()

SDValue combineVPMADD ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 62653 of file X86ISelLowering.cpp.

References assert(), E(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::Hi, I, llvm::ISD::isBuildVectorAllZeros(), LHS, llvm::Lo, N, Opc, RHS, SDValue(), llvm::APInt::sext(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::APInt::zext().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVPMADD52LH()

SDValue combineVPMADD52LH ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 62699 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86::isConstantSplat(), llvm::APInt::isOne(), N, SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::APInt::trunc().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVSelectToBLENDV()

SDValue combineVSelectToBLENDV ( SDNode * N, SelectionDAG & DAG, const SDLoc & DL, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.

This function will also call SimplifyDemandedBits on already created BLENDV to perform additional simplifications.

Definition at line 48507 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::Use::getOperandNo(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Use::getUser(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::Known, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::VSELECT.

Referenced by combineSelect().

combineVTRUNC()

SDValue combineVTRUNC ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 56303 of file X86ISelLowering.cpp.

References detectSSatPattern(), detectUSatPattern(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVTRUNCSAT()

SDValue combineVTRUNCSAT ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 56322 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getV4X86ShuffleImm8ForMask(), getZeroVector(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), isFreeToSplitVector(), LHS, llvm::SDPatternMatch::m_SMaxLike(), llvm::SDPatternMatch::m_UnaryOp(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), N, Opc, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), and splitVector().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinevXi1ConstantToInteger()

SDValue combinevXi1ConstantToInteger ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 46619 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), Imm, and llvm::ISD::isBuildVectorOfConstantSDNodes().

Referenced by combineBitcast(), combineCastedMaskArithmetic(), combineSelect(), and combineStore().

combineX86AddSub()

SDValue combineX86AddSub ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & ST )

static

Definition at line 59609 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), combineAddOrSubToADCOrSBB(), combineX86SubCmpForFlags(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::isOneConstant(), LHS, matchGenericOp(), N, needCarryOrOverflowFlag(), RHS, SDValue(), and llvm::ISD::SUB.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86CloadCstore()

SDValue combineX86CloadCstore ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 60393 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, llvm::cast(), llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::X86::isZeroNode(), N, SDValue(), llvm::Sub, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86FPLogicOp()

SDValue combineX86FPLogicOp ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 56647 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::MVT::changeTypeToInteger(), llvm::ISD::ConstantFP, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLoweringBase::isCommutativeBinOp(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), llvm_unreachable, N, llvm::ISD::OR, SDValue(), and llvm::ISD::XOR.

Referenced by combineFAnd(), combineFAndn(), and combineFOr().

combineX86GatherScatter()

SDValue combineX86GatherScatter ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 58732 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::cast(), llvm::ISD::DELETED_NODE, llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86INT_TO_FP()

SDValue combineX86INT_TO_FP ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 57205 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86SetCC()

SDValue combineX86SetCC ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 58933 of file X86ISelLowering.cpp.

References combineSetCCEFLAGS(), DL, getSETCC(), N, and SDValue().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86ShuffleChain()

SDValue combineX86ShuffleChain ( ArrayRef< SDValue > Inputs, unsigned RootOpc, MVT RootVT, ArrayRef< int > BaseMask, int Depth, ArrayRef< const SDNode * > SrcNodes, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Combine an arbitrary chain of shuffles into a single instruction if possible.

This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.

Definition at line 40859 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), llvm::AllOnes, llvm::ISD::AND, llvm::any_of(), assert(), combineX86ShuffleChainWithExtract(), llvm::ISD::CONCAT_VECTORS, llvm::count_if(), llvm::Depth, DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), extractSubVector(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isFreeToSplitVector(), isInRange(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::EVT::isSimple(), llvm::SelectionDAG::isSplatValue(), isTargetShuffleEquivalent(), isTargetShuffleVariableMask(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), isUndefOrEqual(), isUndefOrInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::EVT::isVector(), llvm::MVT::isVector(), LHS, llvm::Lo, lowerShuffleWithPERMV(), matchBinaryPermuteShuffle(), matchBinaryShuffle(), matchShuffleAsEXTRQ(), matchShuffleAsInsertPS(), matchShuffleAsINSERTQ(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), matchUnaryShuffle(), llvm::X86::mayFoldLoad(), N, llvm::narrowShuffleMaskElts(), Opc, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveZeroablesFromTargetShuffle(), RHS, llvm::ISD::SCALAR_TO_VECTOR, scaleShuffleElements(), SDValue(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, uint64_t, llvm::X86Subtarget::useAVX512Regs(), llvm::V1, and widenSubVector().

Referenced by combineX86ShuffleChainWithExtract(), and combineX86ShufflesRecursively().

combineX86ShuffleChainWithExtract()

SDValue combineX86ShuffleChainWithExtract ( ArrayRef< SDValue > Inputs, unsigned RootOpcode, MVT RootVT, ArrayRef< int > BaseMask, int Depth, ArrayRef< const SDNode * > SrcNodes, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 41612 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), combineX86ShuffleChain(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::Depth, DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), growShuffleMask(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::peekThroughBitcasts(), resolveTargetShuffleInputsAndMask(), SDValue(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), std::swap(), and uint64_t.

Referenced by combineX86ShuffleChain(), and combineX86ShufflesRecursively().

combineX86ShufflesConstants()

SDValue combineX86ShufflesConstants ( MVT VT, ArrayRef< SDValue > Ops, ArrayRef< int > Mask, ArrayRef< const SDNode * > SrcNodes, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 41962 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), assert(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), I, llvm::MVT::isFloatingPoint(), isTargetShuffleVariableMask(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::none_of(), SDValue(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineTargetShuffle(), and combineX86ShufflesRecursively().

combineX86ShufflesRecursively() [1/2]

SDValue combineX86ShufflesRecursively ( ArrayRef< SDValue > SrcOps, int SrcOpIndex, unsigned RootOpc, MVT RootVT, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, bool IsMaskedShuffle, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Fully generic combining of x86 shuffle instructions.

This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:

1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.

Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.

FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.

Definition at line 42087 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ISD::BITCAST, canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::countr_zero(), llvm::Depth, DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::enumerate(), llvm::equal_to(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::extractBits(), llvm::SelectionDAG::getBitcast(), getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), getZeroVector(), llvm::has_single_bit(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::Hi, I, llvm::ISD::INSERT_SUBVECTOR, llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllOnes(), isInRange(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::APInt::isZero(), LHS, llvm::Lo, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleFromZeroables(), resolveTargetShuffleInputsAndMask(), RHS, llvm::APIntOps::ScaleBitMask(), SDValue(), llvm::APInt::setBit(), llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::APInt::trunc(), widenSubVector(), and llvm::APInt::zext().

Referenced by combineAnd(), combineAndnp(), combineEXTEND_VECTOR_INREG(), combineINSERT_SUBVECTOR(), combineOr(), combineSelect(), combineShuffle(), combineTargetShuffle(), combineVectorInsert(), combineVectorPack(), combineVectorShiftImm(), combineX86ShufflesRecursively(), combineX86ShufflesRecursively(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

combineX86ShufflesRecursively() [2/2]

SDValue combineX86ShufflesRecursively ( SDValue Op, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Helper entry wrapper to combineX86ShufflesRecursively.

Definition at line 42542 of file X86ISelLowering.cpp.

References combineX86ShufflesRecursively(), isMaskableNode(), and llvm::X86::MaxShuffleCombineDepth.

combineX86SubCmpForFlags()

SDValue combineX86SubCmpForFlags ( SDNode * N, SDValue Flag, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & ST )

static

Definition at line 52801 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_NE, llvm::SDNode::getAsAPIntVal(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::isNullConstant(), llvm::isOneConstant(), llvm_unreachable, N, llvm::SDNode::op_values(), SDValue(), and X.

Referenced by combineCMP(), and combineX86AddSub().

combineX86XOR()

SDValue combineX86XOR ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 59681 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), LHS, matchGenericOp(), N, RHS, SDValue(), and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineXor()

SDValue combineXor ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 56851 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineFneg(), combineMaskBitOp(), combineOrXorWithSETCC(), combineXorSubCTLZ(), combineXorWithGF2P8AFFINEQB(), combineXorWithTwoGF2P8AFFINEQB(), convertIntLogicToFPLogic(), DL, llvm::dyn_cast(), foldVectorXorShiftIntoCmp(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), LHS, N, RHS, SDValue(), llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineXorOnGF2P8AFFINEQBOperand()

SDValue combineXorOnGF2P8AFFINEQBOperand ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 62994 of file X86ISelLowering.cpp.

References llvm::all_equal(), DL, getGFNIByteAffine(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), Imm, llvm::X86::isConstantSplat(), llvm::SDPatternMatch::m_ConstInt(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Xor(), N, llvm::SDPatternMatch::sd_match(), SDValue(), X, and Y.

Referenced by combineGF2P8AFFINEQB().

combineXorSubCTLZ()

SDValue combineXorSubCTLZ ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 56795 of file X86ISelLowering.cpp.

References assert(), C(), llvm::ISD::CTLZ_ZERO_POISON, DL, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), N, SDValue(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, uint64_t, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSub(), and combineXor().

combineXorWithGF2P8AFFINEQB()

SDValue combineXorWithGF2P8AFFINEQB ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 56698 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), I, Imm, llvm::X86::isConstantSplat(), llvm::SDPatternMatch::m_ConstInt(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Xor(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SDPatternMatch::sd_match(), SDValue(), uint64_t, X, and Y.

Referenced by combineXor().

combineXorWithTwoGF2P8AFFINEQB()

SDValue combineXorWithTwoGF2P8AFFINEQB ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 56749 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::M0(), llvm::M1(), llvm::SDPatternMatch::m_ConstInt(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_TernaryOp(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Xor(), N, llvm::SDPatternMatch::sd_match(), SDValue(), uint64_t, and llvm::ISD::XOR.

Referenced by combineXor().

combineZext()

SDValue combineZext ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

Definition at line 58151 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, combineExtSetcc(), combineOrCmpEqZeroToCtlzSrl(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), concatSubVectors(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, promoteExtBeforeAdd(), PromoteMaskArithmetic(), SDValue(), llvm::ISD::TRUNCATE, and widenBuildVec().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

commuteSelect()

SDValue commuteSelect ( SDNode * N, SelectionDAG & DAG, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 48652 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), canCombineAsMaskOperation(), Cond, DL, llvm::User::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::X86Subtarget::hasAVX512(), LHS, llvm::SDPatternMatch::m_AllOf(), llvm::SDPatternMatch::m_CondCode(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::m_VSelect(), llvm::make_early_inc_range(), N, RHS, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::SelectionDAG::UpdateNodeOperands(), X, and Y.

Referenced by combineSelect().

computeInLaneShuffleMask()

void computeInLaneShuffleMask ( const ArrayRef< int > & Mask, int LaneSize, SmallVector< int > & InLaneMask )

static

Helper to get compute inlane shuffle mask for a complete shuffle mask.

Definition at line 15795 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::assign(), and Size.

Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV8F32Shuffle().

computeKnownBitsForHorizontalOperation()

KnownBits computeKnownBitsForHorizontalOperation ( const SDValue Op, const APInt & DemandedElts, unsigned Depth, const SelectionDAG & DAG, const function_ref< KnownBits(const KnownBits &, const KnownBits &)> KnownBitsFunc )

static

Definition at line 39403 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::getHorizDemandedEltsForFirstOperand(), and llvm::APInt::isZero().

computeKnownBitsForPMADDUBSW()

void computeKnownBitsForPMADDUBSW ( SDValue LHS, SDValue RHS, KnownBits & Known, const APInt & DemandedElts, const SelectionDAG & DAG, unsigned Depth )

static

Definition at line 39380 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, llvm::Known, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::KnownBits::sadd_sat(), llvm::APIntOps::ScaleBitMask(), llvm::KnownBits::sext(), and llvm::KnownBits::zext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeKnownBitsForPMADDWD()

void computeKnownBitsForPMADDWD ( SDValue LHS, SDValue RHS, KnownBits & Known, const APInt & DemandedElts, const SelectionDAG & DAG, unsigned Depth )

static

Definition at line 39358 of file X86ISelLowering.cpp.

References llvm::KnownBits::add(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, llvm::Known, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::sext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeKnownBitsForPSADBW()

void computeKnownBitsForPSADBW ( SDValue LHS, SDValue RHS, KnownBits & Known, const APInt & DemandedElts, const SelectionDAG & DAG, unsigned Depth )

static

Definition at line 39341 of file X86ISelLowering.cpp.

References llvm::KnownBits::abdu(), llvm::KnownBits::add(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::Known, LHS, RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::zext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeZeroableShuffleElements()

void computeZeroableShuffleElements ( ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt & KnownUndef, APInt & KnownZero )

static

Compute whether each element of a shuffle is zeroable.

A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.

Definition at line 6344 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::APInt::extractBits(), llvm::SDValue::getNode(), llvm::APInt::getZero(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::peekThroughBitcasts(), llvm::APInt::setBit(), Size, and llvm::V1.

concatSubVectors()

SDValue concatSubVectors ( SDValue V1, SDValue V2, SelectionDAG & DAG, const SDLoc & dl )

static

Definition at line 4996 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), insertSubVector(), and llvm::V1.

Referenced by combineConcatVectorOps(), combineTargetShuffle(), combineZext(), EltsFromConsecutiveLoads(), lowerV8I16Shuffle(), and llvm::X86TargetLowering::ReplaceNodeResults().

convertIntLogicToFPLogic()

SDValue convertIntLogicToFPLogic ( unsigned Opc, const SDLoc & DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const X86Subtarget & Subtarget )

static

If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.

Definition at line 52237 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::cast(), cheapX86FSETCC_SSE(), convertIntLogicToFPLogicOpcode(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), Opc, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SETCC, and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

convertIntLogicToFPLogicOpcode()

unsigned convertIntLogicToFPLogicOpcode ( unsigned Opcode )

static

Definition at line 52221 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by combineBitOpWithMOVMSK(), and convertIntLogicToFPLogic().

convertShiftLeftToScale()

SDValue convertShiftLeftToScale ( SDValue Amt, const SDLoc & dl, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Definition at line 31221 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, convertShiftLeftToScale(), llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPack(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, I, llvm::Lo, SDValue(), llvm::ISD::SHL, llvm::APInt::shl(), and uint64_t.

Referenced by convertShiftLeftToScale(), LowerRotate(), and LowerShift().

createMMXBuildVector()

SDValue createMMXBuildVector ( BuildVectorSDNode * BV, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 46682 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::NumOps, AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::Splat.

Referenced by combineBitcast().

createPackShuffleMask()

void createPackShuffleMask ( MVT VT, SmallVectorImpl< int > & Mask, bool Unary, unsigned NumStages = 1 )

static

Create a shuffle mask that matches the PACKSS/PACKUS truncation.

A multi-stage pack shuffle mask is created by specifying NumStages > 1. Note: This ignores saturation, so inputs must be checked first.

Definition at line 5918 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::Increment, and llvm::Offset.

Referenced by getFauxShuffleMask(), and matchShuffleWithPACK().

createPHIsForCMOVsInSinkBB()

MachineInstrBuilder createPHIsForCMOVsInSinkBB ( MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock * TrueMBB, MachineBasicBlock * FalseMBB, MachineBasicBlock * SinkMBB )

static

Definition at line 37088 of file X86ISelLowering.cpp.

References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::X86::GetOppositeBranchCondition(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), std::swap(), and TII.

createPSADBW()

SDValue createPSADBW ( SelectionDAG & DAG, SDValue N0, SDValue N1, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 47234 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), RegSize, and SplitOpsAndApply().

Referenced by combineBasicSADPattern().

createSetFPEnvNodes()

SDValue createSetFPEnvNodes ( SDValue Ptr, SDValue Chain, const SDLoc & DL, EVT MemVT, MachineMemOperand * MMO, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 29340 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_VOID, and X87StateSize.

createShuffleMaskFromVSELECT()

bool createShuffleMaskFromVSELECT ( SmallVectorImpl< int > & Mask, SDValue Cond, bool IsBLENDV = false )

static

Definition at line 6562 of file X86ISelLowering.cpp.

References Cond, llvm::EVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), isZero(), and llvm::SM_SentinelUndef.

Referenced by combineSelect(), getFauxShuffleMask(), and lowerVSELECTtoVectorShuffle().

createVariablePermute()

SDValue createVariablePermute ( MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.

The vectors might be manipulated to match the type of the permute op.

Definition at line 9122 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, createVariablePermute(), DL, extract128BitVector(), extract256BitVector(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelectCC(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::isPowerOf2_64(), llvm::Lo, llvm::ISD::MUL, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::MVT::SimpleTy, SplitOpsAndApply(), uint64_t, widenSubVector(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by createVariablePermute(), and LowerBUILD_VECTORAsVariablePermute().

createVPDPBUSD()

SDValue createVPDPBUSD ( SelectionDAG & DAG, SDValue LHS, SDValue RHS, unsigned & LogBias, const SDLoc & DL, const X86Subtarget & Subtarget )

static

Definition at line 47189 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), LHS, RegSize, RHS, and SplitOpsAndApply().

Referenced by combineVPDPBUSDPattern().

detectExtMul()

bool detectExtMul ( SelectionDAG & DAG, const SDValue & Mul, SDValue & Op0, SDValue & Op1 )

static

Definition at line 47159 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::BuildVectorSDNode::isConstant(), IsFreeTruncation, Mul, llvm::ISD::SIGN_EXTEND, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineVPDPBUSDPattern().

detectPMADDUBSW()

SDValue detectPMADDUBSW ( SDValue In, EVT VT, SelectionDAG & DAG, const X86Subtarget & Subtarget, const SDLoc & DL )

static

Definition at line 56109 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_VECTOR, detectSSatPattern(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExt(), llvm::SDPatternMatch::sd_match(), SDValue(), SplitOpsAndApply(), and std::swap().

Referenced by combineTruncate().

detectSSatPattern()

SDValue detectSSatPattern ( SDValue In, EVT VT, bool MatchPackUS = false )

static

Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).

With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type]. Return the source value to be truncated or SDValue() if the pattern was not matched.

Definition at line 54147 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::APInt::getZero(), llvm::PatternMatch::m_SMax(), llvm::PatternMatch::m_SMin(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::APInt::sext(), llvm::SMax, llvm::SMin, and llvm::APInt::zext().

Referenced by combineMaskedStore(), combineStore(), combineTruncateWithSat(), combineVTRUNC(), and detectPMADDUBSW().

detectUSatPattern()

SDValue detectUSatPattern ( SDValue In, EVT VT, SelectionDAG & DAG, const SDLoc & DL )

static

Detect patterns of truncation with unsigned saturation:

1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type). Return the source value x to be truncated or SDValue() if the pattern was not matched.

2. (truncate (smin (smax (x, C1), C2)) to dest_type), where C1 >= 0 and C2 is unsigned max of destination type.

   (truncate (smax (smin (x, C2), C1)) to dest_type) where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.

   These two patterns are equivalent to: (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type) So return the smax(x, C1) value to be truncated or SDValue() if the pattern was not matched.

Definition at line 54107 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::isMask(), llvm::APInt::isNonNegative(), llvm::SDPatternMatch::m_ConstInt(), llvm::PatternMatch::m_SMax(), llvm::PatternMatch::m_SMin(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SMAX, llvm::SMax, llvm::SMin, llvm::APInt::uge(), and llvm::UMin.

Referenced by combineMaskedStore(), combineStore(), combineTruncateWithSat(), and combineVTRUNC().

EltsFromConsecutiveLoads()

SDValue EltsFromConsecutiveLoads ( EVT VT, ArrayRef< SDValue > Elts, const SDLoc & DL, SelectionDAG & DAG, const X86Subtarget & Subtarget, bool IsAfterLegalize, unsigned Depth = 0 )

static

Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.

Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a

Definition at line 7810 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::all_of(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::sampleprof::Base, llvm::commonAlignment(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), llvm::APInt::countr_zero(), llvm::Depth, DL, llvm::ArrayRef< T >::drop_back(), EltsFromConsecutiveLoads(), llvm::APInt::extractBits(), findEltLoadSrc(), llvm::ISD::FREEZE, llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::MemSDNode::getAlign(), llvm::MemSDNode::getBaseAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::TypeSize::getFixed(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getFloatingPointVT(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getFreeze(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MachinePointerInfo::isDereferenceable(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegal(), llvm::isPowerOf2_32(), llvm::MemSDNode::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::SelectionDAG::MaxRecursionDepth, llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::MachineMemOperand::MOLoad, llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughFreeze(), llvm::APInt::popcount(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::rbegin(), llvm::SmallVectorTemplateCommon< T, typename >::rbegin(), llvm::ArrayRef< T >::rend(), llvm::SmallVectorTemplateCommon< T, typename >::rend(), SDValue(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), llvm::ISD::SRL, and llvm::ISD::TRUNCATE.

Referenced by combineConcatVectorOps(), combineToConsecutiveLoads(), EltsFromConsecutiveLoads(), and lowerBuildVectorToBitOp().

EmitAVX512Test()

SDValue EmitAVX512Test ( SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc & dl, SelectionDAG & DAG, const X86Subtarget & Subtarget, SDValue & X86CC )

static

Definition at line 25048 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), LHS, llvm::ISD::OR, RHS, SDValue(), llvm::ISD::SETEQ, and llvm::ISD::SETNE.

EmitCmp()

SDValue EmitCmp ( SDValue Op0, SDValue Op1, X86::CondCode X86CC, const SDLoc & dl, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.

Definition at line 24021 of file X86ISelLowering.cpp.

References llvm::Add, assert(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SelectionDAG::doesNodeExist(), llvm::dyn_cast(), EmitTest(), llvm::MachineFunction::getFunction(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::Function::hasMinSize(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), isX86CCSigned(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::mayFoldLoad(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by emitConjunctionForCCMPRec().

emitConjunctionForCCMP()

SDValue emitConjunctionForCCMP ( SDValue Val, X86::CondCode & OutCC, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 25722 of file X86ISelLowering.cpp.

References canEmitConjunctionForCCMP(), llvm::X86::COND_INVALID, emitConjunctionForCCMPRec(), and SDValue().

emitConjunctionForCCMPRec()

SDValue emitConjunctionForCCMPRec ( SDValue Val, X86::CondCode & OutCC, bool Negate, SDValue CCOp, X86::CondCode Predicate, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Emit a conjunction or disjunction tree as a CMP followed by a chain of CCMP operations.

Mirrors AArch64's emitConjunctionRec. CCOp: incoming flags value (null for the first/root comparison) Predicate: condition under which CCOp was produced (COND_INVALID at root) Negate: true if this sub-tree should be negated by inverting the conditions on its SETCC leaves OutCC: set to the condition code to test after the whole chain Returns the flags-producing node (CMP/SUB or CCMP).

Definition at line 25617 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), canEmitConjunctionForCCMP(), llvm::cast(), llvm::X86::COND_INVALID, DL, EmitCmp(), emitConjunctionForCCMPRec(), llvm::X86::getCCMPCondFlagsFromCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, std::swap(), and TranslateX86CC().

Referenced by emitConjunctionForCCMP(), and emitConjunctionForCCMPRec().

emitLockedStackOp()

SDValue emitLockedStackOp ( SelectionDAG & DAG, const X86Subtarget & Subtarget, SDValue Chain, const SDLoc & DL )

static

Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.

Location is chosen to be a) very likely accessed only by a single thread to minimize cache traffic, and b) definitely dereferenceable. Returns the new Chain result.

Definition at line 33153 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::X86Subtarget::getFrameLowering(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::X86FrameLowering::has128ByteRedZone(), and SDValue().

Referenced by LowerATOMIC_FENCE(), LowerATOMIC_STORE(), and lowerAtomicArith().

EmitMaskedTruncSStore()

SDValue EmitMaskedTruncSStore ( bool SignedSat, SDValue Chain, const SDLoc & DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand * MMO, SelectionDAG & DAG )

static

Emit Masked Truncating Store with signed or unsigned saturation.

Definition at line 28250 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), and Opc.

Referenced by LowerINTRINSIC_W_CHAIN().

emitOrXorXorTree()

template<typename F>

SDValue emitOrXorXorTree ( SDValue X, const SDLoc & DL, SelectionDAG & DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV )

static

Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.

Definition at line 23353 of file X86ISelLowering.cpp.

References A(), llvm::ISD::AND, B(), DL, emitOrXorXorTree(), F, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SETEQ, llvm::ISD::SETNE, X, and llvm::ISD::XOR.

Referenced by combineVectorSizedSetCCEquality(), and emitOrXorXorTree().

EmitTest()

SDValue EmitTest ( SDValue Op, X86::CondCode X86CC, const SDLoc & dl, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Emit nodes that will be selected as "test Op0,Op0", or something equivalent.

Definition at line 23911 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ISD::AND, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), hasNonFlagsUse(), isProfitableToUseFlagOp(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SHL, llvm::ISD::SSUBO, llvm::ISD::SUB, llvm::ISD::USUBO, and llvm::ISD::XOR.

Referenced by EmitCmp().

EmitTruncSStore()

SDValue EmitTruncSStore ( bool SignedSat, SDValue Chain, const SDLoc & DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand * MMO, SelectionDAG & DAG )

static

Emit Truncating Store with signed or unsigned saturation.

Definition at line 28240 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), and Opc.

Referenced by combineStore(), and LowerINTRINSIC_W_CHAIN().

emitXBegin()

MachineBasicBlock * emitXBegin ( MachineInstr & MI, MachineBasicBlock * MBB, const TargetInstrInfo * TII )

static

Utility function to emit xbegin specifying the start of an RTM region.

Definition at line 36668 of file X86ISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::addLiveIn(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineFunction::insert(), llvm::isPhysRegUsedAfter(), MBB, MI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().

expandFP_TO_UINT_SSE()

SDValue expandFP_TO_UINT_SSE ( MVT VT, SDValue Src, const SDLoc & dl, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 22056 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::FSUB, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), getTargetVShiftByConstNode(), llvm::X86Subtarget::hasAVX2(), and llvm::ISD::OR.

Referenced by llvm::X86TargetLowering::ReplaceNodeResults().

expandIntrinsicWChainHelper()

SDValue expandIntrinsicWChainHelper ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget & Subtarget, SmallVectorImpl< SDValue > & Results )

static

Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.

If operand ScrReg is a valid register identifier, then operand 2 of N is copied to SrcReg. The assumption is that SrcReg is an implicit input to TargetOpcode. Returns a Glue value which can be used to add extra copy-from-reg if the expanded intrinsics implicitly defines extra registers (i.e. not just EDX:EAX).

Definition at line 28117 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_PAIR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), N, llvm::ISD::OR, Results, SDValue(), and llvm::ISD::SHL.

Referenced by getReadTimeStampCounter(), LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().

ExtendToType()

SDValue ExtendToType ( SDValue InOp, MVT NVT, SelectionDAG & DAG, bool FillWithZeroes = false )

static

Widen a vector input to a vector of NVT.

The input vector must have the same element type as NVT.

Definition at line 34088 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::SDNode::ops().

Referenced by LowerMGATHER(), LowerMLOAD(), LowerMSCATTER(), and LowerMSTORE().

extract128BitVector()

SDValue extract128BitVector ( SDValue Vec, unsigned IdxVal, SelectionDAG & DAG, const SDLoc & dl )

static

Generate a DAG to grab 128-bits from a vector > 128 bits.

This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 4417 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineExtractWithShuffle(), combineTargetShuffle(), combineToHorizontalAddSub(), combineX86ShuffleChain(), createVariablePermute(), getTargetVShiftNode(), lowerAddSubToHorizontalOp(), lowerShuffleAsBroadcast(), LowerTruncateVecI1(), lowerV8I16Shuffle(), narrowExtractedVectorSelect(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), vectorizeExtractedCast(), and widenSubVector().

extract256BitVector()

SDValue extract256BitVector ( SDValue Vec, unsigned IdxVal, SelectionDAG & DAG, const SDLoc & dl )

static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 4426 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().

Referenced by createVariablePermute().

ExtractBitFromMaskVector()

SDValue ExtractBitFromMaskVector ( SDValue Op, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Extract one bit from mask vector, like v16i1 or v8i1.

AVX-512 feature.

Definition at line 19017 of file X86ISelLowering.cpp.

References assert(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and widenMaskVector().

extractSubVector()

SDValue extractSubVector ( SDValue Vec, unsigned IdxVal, SelectionDAG & DAG, const SDLoc & dl, unsigned vectorWidth )

static

Definition at line 4378 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::SDValue::isUndef(), and llvm::SDNode::ops().

findEltLoadSrc()

bool findEltLoadSrc ( SDValue Elt, LoadSDNode *& Ld, int64_t & ByteOffset )

static

Definition at line 7763 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::cast(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, findEltLoadSrc(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::ISD::isNON_EXTLoad(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and uint64_t.

Referenced by EltsFromConsecutiveLoads(), and findEltLoadSrc().

FindSingleBitChange()

std::pair< const Value *, BitTestKind > FindSingleBitChange ( const Value * V )

static

Definition at line 32606 of file X86ISelLowering.cpp.

References C(), ConstantBit, llvm::dyn_cast(), I, llvm::isPowerOf2_64(), llvm::MIPatternMatch::m_AllOnes(), llvm::PatternMatch::m_c_And(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), NotConstantBit, NotShiftBit, ShiftBit, uint64_t, and UndefBit.

FixupMMXIntrinsicTypes()

SDValue FixupMMXIntrinsicTypes ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 63370 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), N, llvm::SDVTList::NumVTs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and llvm::SDVTList::VTs.

Referenced by combineINTRINSIC_VOID(), combineINTRINSIC_W_CHAIN(), and combineINTRINSIC_WO_CHAIN().

foldVectorXorShiftIntoCmp()

SDValue foldVectorXorShiftIntoCmp ( SDNode * N, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.

This should be called before type legalization because the pattern may not persist after that.

Definition at line 54051 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::isConstOrConstSplat(), llvm::EVT::isSimple(), N, SDValue(), llvm::ISD::SETGT, llvm::MVT::SimpleTy, and llvm::ISD::SRA.

foldXor1SetCC()

SDValue foldXor1SetCC ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)

Definition at line 56685 of file X86ISelLowering.cpp.

References DL, llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::isOneConstant(), LHS, N, SDValue(), and llvm::ISD::XOR.

Referenced by combineXor().

foldXorTruncShiftIntoCmp()

SDValue foldXorTruncShiftIntoCmp ( SDNode * N, const SDLoc & DL, SelectionDAG & DAG )

static

Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)

Definition at line 53995 of file X86ISelLowering.cpp.

References Cond, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::isOneConstant(), N, SDValue(), llvm::ISD::SETGT, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineXor().

getAltBitOpcode()

unsigned getAltBitOpcode ( unsigned Opcode )

static

Definition at line 46391 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by adjustBitcastSrcVectorSSE1().

getAVX2GatherNode()

SDValue getAVX2GatherNode ( unsigned Opc, SDValue Op, SelectionDAG & DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget & Subtarget )

static

Definition at line 27986 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, C(), llvm::cast(), llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::X86ISD::MGATHER, Opc, and SDValue().

Referenced by LowerINTRINSIC_W_CHAIN().

getAVX512Node()

SDValue getAVX512Node ( unsigned Opcode, const SDLoc & DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 4767 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BITCAST, DL, llvm::dyn_cast(), extractSubVector(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::peekThroughBitcasts(), SDValue(), Widen(), and widenSubVector().

Referenced by canonicalizeBitSelect().

getAVX512TruncNode()

SDValue getAVX512TruncNode ( const SDLoc & DL, MVT DstVT, SDValue Src, const X86Subtarget & Subtarget, SelectionDAG & DAG, bool ZeroUppers )

static

Definition at line 10893 of file X86ISelLowering.cpp.

References DL, extractSubVector(), getAVX512TruncNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), SDValue(), llvm::ISD::TRUNCATE, and widenSubVector().

Referenced by getAVX512TruncNode(), lowerShuffleAsVTRUNC(), and lowerShuffleWithVPMOV().

getBitSelect()

SDValue getBitSelect ( const SDLoc & DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG & DAG )

static

Definition at line 5228 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), LHS, llvm::ISD::OR, and RHS.

Referenced by lowerShuffleAsBitBlend().

getBLENDIBlendMask()

APInt getBLENDIBlendMask ( SDValue V )

static

Get the expanded blend mask from a BLENDI node.

For v16i16 nodes, this will splat the repeated i8 mask.

Definition at line 42594 of file X86ISelLowering.cpp.

References assert(), and llvm::APInt::getSplat().

Referenced by combineConcatVectorOps(), combineTargetShuffle(), and llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode().

getBMIMatchingOp()

SDValue getBMIMatchingOp ( unsigned Opc, SelectionDAG & DAG, SDValue OpMustEq, SDValue Op, unsigned Depth )

static

Definition at line 52712 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::Depth, DL, getBMIMatchingOp(), llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::isOneConstant(), Opc, SDValue(), and llvm::ISD::SUB.

Referenced by combineBMILogicOp(), and getBMIMatchingOp().

getBROADCAST_LOAD()

SDValue getBROADCAST_LOAD ( unsigned Opcode, const SDLoc & DL, EVT VT, EVT MemVT, MemSDNode * Mem, unsigned Offset, SelectionDAG & DAG )

static

Definition at line 7285 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), DL, llvm::MemSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::EVT::getStoreSize(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MemSDNode::isNonTemporal(), llvm::MemSDNode::isSimple(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Offset, llvm::MemSDNode::readMem(), and SDValue().

Referenced by combineConcatVectorOps(), combineINSERT_SUBVECTOR(), combineTargetShuffle(), lowerV2X128Shuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getBT()

SDValue getBT ( SDValue Src, SDValue BitNo, const SDLoc & DL, SelectionDAG & DAG )

static

Helper for attempting to create a X86ISD::BT node.

Definition at line 23292 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::MaskedValueIsZero(), SDValue(), and llvm::ISD::TRUNCATE.

Referenced by combineAnd(), combineCarryThroughADD(), and LowerAndToBT().

getCmpArithCC()

X86::CondCode getCmpArithCC ( const AtomicRMWInst * AI )

static

Definition at line 32966 of file X86ISelLowering.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::X86::COND_INVALID, llvm::AtomicRMWInst::getOperation(), llvm::Value::hasOneUse(), I, matchAddCC(), matchAndCC(), matchOrCC(), matchSubCC(), matchXorCC(), llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::Instruction::user_back(), and llvm::AtomicRMWInst::Xor.

Referenced by shouldExpandCmpArithRMWInIR().

getConstantVector() [1/2]

Constant * getConstantVector ( MVT VT, ArrayRef< APInt > Bits, const APInt & Undefs, LLVMContext & C )

static

Definition at line 8243 of file X86ISelLowering.cpp.

References assert(), C(), E(), llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::Constant::getIntegerValue(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getTypeForEVT(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by combineConcatVectorOps(), and lowerBuildVectorAsBroadcast().

getConstantVector() [2/2]

Constant * getConstantVector ( MVT VT, const APInt & SplatValue, unsigned SplatBitSize, LLVMContext & C )

static

Definition at line 8268 of file X86ISelLowering.cpp.

References assert(), C(), llvm::APInt::extractBits(), llvm::ConstantVector::get(), llvm::Constant::getIntegerValue(), llvm::Type::getIntNTy(), llvm::MVT::getScalarSizeInBits(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

getConstVector() [1/3]

SDValue getConstVector ( ArrayRef< APInt > Bits, const APInt & Undefs, MVT VT, SelectionDAG & DAG, const SDLoc & dl )

static

Definition at line 4285 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::MVT::changeTypeToInteger(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), and llvm::TargetLoweringBase::isTypeLegal().

getConstVector() [2/3]

SDValue getConstVector ( ArrayRef< APInt > Bits, MVT VT, SelectionDAG & DAG, const SDLoc & dl )

static

Definition at line 4321 of file X86ISelLowering.cpp.

References getConstVector(), and llvm::APInt::getZero().

getConstVector() [3/3]

SDValue getConstVector ( ArrayRef< int > Values, MVT VT, SelectionDAG & DAG, const SDLoc & dl, bool IsMask = false )

static

Definition at line 4255 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::Values.

Referenced by combineAndnp(), combineINSERT_SUBVECTOR(), combineSelect(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVPMADD(), combineX86ShuffleChain(), combineX86ShufflesConstants(), getConstVector(), incDecVectorConstant(), IsNOT(), LowerShift(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsElementInsertion(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerV16F32Shuffle(), lowerV16I8Shuffle(), lowerV8F32Shuffle(), lowerV8I32Shuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getEXTEND_VECTOR_INREG()

SDValue getEXTEND_VECTOR_INREG ( unsigned Opcode, const SDLoc & DL, EVT VT, SDValue In, SelectionDAG & DAG )

static

Definition at line 5187 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, assert(), DL, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::EVT::isVector(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by combineConcatVectorOps(), combineVectorPack(), LowerBUILD_VECTORvXi1(), lowerShuffleAsSpecificExtension(), and llvm::X86TargetLowering::ReplaceNodeResults().

getExtractedDemandedElts()

APInt getExtractedDemandedElts ( SDNode * N )

static

Definition at line 19063 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, getExtractedDemandedElts(), llvm::User::getOperand(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::isa(), N, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setAllBits(), and llvm::APInt::setBit().

Referenced by getExtractedDemandedElts().

getFauxShuffleMask()

bool getFauxShuffleMask ( SDValue N, const APInt & DemandedElts, SmallVectorImpl< int > & Mask, SmallVectorImpl< SDValue > & Ops, const SelectionDAG & DAG, unsigned Depth, bool ResolveKnownElts )

static

Definition at line 6601 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::any_of(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ISD::BITCAST, llvm::cast(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, Cond, createPackShuffleMask(), createShuffleMaskFromVSELECT(), llvm::DecodeZeroExtendMask(), llvm::Depth, E(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::extractBits(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::EVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPackDemandedElts(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::Hi, I, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::isa(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isByteSized(), isInRange(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::Lo, llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::Sub, llvm::ISD::TRUNCATE, uint64_t, llvm::ISD::VECTOR_SHUFFLE, llvm::ISD::VSELECT, llvm::ISD::ZERO_EXTEND, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::APInt::zextOrTrunc().

Referenced by getTargetShuffleInputs().

getFlagsOfCmpZeroFori1()

SDValue getFlagsOfCmpZeroFori1 ( SelectionDAG & DAG, const SDLoc & DL, SDValue Mask )

static

Definition at line 34475 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), and SDValue().

Referenced by llvm::X86TargetLowering::visitMaskedLoad(), and llvm::X86TargetLowering::visitMaskedStore().

getGatherNode()

SDValue getGatherNode ( SDValue Op, SelectionDAG & DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget & Subtarget )

static

Definition at line 28018 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, C(), llvm::cast(), llvm::dyn_cast(), llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::X86ISD::MGATHER, and SDValue().

Referenced by LowerINTRINSIC_W_CHAIN().

getGFNIByteAffine()

APInt getGFNIByteAffine ( const APInt & ByteToAffine, const APInt & Matrix64, const APInt & Addend8 )

static

Definition at line 29449 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::byteSwap(), llvm::APInt::getBitWidth(), llvm::APInt::getSplat(), I, llvm::APInt::lshr(), and llvm::APIntOps::ScaleBitMask().

Referenced by combineNestedGF2P8AFFINEQB(), and combineXorOnGF2P8AFFINEQBOperand().

getGFNICtrlImm()

uint64_t getGFNICtrlImm	(	unsigned	Opcode,
		unsigned	Amt = 0 )

Definition at line 29410 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, getGFNICtrlImm(), llvm_unreachable, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.

Referenced by getGFNICtrlImm(), and getGFNICtrlMask().

getGFNICtrlMask()

SDValue getGFNICtrlMask	(	unsigned	Opcode,
		SelectionDAG &	DAG,
		const SDLoc &	DL,
		MVT	VT,
		unsigned	Amt = 0 )

Definition at line 29436 of file X86ISelLowering.cpp.

References assert(), DL, E(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNICtrlImm(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), I, Imm, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and uint64_t.

Referenced by LowerBITREVERSE(), LowerCTTZ(), LowerRotate(), and LowerShiftByScalarImmediate().

getHalfShuffleMask()

bool getHalfShuffleMask ( ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int & HalfIdx1, int & HalfIdx2 )

static

If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.

A mask of half the width with mask indexes adjusted to access the extracted halves of the original shuffle operands is returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or lower half of each input operand is accessed.

Definition at line 16166 of file X86ISelLowering.cpp.

References assert(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::ArrayRef< T >::size().

Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().

getHorizDemandedElts()

void getHorizDemandedElts ( EVT VT, const APInt & DemandedElts, APInt & DemandedLHS, APInt & DemandedRHS )

static

Definition at line 5965 of file X86ISelLowering.cpp.

References llvm::getHorizDemandedEltsForFirstOperand(), and llvm::EVT::getSizeInBits().

Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getIndexFromUnindexedLoad()

SDValue getIndexFromUnindexedLoad ( LoadSDNode * Ld )

static

Definition at line 52469 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::LoadSDNode::getBasePtr(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::LSBaseSDNode::isIndexed(), SDValue(), and llvm::ISD::SHL.

Referenced by combineAndLoadToBZHI().

getIndirectThunkSymbol()

const char * getIndirectThunkSymbol ( const X86Subtarget & Subtarget, Register Reg )

static

Definition at line 37784 of file X86ISelLowering.cpp.

References assert(), llvm_unreachable, and Reg.

getInstrStrFromOpNo()

StringRef getInstrStrFromOpNo ( const SmallVectorImpl< StringRef > & AsmStrs, unsigned OpNo )

static

Definition at line 34421 of file X86ISelLowering.cpp.

References assert(), I, llvm::isAlpha(), llvm::StringRef::npos, and llvm::toString().

Referenced by llvm::X86TargetLowering::isInlineAsmTargetBranch().

getInvertedVectorForFMA()

SDValue getInvertedVectorForFMA ( SDValue V, SelectionDAG & DAG )

static

Definition at line 57910 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::any_of(), assert(), llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::ISD::FMA, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), SDValue(), and llvm::ISD::STRICT_FMA.

Referenced by combineFMA().

getMaskNode()

SDValue getMaskNode ( SDValue Mask, MVT MaskVT, const X86Subtarget & Subtarget, SelectionDAG & DAG, const SDLoc & dl )

static

Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.

Definition at line 10651 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::bitsLE(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::isAllOnesConstant(), llvm::MVT::isScalarInteger(), llvm::X86::isZeroNode(), llvm::Lo, llvm::SelectionDAG::SplitScalar(), and llvm::ISD::TRUNCATE.

Referenced by getGatherNode(), getPrefetchNode(), getScatterNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), lowerShuffleWithEXPAND(), and llvm::X86TargetLowering::ReplaceNodeResults().

getMOVMSK()

SDValue getMOVMSK ( const SDLoc & DL, SDValue V, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

Definition at line 33276 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getConstant(), getMOVMSK(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::Hi, llvm::Lo, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SplitVector(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineBitcastvxi1(), combinePTESTCC(), combineShiftRightLogical(), combineVECREDUCE_LOGIC(), getMOVMSK(), and LowerBITCAST().

getNullFPConstForNullVal()

SDValue getNullFPConstForNullVal ( SDValue V, SelectionDAG & DAG, const X86Subtarget & Subtarget )

static

If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.

In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.

Definition at line 57012 of file X86ISelLowering.cpp.

References getZeroVector(), isNullFPScalarOrVectorConst(), and SDValue().

Referenced by combineFAnd(), and combineFAndn().

getOnesVector()

SDValue getOnesVector ( EVT VT, SelectionDAG & DAG, const SDLoc & dl )

static

Returns a vector of specified type with all bits set.

Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 5011 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by combineEXTRACT_SUBVECTOR(), combineX86ShufflesRecursively(), and materializeVectorConstant().

getOneTrueElt()

int getOneTrueElt ( SDValue V )

static

If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.

Otherwise, return -1.

Definition at line 54500 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), and llvm::EVT::getVectorNumElements().

Referenced by getParamsForOneTrueMaskedElt().

getOpcodeForIndirectThunk()

unsigned getOpcodeForIndirectThunk ( unsigned RPOpc )

static

Definition at line 37770 of file X86ISelLowering.cpp.

References llvm_unreachable.

getPack()

SDValue getPack ( SelectionDAG & DAG, const X86Subtarget & Subtarget, const SDLoc & dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf = false )

static

Returns a node that packs the LHS + RHS nodes together at half width.

May return X86ISD::PACKSS/PACKUS, packing the top/bottom half. TODO: Add subvector splitting if/when we have a need for it.

Definition at line 5306 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), I, LHS, llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and RHS.

Referenced by convertShiftLeftToScale(), LowerFunnelShift(), LowerMUL(), LowerRotate(), and LowervXi8MulWithUNPCK().

getPackDemandedElts()

void getPackDemandedElts ( EVT VT, const APInt & DemandedElts, APInt & DemandedLHS, APInt & DemandedRHS )

static

Definition at line 5940 of file X86ISelLowering.cpp.

References llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::APInt::getZero(), and llvm::APInt::setBit().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getFauxShuffleMask(), llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getParamsForOneTrueMaskedElt()

bool getParamsForOneTrueMaskedElt ( MaskedLoadStoreSDNode * MaskedOp, SelectionDAG & DAG, SDValue & Addr, SDValue & Index, Align & Alignment, unsigned & Offset )

static

Given a masked memory load/store operation, return true if it has one mask bit set.

If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.

Definition at line 54535 of file X86ISelLowering.cpp.

References llvm::commonAlignment(), llvm::MemSDNode::getBaseAlign(), llvm::MemSDNode::getBasePtr(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::Offset.

Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().

getPrefetchNode()

SDValue getPrefetchNode ( unsigned Opc, SDValue Op, SelectionDAG & DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget & Subtarget )

static

Definition at line 28087 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, C(), llvm::dyn_cast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), Opc, and SDValue().

Referenced by LowerINTRINSIC_W_CHAIN().

getPSHUFShuffleMask()

SmallVector< int, 4 > getPSHUFShuffleMask ( SDValue N )

static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 42555 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, and N.

Referenced by combineRedundantDWordShuffle(), and combineTargetShuffle().

getReadTimeStampCounter()

void getReadTimeStampCounter ( SDNode * N, const SDLoc & DL, unsigned Opcode, SelectionDAG & DAG, const X86Subtarget & Subtarget, SmallVectorImpl< SDValue > & Results )

static

Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).

This function is also used to custom lower READCYCLECOUNTER nodes.

Definition at line 28172 of file X86ISelLowering.cpp.

References DL, expandIntrinsicWChainHelper(), llvm::SelectionDAG::getCopyFromReg(), llvm::SDValue::getValue(), N, and Results.

Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().

getScalarMaskingNode()

SDValue getScalarMaskingNode ( SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget & Subtarget, SelectionDAG & DAG, unsigned UpperEltOpSrc = 0 )

static

Creates an SDNode for a predicated scalar operation.

Returns

(X86vselect Mask, Op, PreservedSrc). The mask is coming as MVT::i8 and it should be transformed to MVT::v1i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.

Definition at line 26834 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and llvm::SDValue::isUndef().

getScalarValueForVectorElement()

SDValue getScalarValueForVectorElement ( SDValue V, int Idx, SelectionDAG & DAG )

static

Try to get a scalar value for a specific element of a vector.

Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.

Definition at line 13012 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::isVector(), llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, and SDValue().

Referenced by lowerShuffleAsElementInsertion(), and lowerV2F64Shuffle().

getScatterNode()

SDValue getScatterNode ( unsigned Opc, SDValue Op, SelectionDAG & DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget & Subtarget )

static

Definition at line 28056 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, C(), llvm::cast(), llvm::dyn_cast(), llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::MSCATTER, Opc, and SDValue().

Referenced by LowerINTRINSIC_W_CHAIN().

getSEHRegistrationNodeSize()

int getSEHRegistrationNodeSize ( const Function * Fn )

static

Definition at line 26871 of file X86ISelLowering.cpp.

References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().

Referenced by recoverFramePointer().

getSETCC()

SDValue getSETCC ( X86::CondCode Cond, SDValue EFLAGS, const SDLoc & dl, SelectionDAG & DAG )

static

Helper for creating a X86ISD::SETCC node.

Definition at line 23333 of file X86ISelLowering.cpp.

References Cond, llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getTargetConstant().

getShuffleHalfVectors()

SDValue getShuffleHalfVectors ( const SDLoc & DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG & DAG, bool UseConcat = false )

static

Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.

Definition at line 16217 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_SUBVECTOR, llvm::Offset, std::swap(), and llvm::V1.

Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().

getShuffleScalarElt()

SDValue getShuffleScalarElt ( SDValue Op, unsigned Index, SelectionDAG & DAG, unsigned Depth )

static

Returns the scalar element that will make up the i'th element of the result of the vector shuffle.

Definition at line 7310 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::Depth, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getOperand(), getShuffleScalarElt(), llvm::EVT::getSimpleVT(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::isa(), llvm::MVT::isInteger(), isTargetShuffle(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::Sub, and uint64_t.

Referenced by combineToConsecutiveLoads(), and getShuffleScalarElt().

getShuffleVectorZeroOrUndef()

SDValue getShuffleVectorZeroOrUndef ( SDValue V2, int Idx, bool IsZero, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Return a vector_shuffle of the specified vector of zero or undef vector.

This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 5373 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and llvm::V1.

Referenced by LowerUINT_TO_FP_i32().

getSHUFPDImm()

unsigned getSHUFPDImm ( ArrayRef< int > Mask )

static

Definition at line 10526 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::count_if(), E(), llvm::find_if(), I, and Imm.

Referenced by getSHUFPDImmForMask(), and matchShuffleWithSHUFPD().

getSHUFPDImmForMask()

SDValue getSHUFPDImmForMask ( ArrayRef< int > Mask, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 10556 of file X86ISelLowering.cpp.

References DL, getSHUFPDImm(), and llvm::SelectionDAG::getTargetConstant().

Referenced by lowerShuffleAsLanePermuteAndSHUFP().

getSplitVectorSrc()

SDValue getSplitVectorSrc ( SDValue LHS, SDValue RHS, bool AllowCommute )

static

Definition at line 4357 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_SUBVECTOR, LHS, RHS, and SDValue().

Referenced by combineHorizOpWithShuffle(), combinePTESTCC(), and combineSetCCMOVMSK().

getSUBriOpcode()

unsigned getSUBriOpcode ( bool IsLP64 )

static

Definition at line 37438 of file X86ISelLowering.cpp.

getTargetConstantBitsFromNode()

bool getTargetConstantBitsFromNode ( SDValue Op, unsigned EltSizeInBits, APInt & UndefElts, SmallVectorImpl< APInt > & EltBits, bool AllowWholeUndefs = true, bool AllowPartialUndefs = false )

static

Definition at line 5431 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), C(), llvm::cast(), llvm::dyn_cast(), E(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::Constant::getAggregateElement(), llvm::APInt::getAllOnes(), getBitWidth(), llvm::APInt::getBitWidth(), llvm::APInt::getBoolValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::APInt::getZero(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::APInt::insertBits(), llvm::isa(), llvm::APInt::isAllOnes(), isAnyInRange(), llvm::Type::isDoubleTy(), llvm::Type::isFloatTy(), llvm::Type::isHalfTy(), llvm::Type::isIntegerTy(), llvm::Type::isVectorTy(), llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), llvm::APInt::setBits(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by canonicalizeBitSelect(), combineAnd(), combineAndnp(), combineAndOnGF2P8AFFINEQBOperand(), combineAVX512SetCCToKMOV(), combineBitcast(), combineConcatVectorOps(), combineConstantPoolLoads(), combineExtractVectorElt(), combineINSERT_SUBVECTOR(), combineMOVMSK(), combineNestedGF2P8AFFINEQB(), combineOr(), combineOrWithGF2P8AFFINEQB(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftVar(), combineVPMADD(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), combineXorOnGF2P8AFFINEQBOperand(), combineXorWithGF2P8AFFINEQB(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), convertShiftLeftToScale(), createShuffleMaskFromVSELECT(), getFauxShuffleMask(), getTargetConstantBitsFromNode(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), getTargetShuffleMaskIndices(), isConstantPowerOf2(), llvm::X86::isConstantSplat(), isFNEG(), IsNOT(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getTargetConstantFromBasePtr()

const Constant * getTargetConstantFromBasePtr ( SDValue Ptr )

static

Definition at line 5396 of file X86ISelLowering.cpp.

References llvm::ConstantPoolSDNode::getConstVal(), llvm::ConstantPoolSDNode::getOffset(), getTargetConstantPoolFromBasePtr(), and llvm::ConstantPoolSDNode::isMachineConstantPoolEntry().

Referenced by combineConstantPoolLoads(), getTargetConstantBitsFromNode(), and getTargetConstantFromNode().

getTargetConstantFromNode() [1/2]

const Constant * getTargetConstantFromNode ( LoadSDNode * Load )

static

Definition at line 5403 of file X86ISelLowering.cpp.

References getTargetConstantFromBasePtr(), llvm::ISD::isNormalLoad(), and llvm::Load.

Referenced by canonicalizeShuffleWithOp(), combineAnd(), combineConcatVectorOps(), getTargetConstantBitsFromNode(), llvm::X86TargetLowering::getTargetConstantFromLoad(), getTargetConstantFromNode(), lowerShuffleAsElementInsertion(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle().

getTargetConstantFromNode() [2/2]

const Constant * getTargetConstantFromNode ( SDValue Op )

static

Definition at line 5409 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), getTargetConstantFromNode(), and llvm::peekThroughBitcasts().

getTargetConstantPoolFromBasePtr()

ConstantPoolSDNode * getTargetConstantPoolFromBasePtr ( SDValue Ptr )

static

Definition at line 5388 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDValue::getOpcode(), and llvm::SDValue::getOperand().

Referenced by getTargetConstantFromBasePtr().

getTargetShuffleAndZeroables()

bool getTargetShuffleAndZeroables ( SDValue N, SmallVectorImpl< int > & Mask, SmallVectorImpl< SDValue > & Ops, APInt & KnownUndef, APInt & KnownZero )

static

Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.

Returns true if the target shuffle mask was decoded. FIXME: Merge this with computeZeroableShuffleElements?

Definition at line 6427 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::isFloatingPoint(), isTargetShuffle(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::X86::isZeroNode(), N, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), Size, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, and llvm::V1.

Referenced by getTargetShuffleInputs().

getTargetShuffleInputs() [1/3]

bool getTargetShuffleInputs ( SDValue Op, const APInt & DemandedElts, SmallVectorImpl< SDValue > & Inputs, SmallVectorImpl< int > & Mask, APInt & KnownUndef, APInt & KnownZero, const SelectionDAG & DAG, unsigned Depth, bool ResolveKnownElts )

static

Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.

Returns true if the target shuffle mask was decoded.

Definition at line 7234 of file X86ISelLowering.cpp.

References llvm::Depth, getFauxShuffleMask(), getTargetShuffleAndZeroables(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, resolveTargetShuffleFromZeroables(), and resolveZeroablesFromTargetShuffle().

getTargetShuffleInputs() [2/3]

bool getTargetShuffleInputs ( SDValue Op, const APInt & DemandedElts, SmallVectorImpl< SDValue > & Inputs, SmallVectorImpl< int > & Mask, const SelectionDAG & DAG, unsigned Depth, bool ResolveKnownElts )

static

Definition at line 7260 of file X86ISelLowering.cpp.

References llvm::Depth, and getTargetShuffleInputs().

Referenced by combineConcatVectorOps(), combineEXTRACT_SUBVECTOR(), combineExtractWithShuffle(), combineHorizOpWithShuffle(), combinePCLMULQDQ(), combineSetCCMOVMSK(), combineShuffleToFMAddSub(), combineTargetShuffle(), combineX86ShufflesRecursively(), getFauxShuffleMask(), getTargetShuffleInputs(), getTargetShuffleInputs(), isAddSubOrSubAdd(), isHorizontalBinOp(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and llvm::X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode().

getTargetShuffleInputs() [3/3]

bool getTargetShuffleInputs ( SDValue Op, SmallVectorImpl< SDValue > & Inputs, SmallVectorImpl< int > & Mask, const SelectionDAG & DAG, unsigned Depth = 0, bool ResolveKnownElts = true )

static

Definition at line 7270 of file X86ISelLowering.cpp.

References llvm::Depth, llvm::APInt::getAllOnes(), getTargetShuffleInputs(), llvm::EVT::isSimple(), and llvm::EVT::isVector().

getTargetShuffleMask() [1/2]

bool getTargetShuffleMask ( SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > & Ops, SmallVectorImpl< int > & Mask )

static

Definition at line 6329 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, getTargetShuffleMask(), and N.

getTargetShuffleMask() [2/2]

bool getTargetShuffleMask ( SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > & Ops, SmallVectorImpl< int > & Mask, bool & IsUnary )

static

Calculates the shuffle mask corresponding to the target-specific opcode.

If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.

Definition at line 5980 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::DecodeBLENDMask(), llvm::DecodeEXTRQIMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeINSERTQIMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVALIGNMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::decodeVSHUF64x2FamilyMask(), llvm::DecodeZeroMoveLowMask(), llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantBitsFromNode(), getTargetShuffleMaskIndices(), llvm::MVT::getVectorNumElements(), I, llvm::isa(), isAnyZero(), isTargetShuffle(), isZero(), llvm_unreachable, N, llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.

Referenced by canonicalizeShuffleWithOp(), combineBlendOfPermutes(), combineConcatVectorOps(), combineSelect(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getPSHUFShuffleMask(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getTargetShuffleMaskIndices()

bool getTargetShuffleMaskIndices ( SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > & RawMask, APInt & UndefElts )

static

Definition at line 5809 of file X86ISelLowering.cpp.

References getTargetConstantBitsFromNode(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by getTargetShuffleMask().

getTargetVShiftByConstNode()

SDValue getTargetVShiftByConstNode ( unsigned Opc, const SDLoc & dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG & DAG )

static

Handle vector element shifts where the shift amount is a constant.

Takes immediate version of shift as input.

Definition at line 5040 of file X86ISelLowering.cpp.

References assert(), C(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm_unreachable, Opc, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.

Referenced by combineAdd(), combineAndMaskToShift(), combineMOVMSK(), combineSCALAR_TO_VECTOR(), combineVectorShiftVar(), expandFP_TO_UINT_SSE(), LowerEXTEND_VECTOR_INREG(), LowerFunnelShift(), LowerMUL(), LowerMULH(), LowerMULO(), LowerRotate(), LowerShift(), LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), lowerShuffleAsBitRotate(), and lowerShuffleAsVTRUNC().

getTargetVShiftNode()

SDValue getTargetVShiftNode ( unsigned Opc, const SDLoc & dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Handle vector element shifts by a splat shift amount.

Definition at line 5092 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, extract128BitVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::isSimple(), llvm::MVT::isVector(), Opc, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by LowerFunnelShift(), LowerRotate(), and LowerShiftByScalarVariable().

getTargetVShiftUniformOpcode()

unsigned getTargetVShiftUniformOpcode ( unsigned Opc, bool IsVariable )

static

Definition at line 5020 of file X86ISelLowering.cpp.

References llvm_unreachable, Opc, llvm::ISD::SHL, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by combineSCALAR_TO_VECTOR(), combineVectorShiftVar(), getTargetVShiftNode(), LowerShift(), LowerShiftByScalarImmediate(), and LowerShiftByScalarVariable().

GetTLSADDR()

SDValue GetTLSADDR ( SelectionDAG & DAG, GlobalAddressSDNode * GA, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LoadGlobalBaseReg = false, bool LocalDynamic = false )

static

Definition at line 19823 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::CALLSEQ_END, llvm::ISD::CopyFromReg, llvm::X86AS::FS, llvm::PointerType::get(), llvm::SelectionDAG::getCALLSEQ_END(), llvm::SelectionDAG::getCALLSEQ_START(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SDNode::getGluedUser(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SDNode::getOpcode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::GlobalBaseReg, llvm::X86AS::GS, llvm::SDNode::hasOneUse(), llvm::Offset, SDValue(), llvm::MachineFrameInfo::setHasCalls(), llvm::SDNode::user_begin(), and llvm::TargetMachine::useTLSDESC().

Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), LowerToTLSGeneralDynamicModelX32(), and LowerToTLSLocalDynamicModel().

getUnderlyingExtractedFromVec()

int getUnderlyingExtractedFromVec ( SDValue & ExtractedFromVec, SDValue ExtIdx )

static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 8585 of file X86ISelLowering.cpp.

References assert(), llvm::cast(), llvm::SDNode::getAsZExtVal(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::isa(), and isUndefOrInRange().

Referenced by buildFromShuffleMostly().

getUnpackh()

SDValue getUnpackh ( SelectionDAG & DAG, const SDLoc & dl, EVT VT, SDValue V1, SDValue V2 )

static

Returns a vector_shuffle node for an unpackh operation.

Definition at line 5296 of file X86ISelLowering.cpp.

References llvm::createUnpackShuffleMask(), getVectorShuffle(), and llvm::V1.

Referenced by combineVECREDUCE_MUL(), convertShiftLeftToScale(), LowerAVXExtend(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), and LowervXi8MulWithUNPCK().

getUnpackl()

SDValue getUnpackl ( SelectionDAG & DAG, const SDLoc & dl, EVT VT, SDValue V1, SDValue V2 )

static

Returns a vector_shuffle node for an unpackl operation.

Definition at line 5288 of file X86ISelLowering.cpp.

References llvm::createUnpackShuffleMask(), getVectorShuffle(), and llvm::V1.

Referenced by combineVECREDUCE_MUL(), convertShiftLeftToScale(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), LowerUINT_TO_FP_i64(), and LowervXi8MulWithUNPCK().

getV4X86ShuffleImm()

unsigned getV4X86ShuffleImm ( ArrayRef< int > Mask )

static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 10495 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::find_if(), and Imm.

Referenced by combineVectorShiftImm(), getV4X86ShuffleImm8ForMask(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().

getV4X86ShuffleImm8ForMask()

SDValue getV4X86ShuffleImm8ForMask ( ArrayRef< int > Mask, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 10519 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getTargetConstant(), and getV4X86ShuffleImm().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineConcatVectorOps(), combineINSERT_SUBVECTOR(), combineRedundantDWordShuffle(), combineTargetShuffle(), combineVectorHADDSUB(), combineVectorShiftImm(), combineVTRUNCSAT(), combineX86ShuffleChain(), LowerShift(), lowerShuffleAsSpecificExtension(), lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16GeneralSingleInputShuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

getVectorMaskingNode()

SDValue getVectorMaskingNode ( SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget & Subtarget, SelectionDAG & DAG )

static

Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.

Definition at line 26808 of file X86ISelLowering.cpp.

References llvm::MVT::changeElementType(), getMaskNode(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), and llvm::ISD::VSELECT.

Referenced by lowerShuffleAsBlend().

getVectorShuffle()

SDValue getVectorShuffle ( SelectionDAG & DAG, EVT VT, const SDLoc & dl, SDValue V1, SDValue V2, ArrayRef< int > Mask )

static

Definition at line 5267 of file X86ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::V1.

Referenced by getUnpackh(), and getUnpackl().

getVShift()

SDValue getVShift ( bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG & DAG, const TargetLowering & TLI, const SDLoc & dl )

static

Return a vector logical shift node.

Definition at line 7676 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::is128BitVector(), and Opc.

getX86XALUOOp()

std::pair< SDValue, SDValue > getX86XALUOOp ( X86::CondCode & Cond, SDValue Op, SelectionDAG & DAG )

static

Definition at line 25318 of file X86ISelLowering.cpp.

References assert(), Cond, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_O, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isOneConstant(), LHS, llvm_unreachable, RHS, llvm::ISD::SADDO, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::ISD::UADDO, llvm::ISD::UMULO, and llvm::ISD::USUBO.

Referenced by LowerXALUO().

getZeroVector()

SDValue getZeroVector ( MVT VT, const X86Subtarget & Subtarget, SelectionDAG & DAG, const SDLoc & dl )

static

Returns a vector of specified type with all zero elements.

Definition at line 4328 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), and llvm::TargetLoweringBase::isTypeLegal().

growShuffleMask()

void growShuffleMask ( ArrayRef< int > SrcMask, SmallVectorImpl< int > & DstMask, unsigned SrcSizeInBits, unsigned DstSizeInBits )

static

Definition at line 4231 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ArrayRef< T >::end(), llvm::ArrayRef< T >::size(), and llvm::SM_SentinelUndef.

Referenced by combineX86ShuffleChainWithExtract().

hasBZHI()

bool hasBZHI ( const X86Subtarget & Subtarget, MVT VT )

static

Definition at line 52484 of file X86ISelLowering.cpp.

Referenced by combineAndLoadToBZHI().

hasFPCMov()

bool hasFPCMov ( unsigned X86CC )

static

Is there a floating point cmov for the specific X86 condition code?

Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 3289 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.

Referenced by combineCMov().

<