LLVM 24.0.0git
RISCVISelLowering.cpp
Go to the documentation of this file.
1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineJumpTableInfo.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SDPatternMatch.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsRISCV.h"
44#include "llvm/MC/MCCodeEmitter.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/InstructionCost.h"
50#include "llvm/Support/KnownBits.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "riscv-lower"
58
59STATISTIC(NumTailCalls, "Number of tail calls");
60
61static cl::opt<unsigned> ExtensionMaxWebSize(
62 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
63 cl::desc("Give the maximum size (in number of nodes) of the web of "
64 "instructions that we will consider for VW expansion"),
65 cl::init(18));
66
67static cl::opt<bool>
68 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
69 cl::desc("Allow the formation of VW_W operations (e.g., "
70 "VWADD_W) with splat constants"),
71 cl::init(false));
72
73static cl::opt<unsigned> NumRepeatedDivisors(
74 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
75 cl::desc("Set the minimum number of repetitions of a divisor to allow "
76 "transformation to multiplications by the reciprocal"),
77 cl::init(2));
78
79static cl::opt<int>
80 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
81 cl::desc("Give the maximum number of instructions that we will "
82 "use for creating a floating-point immediate value"),
83 cl::init(3));
84
85static cl::opt<bool>
86 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
87 cl::desc("Swap add and addi in cases where the add may "
88 "be combined with a shift"),
89 cl::init(true));
90
91static cl::opt<int> BrMergingBaseCostThresh(
92 "riscv-br-merging-base-cost", cl::init(2),
93 cl::desc(
94 "Sets the cost threshold for when multiple conditionals will be merged "
95 "into one branch versus be split in multiple branches. Merging "
96 "conditionals saves branches at the cost of additional instructions. "
97 "This value sets the instruction cost limit, below which conditionals "
98 "will be merged, and above which conditionals will be split. Set to -1 "
99 "to never merge branches."),
100 cl::Hidden);
101
102static cl::opt<int> BrMergingLikelyBias(
103 "riscv-br-merging-likely-bias", cl::init(0),
104 cl::desc(
105 "Increases 'riscv-br-merging-base-cost' in cases that it is "
106 "likely that all conditionals will be executed. For example for "
107 "merging the conditionals (a == b && c > d), if its known that "
108 "a == b is likely, then it is likely that if the conditionals are "
109 "split both sides will be executed, so it may be desirable to "
110 "increase the instruction cost threshold. Set to -1 to never merge "
111 "likely branches."),
112 cl::Hidden);
113
114static cl::opt<int> BrMergingUnlikelyBias(
115 "riscv-br-merging-unlikely-bias", cl::init(-1),
116 cl::desc(
117 "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
118 "that all conditionals will be executed. For example for merging "
119 "the conditionals (a == b && c > d), if its known that a == b is "
120 "unlikely, then it is unlikely that if the conditionals are split "
121 "both sides will be executed, so it may be desirable to decrease "
122 "the instruction cost threshold. Set to -1 to never merge unlikely "
123 "branches."),
124 cl::Hidden);
125
126// TODO: Support more ops
133
134RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
135 const RISCVSubtarget &STI)
136 : TargetLowering(TM, STI), Subtarget(STI) {
137
138 RISCVABI::ABI ABI = Subtarget.getTargetABI();
139 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
140
141 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
142 !Subtarget.hasStdExtF()) {
143 errs() << "Hard-float 'f' ABI can't be used for a target that "
144 "doesn't support the F instruction set extension (ignoring "
145 "target-abi)\n";
146 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
147 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
148 !Subtarget.hasStdExtD()) {
149 errs() << "Hard-float 'd' ABI can't be used for a target that "
150 "doesn't support the D instruction set extension (ignoring "
151 "target-abi)\n";
152 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
153 }
154
155 switch (ABI) {
156 default:
157 reportFatalUsageError("Don't know how to lower this ABI");
158 case RISCVABI::ABI_ILP32:
159 case RISCVABI::ABI_ILP32E:
160 case RISCVABI::ABI_LP64E:
161 case RISCVABI::ABI_ILP32F:
162 case RISCVABI::ABI_ILP32D:
163 case RISCVABI::ABI_LP64:
164 case RISCVABI::ABI_LP64F:
165 case RISCVABI::ABI_LP64D:
166 break;
167 }
168
169 MVT XLenVT = Subtarget.getXLenVT();
170
171 // Set up the register classes.
172 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
173
174 if (Subtarget.hasStdExtZfhmin())
175 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
176 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
177 addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
178 if (Subtarget.hasStdExtF())
179 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
180 if (Subtarget.hasStdExtD())
181 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
182 if (Subtarget.hasStdExtZhinxmin())
183 addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
184 if (Subtarget.hasStdExtZfinx())
185 addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
186 if (Subtarget.hasStdExtZdinx()) {
187 if (Subtarget.is64Bit())
188 addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
189 else
190 addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
191 }
192
193 static const MVT::SimpleValueType BoolVecVTs[] = {
194 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
195 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
196 static const MVT::SimpleValueType IntVecVTs[] = {
197 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
198 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
199 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
200 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
201 MVT::nxv4i64, MVT::nxv8i64};
202 static const MVT::SimpleValueType F16VecVTs[] = {
203 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
204 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
205 static const MVT::SimpleValueType BF16VecVTs[] = {
206 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
207 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
208 static const MVT::SimpleValueType F32VecVTs[] = {
209 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
210 static const MVT::SimpleValueType F64VecVTs[] = {
211 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
212 static const MVT::SimpleValueType VecTupleVTs[] = {
213 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
214 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
215 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
216 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
217 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
218 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
219 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
220 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
221 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
222 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
223 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
224
225 if (Subtarget.hasVInstructions()) {
226 auto addRegClassForRVV = [this](MVT VT) {
227 // Disable the smallest fractional LMUL types if ELEN is less than
228 // RVVBitsPerBlock.
229 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
230 if (VT.getVectorMinNumElements() < MinElts)
231 return;
232
233 unsigned Size = VT.getSizeInBits().getKnownMinValue();
234 const TargetRegisterClass *RC;
235 if (Size <= RISCV::RVVBitsPerBlock)
236 RC = &RISCV::VRRegClass;
237 else if (Size == 2 * RISCV::RVVBitsPerBlock)
238 RC = &RISCV::VRM2RegClass;
239 else if (Size == 4 * RISCV::RVVBitsPerBlock)
240 RC = &RISCV::VRM4RegClass;
241 else if (Size == 8 * RISCV::RVVBitsPerBlock)
242 RC = &RISCV::VRM8RegClass;
243 else
244 llvm_unreachable("Unexpected size");
245
246 addRegisterClass(VT, RC);
247 };
248
249 for (MVT VT : BoolVecVTs)
250 addRegClassForRVV(VT);
251 for (MVT VT : IntVecVTs) {
252 if (VT.getVectorElementType() == MVT::i64 &&
253 !Subtarget.hasVInstructionsI64())
254 continue;
255 addRegClassForRVV(VT);
256 }
257
258 if (Subtarget.hasVInstructionsF16Minimal() ||
259 Subtarget.hasVendorXAndesVPackFPH())
260 for (MVT VT : F16VecVTs)
261 addRegClassForRVV(VT);
262
263 if (Subtarget.hasVInstructionsBF16Minimal() ||
264 Subtarget.hasVendorXAndesVBFHCvt())
265 for (MVT VT : BF16VecVTs)
266 addRegClassForRVV(VT);
267
268 if (Subtarget.hasVInstructionsF32())
269 for (MVT VT : F32VecVTs)
270 addRegClassForRVV(VT);
271
272 if (Subtarget.hasVInstructionsF64())
273 for (MVT VT : F64VecVTs)
274 addRegClassForRVV(VT);
275
276 if (Subtarget.useRVVForFixedLengthVectors()) {
277 auto addRegClassForFixedVectors = [this](MVT VT) {
278 MVT ContainerVT = getContainerForFixedLengthVector(VT);
279 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
280 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
281 addRegisterClass(VT, TRI.getRegClass(RCID));
282 };
283 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
284 if (useRVVForFixedLengthVectorVT(VT))
285 addRegClassForFixedVectors(VT);
286
287 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
288 if (useRVVForFixedLengthVectorVT(VT))
289 addRegClassForFixedVectors(VT);
290 }
291
292 addRegisterClass(MVT::riscv_nxv1i8x2, &RISCV::VRN2M1RegClass);
293 addRegisterClass(MVT::riscv_nxv1i8x3, &RISCV::VRN3M1RegClass);
294 addRegisterClass(MVT::riscv_nxv1i8x4, &RISCV::VRN4M1RegClass);
295 addRegisterClass(MVT::riscv_nxv1i8x5, &RISCV::VRN5M1RegClass);
296 addRegisterClass(MVT::riscv_nxv1i8x6, &RISCV::VRN6M1RegClass);
297 addRegisterClass(MVT::riscv_nxv1i8x7, &RISCV::VRN7M1RegClass);
298 addRegisterClass(MVT::riscv_nxv1i8x8, &RISCV::VRN8M1RegClass);
299 addRegisterClass(MVT::riscv_nxv2i8x2, &RISCV::VRN2M1RegClass);
300 addRegisterClass(MVT::riscv_nxv2i8x3, &RISCV::VRN3M1RegClass);
301 addRegisterClass(MVT::riscv_nxv2i8x4, &RISCV::VRN4M1RegClass);
302 addRegisterClass(MVT::riscv_nxv2i8x5, &RISCV::VRN5M1RegClass);
303 addRegisterClass(MVT::riscv_nxv2i8x6, &RISCV::VRN6M1RegClass);
304 addRegisterClass(MVT::riscv_nxv2i8x7, &RISCV::VRN7M1RegClass);
305 addRegisterClass(MVT::riscv_nxv2i8x8, &RISCV::VRN8M1RegClass);
306 addRegisterClass(MVT::riscv_nxv4i8x2, &RISCV::VRN2M1RegClass);
307 addRegisterClass(MVT::riscv_nxv4i8x3, &RISCV::VRN3M1RegClass);
308 addRegisterClass(MVT::riscv_nxv4i8x4, &RISCV::VRN4M1RegClass);
309 addRegisterClass(MVT::riscv_nxv4i8x5, &RISCV::VRN5M1RegClass);
310 addRegisterClass(MVT::riscv_nxv4i8x6, &RISCV::VRN6M1RegClass);
311 addRegisterClass(MVT::riscv_nxv4i8x7, &RISCV::VRN7M1RegClass);
312 addRegisterClass(MVT::riscv_nxv4i8x8, &RISCV::VRN8M1RegClass);
313 addRegisterClass(MVT::riscv_nxv8i8x2, &RISCV::VRN2M1RegClass);
314 addRegisterClass(MVT::riscv_nxv8i8x3, &RISCV::VRN3M1RegClass);
315 addRegisterClass(MVT::riscv_nxv8i8x4, &RISCV::VRN4M1RegClass);
316 addRegisterClass(MVT::riscv_nxv8i8x5, &RISCV::VRN5M1RegClass);
317 addRegisterClass(MVT::riscv_nxv8i8x6, &RISCV::VRN6M1RegClass);
318 addRegisterClass(MVT::riscv_nxv8i8x7, &RISCV::VRN7M1RegClass);
319 addRegisterClass(MVT::riscv_nxv8i8x8, &RISCV::VRN8M1RegClass);
320 addRegisterClass(MVT::riscv_nxv16i8x2, &RISCV::VRN2M2RegClass);
321 addRegisterClass(MVT::riscv_nxv16i8x3, &RISCV::VRN3M2RegClass);
322 addRegisterClass(MVT::riscv_nxv16i8x4, &RISCV::VRN4M2RegClass);
323 addRegisterClass(MVT::riscv_nxv32i8x2, &RISCV::VRN2M4RegClass);
324 }
325
326 // fixed vector is stored in GPRs for P extension packed operations
327 if (Subtarget.hasStdExtP()) {
328 if (Subtarget.is64Bit()) {
329 addRegisterClass(MVT::v2i32, &RISCV::GPRRegClass);
330 addRegisterClass(MVT::v4i16, &RISCV::GPRRegClass);
331 addRegisterClass(MVT::v8i8, &RISCV::GPRRegClass);
332 } else {
333 addRegisterClass(MVT::v2i16, &RISCV::GPRRegClass);
334 addRegisterClass(MVT::v4i8, &RISCV::GPRRegClass);
335
336 addRegisterClass(MVT::v2i32, &RISCV::GPRPairRegClass);
337 addRegisterClass(MVT::v4i16, &RISCV::GPRPairRegClass);
338 addRegisterClass(MVT::v8i8, &RISCV::GPRPairRegClass);
339 }
340 }
341
342 // Compute derived properties from the register classes.
343 computeRegisterProperties(STI.getRegisterInfo());
344
345 setStackPointerRegisterToSaveRestore(RISCV::X2);
346
347 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
348 MVT::i1, Promote);
349 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
350 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
351 MVT::i1, Promote);
352
353 // TODO: add all necessary setOperationAction calls.
354 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Custom);
355
356 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
357 setOperationAction(ISD::BR_CC, XLenVT, Expand);
358 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
359 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
360
361 setCondCodeAction(ISD::SETGT, XLenVT, Custom);
362 setCondCodeAction(ISD::SETGE, XLenVT, Expand);
363 setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
364 setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
365 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
366 setCondCodeAction(ISD::SETULE, XLenVT, Expand);
367 setCondCodeAction(ISD::SETLE, XLenVT, Expand);
368 }
369
370 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
371
372 setOperationAction(ISD::VASTART, MVT::Other, Custom);
373 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
374
375 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
376 !Subtarget.hasVendorXAndesPerf())
377 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
378
379 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
380
381 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
382 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
383 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
384 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
385
386 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
387 setOperationAction(ISD::LOAD, MVT::i64, Custom);
388 setOperationAction(ISD::STORE, MVT::i64, Custom);
389 }
390
391 if (Subtarget.is64Bit()) {
392 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
393
394 setOperationAction(ISD::LOAD, MVT::i32, Custom);
395 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
396 MVT::i32, Custom);
397 setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
398 setOperationAction({ISD::SADDO, ISD::SSUBO}, MVT::i32, Custom);
399 } else if (Subtarget.hasStdExtP()) {
400 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
401 setOperationAction({ISD::ADD, ISD::SUB}, MVT::i64, Custom);
402 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
403 }
404 if (!Subtarget.hasStdExtZmmul()) {
405 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
406 } else if (Subtarget.is64Bit()) {
407 setOperationAction(ISD::MUL, MVT::i128, Custom);
408 setOperationAction(ISD::MUL, MVT::i32, Custom);
409 } else {
410 setOperationAction(ISD::MUL, MVT::i64, Custom);
411 }
412
413 if (!Subtarget.hasStdExtM()) {
414 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, XLenVT,
415 Expand);
416 } else if (Subtarget.is64Bit()) {
417 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
418 {MVT::i8, MVT::i16, MVT::i32}, Custom);
419 }
420
421 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, XLenVT, Expand);
422
423 // On RV32, the P extension has a WMUL(U) instruction we can use for
424 // (S/U)MUL_LOHI.
425 // FIXME: Does P imply Zmmul?
426 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
427 Subtarget.is64Bit())
428 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, Expand);
429
430 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
431 Custom);
432
433 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
434 if (Subtarget.is64Bit())
435 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
436 } else if (Subtarget.hasVendorXTHeadBb()) {
437 if (Subtarget.is64Bit())
438 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
439 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
440 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
441 setOperationAction(ISD::ROTL, XLenVT, Expand);
442 } else {
443 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
444 }
445
446 if (Subtarget.hasStdExtP())
447 setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Legal);
448
449 setOperationAction(ISD::BSWAP, XLenVT,
450 Subtarget.hasREV8Like() ? Legal : Expand);
451
452 if (Subtarget.hasREVLike()) {
453 setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
454 } else {
455 // Zbkb can use rev8+brev8 to implement bitreverse.
456 setOperationAction(ISD::BITREVERSE, XLenVT,
457 Subtarget.hasStdExtZbkb() ? Custom : Expand);
458 if (Subtarget.hasStdExtZbkb())
459 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
460 }
461
462 if (Subtarget.hasStdExtZbb() ||
463 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
464 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
465 Legal);
466 }
467
468 if (Subtarget.hasCTZLike()) {
469 if (Subtarget.is64Bit())
470 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
471 } else {
472 setOperationAction(ISD::CTTZ, XLenVT, Expand);
473 }
474
475 if (!Subtarget.hasCPOPLike()) {
476 // TODO: These should be set to LibCall, but this currently breaks
477 // the Linux kernel build. See #101786. Lacks i128 tests, too.
478 if (Subtarget.is64Bit())
479 setOperationAction(ISD::CTPOP, MVT::i128, Expand);
480 else
481 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
482 setOperationAction(ISD::CTPOP, MVT::i64, Expand);
483 }
484
485 if (Subtarget.hasCLZLike()) {
486 // We need the custom lowering to make sure that the resulting sequence
487 // for the 32bit case is efficient on 64bit targets.
488 // Use default promotion for i32 without Zbb.
489 if (Subtarget.is64Bit() &&
490 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
491 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
492 } else {
493 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
494 setOperationAction(ISD::CTLZ_ZERO_POISON, XLenVT, Legal);
495 setOperationAction(ISD::CTLZ, XLenVT, Expand);
496 }
497
498 if (Subtarget.hasStdExtP()) {
499 setOperationAction(ISD::CTLS, XLenVT, Legal);
500 if (Subtarget.is64Bit())
501 setOperationAction(ISD::CTLS, MVT::i32, Custom);
502 }
503
504 if (Subtarget.hasStdExtP() ||
505 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
506 setOperationAction(ISD::ABS, XLenVT, Legal);
507 if (Subtarget.is64Bit())
508 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
509 } else if (Subtarget.hasShortForwardBranchIALU()) {
510 // We can use PseudoCCSUB to implement ABS.
511 setOperationAction(ISD::ABS, XLenVT, Legal);
512 } else if (Subtarget.is64Bit()) {
513 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
514 }
515
516 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
517 setOperationAction(ISD::SELECT, XLenVT, Custom);
518
519 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
520 !Subtarget.is64Bit()) {
521 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
522 MVT::i32, Legal);
523 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
524 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
525 MVT::i32, Custom);
526 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
527 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
528 MVT::i32, Custom);
529 }
530
531 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
532 !Subtarget.is64Bit()) {
533 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
534 // pssha.w/psshl.w and extracting.
535 setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
536 setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
537 }
538
539 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
540 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
541 // paadd.w, paaddu.w and extracting.
542 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, MVT::i32, Legal);
543 }
544
545 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) {
546 setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Legal);
547 if (Subtarget.hasStdExtZbc())
548 setOperationAction(ISD::CLMULR, XLenVT, Legal);
549 } else if (Subtarget.hasStdExtZvbc() && Subtarget.is64Bit()) {
550 // FIXME: Support i32 on RV32 by zexting from XLEN to i64 and extracting
551 // half of the result (low for CLMUL, high for CLMULH).
552 // TODO: Zvbc32e allows us to do a lot more here.
553 setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Custom);
554 }
555
556 static const unsigned FPLegalNodeTypes[] = {
557 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
558 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
559 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
560 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
561 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
562 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
563 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
564
565 static const ISD::CondCode FPCCToExpand[] = {
566 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
567 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
568 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
569
570 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
571 ISD::FPOW};
572 static const unsigned FPOpToLibCall[] = {ISD::FREM};
573
574 static const unsigned FPRndMode[] = {
575 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
576 ISD::FROUNDEVEN};
577
578 static const unsigned ZfhminZfbfminPromoteOps[] = {
579 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUM,
580 ISD::FMAXIMUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM,
581 ISD::FADD, ISD::FSUB, ISD::FMUL,
582 ISD::FMA, ISD::FDIV, ISD::FSQRT,
583 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
584 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
585 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
586 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC,
587 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN,
588 ISD::FCANONICALIZE};
589
590 if (Subtarget.hasStdExtP()) {
591 static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
592 static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
593 ArrayRef<MVT> VTs;
594 if (Subtarget.is64Bit()) {
595 VTs = P64VecVTs;
596 // There's no instruction for vector shamt in P extension so we unroll to
597 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
598 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
599 // widen for those operations that will be unrolled.
600 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA},
601 {MVT::v2i16, MVT::v4i8}, Custom);
602 setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::v2i16, MVT::v4i8},
603 Custom);
604 // Operand legalization queries the action using the illegal subvector.
605 setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
606 Custom);
607 } else {
608 VTs = P32VecVTs;
609 }
610 // By default everything must be expanded.
611 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
612 setOperationAction(Op, VTs, Expand);
613
614 for (MVT VT : VTs) {
615 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
616 setTruncStoreAction(VT, OtherVT, Expand);
617 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
618 OtherVT, Expand);
619 }
620 }
621
622 setOperationAction({ISD::LOAD, ISD::STORE}, VTs, Legal);
623 setOperationAction({ISD::ADD, ISD::SUB}, VTs, Legal);
624 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VTs, Legal);
625 setOperationAction(ISD::UADDSAT, VTs, Legal);
626 setOperationAction(ISD::SADDSAT, VTs, Legal);
627 setOperationAction(ISD::USUBSAT, VTs, Legal);
628 setOperationAction(ISD::SSUBSAT, VTs, Legal);
629 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Legal);
630 setOperationAction(ISD::BITREVERSE, VTs, Legal);
631 setOperationAction(ISD::VECTOR_SHUFFLE, VTs, Custom);
632 setOperationAction(ISD::VECTOR_REVERSE, VTs, Legal);
633 for (MVT VT : VTs) {
634 if (VT != MVT::v2i32)
635 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Legal);
636 if (VT.getVectorElementType() != MVT::i8) {
637 setOperationAction(ISD::SSHLSAT, VT, Custom);
638 setOperationAction(ISD::BSWAP, VT, Legal);
639 }
640 }
641 setOperationAction({ISD::UNDEF, ISD::POISON}, VTs, Legal);
642 setOperationAction(ISD::SPLAT_VECTOR, VTs, Legal);
643 setOperationAction(ISD::BUILD_VECTOR, VTs, Legal);
644 setOperationAction(ISD::SCALAR_TO_VECTOR, VTs, Legal);
645 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Custom);
646 setOperationAction(ISD::BITCAST, VTs, Custom);
647 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
648 Custom);
649 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
650 Legal);
651 setOperationAction(ISD::SELECT, VTs, Custom);
652 setOperationAction(ISD::VSELECT, VTs, Legal);
653 setOperationAction(ISD::SETCC, VTs, Legal);
654 setCondCodeAction(
655 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
656 Expand);
657 setCondCodeAction({ISD::SETNE, ISD::SETGT}, VTs, Custom);
658
659 if (!Subtarget.is64Bit())
660 setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v4i8}, Custom);
661
662 // P extension vector comparisons produce all 1s for true, all 0s for false
663 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
664
665 if (!Subtarget.is64Bit()) {
666 // By default everything must be expanded.
667 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
668 setOperationAction(Op, P64VecVTs, Expand);
669
670 for (MVT VT : P64VecVTs) {
671 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
672 setTruncStoreAction(VT, OtherVT, Expand);
673 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
674 OtherVT, Expand);
675 }
676 }
677
678 setOperationAction({ISD::UNDEF, ISD::POISON}, P64VecVTs, Legal);
679 setOperationAction({ISD::LOAD, ISD::STORE}, P64VecVTs, Custom);
680 setOperationAction(ISD::BITCAST, P64VecVTs, Custom);
681 setOperationAction({ISD::ADD, ISD::SUB}, P64VecVTs, Legal);
682 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, {MVT::v4i16, MVT::v8i8},
683 Custom);
684 setOperationAction(
685 {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
686 Legal);
687 setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
688 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
689 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
690 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
691 P64VecVTs, Legal);
692 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU},
693 {MVT::v4i16, MVT::v8i8}, Legal);
694 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, P64VecVTs, Custom);
695 setOperationAction(ISD::SSHLSAT, {MVT::v2i32, MVT::v4i16}, Custom);
696 setOperationAction(ISD::BSWAP, MVT::v4i16, Legal);
697 setOperationAction(ISD::BITREVERSE, {MVT::v4i16, MVT::v8i8}, Legal);
698 setOperationAction(ISD::VECTOR_SHUFFLE, P64VecVTs, Custom);
699 setOperationAction(ISD::VECTOR_REVERSE, P64VecVTs, Custom);
700 setOperationAction(ISD::SPLAT_VECTOR, P64VecVTs, Legal);
701 setOperationAction(ISD::BUILD_VECTOR, P64VecVTs, Legal);
702 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Legal);
703 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
704 {MVT::v4i16, MVT::v8i8}, Custom);
705 setOperationAction(ISD::CONCAT_VECTORS, {MVT::v4i16, MVT::v8i8}, Legal);
706 setOperationAction(ISD::EXTRACT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
707 Legal);
708 setOperationAction({ISD::SELECT, ISD::VSELECT}, {MVT::v4i16, MVT::v8i8},
709 Custom);
710 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU},
711 {MVT::v4i16, MVT::v8i8}, Custom);
712 setOperationAction(ISD::MUL, P32VecVTs, Custom);
713 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v4i8, Custom);
714 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v2i16, Legal);
715 setOperationAction({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
716 {MVT::v4i16, MVT::v2i32}, Legal);
717 setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Legal);
718 setOperationAction(ISD::SETCC, P64VecVTs, Legal);
719 setCondCodeAction(
720 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
721 P64VecVTs, Expand);
722 setCondCodeAction({ISD::SETNE, ISD::SETGT}, P64VecVTs, Custom);
723 // Operation legalization queries the action using the result type.
724 setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v4i16, MVT::v8i8},
725 Custom);
726 } else {
727 setOperationAction(ISD::MUL, P64VecVTs, Legal);
728 setOperationAction({ISD::MULHS, ISD::MULHU}, {MVT::v2i32, MVT::v4i16},
729 Legal);
730 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v8i8, Custom);
731 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG,
732 {MVT::v4i16, MVT::v2i32}, Legal);
733 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, {MVT::v4i16, MVT::v2i32},
734 Custom);
735 setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Custom);
736 }
737 // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
738 setOperationAction(
739 ISD::SIGN_EXTEND_INREG,
740 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}, Legal);
741 }
742
743 if (Subtarget.hasStdExtZfbfmin()) {
744 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
745 setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
746 setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
747 setOperationAction(ISD::SELECT, MVT::bf16, Custom);
748 setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
749 setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
750 setOperationAction(ISD::FREM, MVT::bf16, Promote);
751 setOperationAction(ISD::FABS, MVT::bf16, Custom);
752 setOperationAction(ISD::FNEG, MVT::bf16, Custom);
753 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
754 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
755 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
756 }
757
758 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
759 if (Subtarget.hasStdExtZfhOrZhinx()) {
760 setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
761 setOperationAction(FPRndMode, MVT::f16,
762 Subtarget.hasStdExtZfa() ? Legal : Custom);
763 setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
764 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
765 Subtarget.hasStdExtZfa() ? Legal : Custom);
766 if (Subtarget.hasStdExtZfa())
767 setOperationAction(ISD::ConstantFP, MVT::f16, Custom);
768 } else {
769 setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
770 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Promote);
771 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
772 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
773 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
774 setOperationAction(Op, MVT::f16, Custom);
775 setOperationAction(ISD::FABS, MVT::f16, Custom);
776 setOperationAction(ISD::FNEG, MVT::f16, Custom);
777 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
778 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
779 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
780 }
781
782 if (!Subtarget.hasStdExtD()) {
783 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
784 // instruction runs into double rounding issues, so this is wrong.
785 // Normally we'd use an f64 fma, but without the D extension the f64 type
786 // is not legal. This should probably be a libcall.
787 AddPromotedToType(ISD::FMA, MVT::f16, MVT::f32);
788 AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f32);
789 }
790
791 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
792
793 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
794 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
795 setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
796 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
797 setOperationAction(ISD::SELECT, MVT::f16, Custom);
798 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
799
800 setOperationAction(
801 ISD::FNEARBYINT, MVT::f16,
802 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
803 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
804 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
805 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
806 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
807 MVT::f16, Promote);
808
809 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
810 // complete support for all operations in LegalizeDAG.
811 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
812 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
813 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
814 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
815 MVT::f16, Promote);
816
817 // We need to custom promote this.
818 if (Subtarget.is64Bit())
819 setOperationAction(ISD::FPOWI, MVT::i32, Custom);
820 }
821
822 if (Subtarget.hasStdExtFOrZfinx()) {
823 setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
824 setOperationAction(FPRndMode, MVT::f32,
825 Subtarget.hasStdExtZfa() ? Legal : Custom);
826 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
827 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
828 setOperationAction(ISD::SELECT, MVT::f32, Custom);
829 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
830 setOperationAction(FPOpToExpand, MVT::f32, Expand);
831 setOperationAction(FPOpToLibCall, MVT::f32, LibCall);
832 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
833 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
834 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
835 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
836 setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
837 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
838 setOperationAction(ISD::FP_TO_BF16, MVT::f32,
839 Subtarget.isSoftFPABI() ? LibCall : Custom);
840 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
841 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
842 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
843 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
844
845 if (Subtarget.hasStdExtZfa()) {
846 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
847 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
848 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
849 } else {
850 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
851 }
852 }
853
854 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
855 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
856
857 if (Subtarget.hasStdExtDOrZdinx()) {
858 setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
859
860 if (!Subtarget.is64Bit())
861 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
862
863 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
864 !Subtarget.is64Bit()) {
865 setOperationAction(ISD::LOAD, MVT::f64, Custom);
866 setOperationAction(ISD::STORE, MVT::f64, Custom);
867 }
868
869 if (Subtarget.hasStdExtZfa()) {
870 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
871 setOperationAction(FPRndMode, MVT::f64, Legal);
872 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
873 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
874 } else {
875 if (Subtarget.is64Bit())
876 setOperationAction(FPRndMode, MVT::f64, Custom);
877
878 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
879 }
880
881 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
882 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
883 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
884 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
885 setOperationAction(ISD::SELECT, MVT::f64, Custom);
886 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
887 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
888 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
889 setOperationAction(FPOpToExpand, MVT::f64, Expand);
890 setOperationAction(FPOpToLibCall, MVT::f64, LibCall);
891 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
892 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
893 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
894 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
895 setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
896 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
897 setOperationAction(ISD::FP_TO_BF16, MVT::f64,
898 Subtarget.isSoftFPABI() ? LibCall : Custom);
899 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
900 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
901 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
902 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
903 }
904
905 if (Subtarget.is64Bit()) {
906 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
907 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
908 MVT::i32, Custom);
909 setOperationAction(ISD::LROUND, MVT::i32, Custom);
910 }
911
912 if (Subtarget.hasStdExtFOrZfinx()) {
913 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
914 Custom);
915
916 // f16/bf16 require custom handling.
917 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, XLenVT,
918 Custom);
919 setOperationAction({ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, XLenVT,
920 Custom);
921
922 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
923 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
924 setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
925 setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
926 setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
927 setOperationAction(ISD::GET_FPMODE, XLenVT, Custom);
928 setOperationAction(ISD::SET_FPMODE, XLenVT, Custom);
929 setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
930 }
931
932 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
933 ISD::JumpTable},
934 XLenVT, Custom);
935
936 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
937
938 if (Subtarget.is64Bit())
939 setOperationAction(ISD::Constant, MVT::i64, Custom);
940
941 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
942 // Unfortunately this can't be determined just from the ISA naming string.
943 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
944 Subtarget.is64Bit() ? Legal : Custom);
945 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
946 Subtarget.is64Bit() ? Legal : Custom);
947
948 if (Subtarget.is64Bit()) {
949 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
950 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
951 }
952
953 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
954 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
955 if (Subtarget.is64Bit())
956 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
957
958 if (Subtarget.hasVendorXMIPSCBOP())
959 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
960 else
961 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
962
963 if (Subtarget.hasStdExtZalrsc()) {
964 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
965 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
966 setMinCmpXchgSizeInBits(8);
967 else
968 setMinCmpXchgSizeInBits(32);
969 } else if (Subtarget.hasForcedAtomics()) {
970 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
971 } else {
972 setMaxAtomicSizeInBitsSupported(0);
973 }
974
975 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
976
977 setBooleanContents(ZeroOrOneBooleanContent);
978
979 if (getTargetMachine().getTargetTriple().isOSLinux()) {
980 // Custom lowering of llvm.clear_cache.
981 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
982 }
983
984 if (Subtarget.hasVInstructions()) {
985 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
986
987 setOperationAction(ISD::VSCALE, XLenVT, Custom);
988
989 // RVV intrinsics may have illegal operands.
990 // We also need to custom legalize vmv.x.s.
991 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
992 ISD::INTRINSIC_VOID},
993 {MVT::i8, MVT::i16}, Custom);
994 if (Subtarget.is64Bit())
995 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
996 MVT::i32, Custom);
997 else
998 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
999 MVT::i64, Custom);
1000
1001 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
1002 MVT::Other, Custom);
1003
1004 static const unsigned IntegerVPOps[] = {
1005 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
1006 ISD::VP_UREM, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1007 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
1008 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
1009 ISD::VP_MERGE,
1010 ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
1011 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON};
1012
1013 static const unsigned FloatingPointVPOps[] = {
1014 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
1015 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
1016 ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
1017
1018 static const unsigned IntegerVecReduceOps[] = {
1019 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
1020 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
1021 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
1022
1023 static const unsigned FloatingPointVecReduceOps[] = {
1024 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
1025 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
1026
1027 static const unsigned FloatingPointLibCallOps[] = {
1028 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
1029 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
1030
1031 if (!Subtarget.is64Bit()) {
1032 // We must custom-lower certain vXi64 operations on RV32 due to the vector
1033 // element type being illegal.
1034 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1035 MVT::i64, Custom);
1036
1037 setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
1038
1039 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1040 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
1041 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
1042 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
1043 MVT::i64, Custom);
1044 }
1045
1046 for (MVT VT : BoolVecVTs) {
1047 if (!isTypeLegal(VT))
1048 continue;
1049
1050 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1051
1052 // Mask VTs are custom-expanded into a series of standard nodes
1053 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
1054 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1055 ISD::SCALAR_TO_VECTOR},
1056 VT, Custom);
1057
1058 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1059 Custom);
1060
1061 setOperationAction(ISD::SELECT, VT, Custom);
1062 setOperationAction({ISD::SELECT_CC, ISD::VSELECT}, VT,
1063 Expand);
1064 setOperationAction(ISD::VP_MERGE, VT, Custom);
1065
1066 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
1067 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
1068 VT, Custom);
1069
1070 setOperationAction(
1071 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1072 Custom);
1073
1074 setOperationAction(
1075 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1076 Custom);
1077
1078 // RVV has native int->float & float->int conversions where the
1079 // element type sizes are within one power-of-two of each other. Any
1080 // wider distances between type sizes have to be lowered as sequences
1081 // which progressively narrow the gap in stages.
1082 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1083 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1084 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1085 ISD::STRICT_FP_TO_UINT},
1086 VT, Custom);
1087 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1088 Custom);
1089
1090 // Expand all extending loads to types larger than this, and truncating
1091 // stores from types larger than this.
1092 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1093 setTruncStoreAction(VT, OtherVT, Expand);
1094 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1095 OtherVT, Expand);
1096 }
1097
1098 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1099 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1100
1101 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
1102
1103 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1104 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1105
1106 setOperationPromotedToType(
1107 {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
1108 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
1109 }
1110
1111 for (MVT VT : IntVecVTs) {
1112 if (!isTypeLegal(VT))
1113 continue;
1114
1115 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1116 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1117
1118 // Vectors implement MULHS/MULHU.
1119 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
1120
1121 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1122 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1123 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
1124
1125 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1126 Legal);
1127
1128 if (Subtarget.hasStdExtZvabd()) {
1129 setOperationAction(ISD::ABS, VT, Legal);
1130 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Legal);
1131 } else {
1132 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1133 }
1134
1135 // Custom-lower extensions and truncations from/to mask types.
1136 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1137 VT, Custom);
1138
1139 // RVV has native int->float & float->int conversions where the
1140 // element type sizes are within one power-of-two of each other. Any
1141 // wider distances between type sizes have to be lowered as sequences
1142 // which progressively narrow the gap in stages.
1143 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1144 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1145 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1146 ISD::STRICT_FP_TO_UINT},
1147 VT, Custom);
1148 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1149 Custom);
1150 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1151 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1152 ISD::SSUBSAT, ISD::USUBSAT},
1153 VT, Legal);
1154
1155 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1156 // nodes which truncate by one power of two at a time.
1157 setOperationAction(
1158 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1159 Custom);
1160
1161 // Custom-lower insert/extract operations to simplify patterns.
1162 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1163 Custom);
1164
1165 // Custom-lower reduction operations to set up the corresponding custom
1166 // nodes' operands.
1167 setOperationAction(IntegerVecReduceOps, VT, Custom);
1168
1169 setOperationAction(IntegerVPOps, VT, Custom);
1170
1171 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1172
1173 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1174 VT, Custom);
1175
1176 setOperationAction(
1177 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1178 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1179 VT, Custom);
1180 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1181
1182 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1183 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1184 VT, Custom);
1185
1186 setOperationAction(ISD::SELECT, VT, Custom);
1187 setOperationAction(ISD::SELECT_CC, VT, Expand);
1188
1189 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
1190
1191 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1192 setTruncStoreAction(VT, OtherVT, Expand);
1193 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1194 OtherVT, Expand);
1195 }
1196
1197 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1198 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1199
1200 setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1201 VT, Custom);
1202
1203 if (Subtarget.hasStdExtZvkb()) {
1204 setOperationAction(ISD::BSWAP, VT, Legal);
1205 } else {
1206 setOperationAction(ISD::BSWAP, VT, Expand);
1207 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
1208 }
1209
1210 if (Subtarget.hasStdExtZvbb()) {
1211 setOperationAction(ISD::BITREVERSE, VT, Legal);
1212 } else {
1213 setOperationAction(ISD::BITREVERSE, VT, Expand);
1214 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
1215
1216 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1217 // range of f32.
1218 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1219 if (isTypeLegal(FloatVT)) {
1220 setOperationAction(
1221 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1222 Custom);
1223 }
1224 }
1225
1226 if (VT.getVectorElementType() == MVT::i64) {
1227 if (Subtarget.hasStdExtZvbc())
1228 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1229 } else {
1230 if (Subtarget.hasStdExtZvbc32e()) {
1231 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1232 } else if (Subtarget.hasStdExtZvbc()) {
1233 // Promote to i64 if the lmul is small enough.
1234 // FIXME: Split if necessary to widen.
1235 // FIXME: Promote clmulh directly without legalizing to clmul first.
1236 MVT I64VecVT = MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
1237 if (isTypeLegal(I64VecVT))
1238 setOperationAction(ISD::CLMUL, VT, Custom);
1239 }
1240 }
1241
1242 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1243 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
1244 ISD::MASKED_SREM},
1245 VT, Legal);
1246 }
1247
1248 for (MVT VT : VecTupleVTs) {
1249 if (!isTypeLegal(VT))
1250 continue;
1251
1252 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1253 }
1254
1255 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1256 // but no other unordered comparisons, and supports all ordered comparisons
1257 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1258 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1259 // and we pattern-match those back to the "original", swapping operands once
1260 // more. This way we catch both operations and both "vf" and "fv" forms with
1261 // fewer patterns.
1262 static const ISD::CondCode VFPCCToExpand[] = {
1263 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1264 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1265 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1266 };
1267
1268 // TODO: support more ops.
1269 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1270 ISD::FMINNUM,
1271 ISD::FMAXNUM,
1272 ISD::FMINIMUMNUM,
1273 ISD::FMAXIMUMNUM,
1274 ISD::FADD,
1275 ISD::FSUB,
1276 ISD::FMUL,
1277 ISD::FMA,
1278 ISD::FDIV,
1279 ISD::FSQRT,
1280 ISD::FCEIL,
1281 ISD::FTRUNC,
1282 ISD::FFLOOR,
1283 ISD::FROUND,
1284 ISD::FROUNDEVEN,
1285 ISD::FRINT,
1286 ISD::FNEARBYINT,
1287 ISD::IS_FPCLASS,
1288 ISD::SETCC,
1289 ISD::FMAXIMUM,
1290 ISD::FMINIMUM,
1291 ISD::STRICT_FADD,
1292 ISD::STRICT_FSUB,
1293 ISD::STRICT_FMUL,
1294 ISD::STRICT_FDIV,
1295 ISD::STRICT_FSQRT,
1296 ISD::STRICT_FMA,
1297 ISD::VECREDUCE_FADD,
1298 ISD::VECREDUCE_FMIN,
1299 ISD::VECREDUCE_FMAX,
1300 ISD::VECREDUCE_FMINIMUM,
1301 ISD::VECREDUCE_FMAXIMUM,
1302 ISD::FCANONICALIZE};
1303
1304 // TODO: Make more of these ops legal.
1305 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1306 ISD::FSQRT,
1307 ISD::FCEIL,
1308 ISD::FTRUNC,
1309 ISD::FFLOOR,
1310 ISD::FROUND,
1311 ISD::FROUNDEVEN,
1312 ISD::FRINT,
1313 ISD::FNEARBYINT,
1314 ISD::STRICT_FDIV,
1315 ISD::STRICT_FSQRT,
1316 ISD::VECREDUCE_FADD,
1317 ISD::VECREDUCE_FMIN,
1318 ISD::VECREDUCE_FMAX,
1319 ISD::VECREDUCE_FMINIMUM,
1320 ISD::VECREDUCE_FMAXIMUM};
1321
1322 // TODO: support more vp ops.
1323 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1324 ISD::VP_REDUCE_FMIN,
1325 ISD::VP_REDUCE_FMAX,
1326 ISD::VP_REDUCE_FMINIMUM,
1327 ISD::VP_REDUCE_FMAXIMUM};
1328
1329 // Sets common operation actions on RVV floating-point vector types.
1330 const auto SetCommonVFPActions = [&](MVT VT) {
1331 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1332 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1333 // sizes are within one power-of-two of each other. Therefore conversions
1334 // between vXf16 and vXf64 must be lowered as sequences which convert via
1335 // vXf32.
1336 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1337 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1338 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1339 // Custom-lower insert/extract operations to simplify patterns.
1340 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1341 Custom);
1342 // Expand various condition codes (explained above).
1343 setCondCodeAction(VFPCCToExpand, VT, Expand);
1344
1345 setOperationAction(
1346 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1347 Legal);
1348 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1349
1350 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1351 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1352 ISD::IS_FPCLASS},
1353 VT, Custom);
1354
1355 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1356
1357 // Expand FP operations that need libcalls.
1358 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1359
1360 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1361
1362 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1363
1364 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1365
1366 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1367 VT, Custom);
1368
1369 setOperationAction(
1370 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1371 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1372 VT, Custom);
1373 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1374
1375 setOperationAction(ISD::SELECT, VT, Custom);
1376 setOperationAction(ISD::SELECT_CC, VT, Expand);
1377
1378 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1379 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1380 VT, Custom);
1381
1382 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1383 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1384
1385 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1386 ISD::VECTOR_SPLICE_RIGHT},
1387 VT, Custom);
1388 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1389 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1390
1391 setOperationAction(FloatingPointVPOps, VT, Custom);
1392
1393 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1394 Custom);
1395 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1396 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1397 VT, Legal);
1398 setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1399 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1400 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1401 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1402 VT, Custom);
1403
1404 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1405 };
1406
1407 // Sets common extload/truncstore actions on RVV floating-point vector
1408 // types.
1409 const auto SetCommonVFPExtLoadTruncStoreActions =
1410 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1411 for (auto SmallVT : SmallerVTs) {
1412 setTruncStoreAction(VT, SmallVT, Expand);
1413 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
1414 }
1415 };
1416
1417 // Sets common actions for f16 and bf16 for when there's only
1418 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1419 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1420 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1421 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1422 Custom);
1423 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1424 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1425 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1426 Custom);
1427 setOperationAction(ISD::SELECT_CC, VT, Expand);
1428 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1429 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1430 ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
1431 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1432 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1433 VT, Custom);
1434 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1435 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1436 MVT EltVT = VT.getVectorElementType();
1437 if (isTypeLegal(EltVT))
1438 setOperationAction({ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1439 VT, Custom);
1440 else
1441 setOperationAction(ISD::SPLAT_VECTOR, EltVT, Custom);
1442 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1443 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1444 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1445 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1446 ISD::VP_SCATTER},
1447 VT, Custom);
1448 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1449
1450 setOperationAction(ISD::FNEG, VT, Expand);
1451 setOperationAction(ISD::FABS, VT, Expand);
1452 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1453
1454 // Expand FP operations that need libcalls.
1455 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1456
1457 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1458
1459 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1460 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1461 setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
1462 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1463 } else {
1464 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1465 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1466 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1467 }
1468 };
1469
1470 // Sets common actions for zvfbfa, some of instructions are supported
1471 // natively so that we don't need to promote them.
1472 const auto SetZvfbfaActions = [&](MVT VT) {
1473 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1474 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1475 Custom);
1476 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1477 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1478 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1479 Custom);
1480 setOperationAction(ISD::SELECT_CC, VT, Expand);
1481 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1482 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1483 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
1484 ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
1485 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1486 ISD::VECTOR_COMPRESS},
1487 VT, Custom);
1488 setOperationAction(
1489 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1490 Legal);
1491 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1492 setOperationAction(ISD::IS_FPCLASS, VT, Custom);
1493 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1494 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1495
1496 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1497 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1498 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1499 ISD::STRICT_FMA},
1500 VT, Legal);
1501 setCondCodeAction(VFPCCToExpand, VT, Expand);
1502
1503 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1504 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1505 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1506 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1507 ISD::VP_SCATTER},
1508 VT, Custom);
1509 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1510
1511 // Expand FP operations that need libcalls.
1512 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1513
1514 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1515
1516 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1517 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1518 setOperationAction(ZvfbfaPromoteOps, VT, Custom);
1519 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1520 } else {
1521 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1522 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1523 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1524 }
1525 };
1526
1527 if (Subtarget.hasVInstructionsF16()) {
1528 for (MVT VT : F16VecVTs) {
1529 if (!isTypeLegal(VT))
1530 continue;
1531 SetCommonVFPActions(VT);
1532 }
1533 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1534 for (MVT VT : F16VecVTs) {
1535 if (!isTypeLegal(VT))
1536 continue;
1537 SetCommonPromoteToF32Actions(VT);
1538 }
1539 }
1540
1541 if (Subtarget.hasVInstructionsBF16()) {
1542 for (MVT VT : BF16VecVTs) {
1543 if (!isTypeLegal(VT))
1544 continue;
1545 SetZvfbfaActions(VT);
1546 }
1547 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1548 for (MVT VT : BF16VecVTs) {
1549 if (!isTypeLegal(VT))
1550 continue;
1551 SetCommonPromoteToF32Actions(VT);
1552 }
1553 }
1554
1555 if (Subtarget.hasStdExtZvfofp8min()) {
1556 for (MVT VT : BF16VecVTs) {
1557 if (!isTypeLegal(VT))
1558 continue;
1559 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
1560 }
1561 }
1562
1563 if (Subtarget.hasVInstructionsF32()) {
1564 for (MVT VT : F32VecVTs) {
1565 if (!isTypeLegal(VT))
1566 continue;
1567 SetCommonVFPActions(VT);
1568 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1569 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1570 }
1571 }
1572
1573 if (Subtarget.hasVInstructionsF64()) {
1574 for (MVT VT : F64VecVTs) {
1575 if (!isTypeLegal(VT))
1576 continue;
1577 SetCommonVFPActions(VT);
1578 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1579 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1580 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1581 }
1582 }
1583
1584 if (Subtarget.useRVVForFixedLengthVectors()) {
1585 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1586 if (!useRVVForFixedLengthVectorVT(VT))
1587 continue;
1588
1589 // By default everything must be expanded.
1590 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1591 setOperationAction(Op, VT, Expand);
1592 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1593 setTruncStoreAction(VT, OtherVT, Expand);
1594 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1595 OtherVT, Expand);
1596 }
1597
1598 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1599 // expansion to a build_vector of 0s.
1600 setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
1601
1602 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1603 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1604 Custom);
1605
1606 setOperationAction(
1607 {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1608 Custom);
1609
1610 setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1611 VT, Custom);
1612
1613 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1614 VT, Custom);
1615
1616 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1617
1618 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1619
1620 setOperationAction(ISD::SETCC, VT, Custom);
1621
1622 setOperationAction(ISD::SELECT, VT, Custom);
1623
1624 setOperationAction(
1625 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1626 Custom);
1627
1628 setOperationAction(ISD::BITCAST, VT, Custom);
1629
1630 setOperationAction(
1631 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1632 Custom);
1633
1634 setOperationAction(
1635 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1636 Custom);
1637
1638 setOperationAction(
1639 {
1640 ISD::SINT_TO_FP,
1641 ISD::UINT_TO_FP,
1642 ISD::FP_TO_SINT,
1643 ISD::FP_TO_UINT,
1644 ISD::STRICT_SINT_TO_FP,
1645 ISD::STRICT_UINT_TO_FP,
1646 ISD::STRICT_FP_TO_SINT,
1647 ISD::STRICT_FP_TO_UINT,
1648 },
1649 VT, Custom);
1650 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1651 Custom);
1652
1653 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1654
1655 // Operations below are different for between masks and other vectors.
1656 if (VT.getVectorElementType() == MVT::i1) {
1657 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Custom);
1658
1659 setOperationAction(ISD::VP_MERGE, VT, Custom);
1660
1661 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1662 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1663
1664 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1665 Custom);
1666 continue;
1667 }
1668
1669 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1670 // it before type legalization for i64 vectors on RV32. It will then be
1671 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1672 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1673 // improvements first.
1674 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1675 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1676 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1677
1678 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1679 setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
1680 }
1681
1682 setOperationAction(
1683 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1684
1685 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1686 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1687 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1688 ISD::VP_SCATTER},
1689 VT, Custom);
1690 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1691
1692 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1693 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1694 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1695 VT, Custom);
1696
1697 setOperationAction(
1698 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
1699
1700 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1701
1702 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1703 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1704 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
1705
1706 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1707 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1708 ISD::SSUBSAT, ISD::USUBSAT},
1709 VT, Custom);
1710
1711 setOperationAction(ISD::VSELECT, VT, Custom);
1712
1713 setOperationAction(
1714 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
1715
1716 // Custom-lower reduction operations to set up the corresponding custom
1717 // nodes' operands.
1718 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1719 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1720 ISD::VECREDUCE_UMIN},
1721 VT, Custom);
1722
1723 setOperationAction(IntegerVPOps, VT, Custom);
1724
1725 if (Subtarget.hasStdExtZvkb())
1726 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
1727
1728 if (Subtarget.hasStdExtZvbb()) {
1729 setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
1730 ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
1731 VT, Custom);
1732 } else {
1733 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1734 // range of f32.
1735 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1736 if (isTypeLegal(FloatVT))
1737 setOperationAction(
1738 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1739 Custom);
1740 }
1741
1742 if (VT.getVectorElementType() == MVT::i64) {
1743 if (Subtarget.hasStdExtZvbc())
1744 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
1745 } else {
1746 if (Subtarget.hasStdExtZvbc32e()) {
1747 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
1748 } else if (Subtarget.hasStdExtZvbc()) {
1749 // Promote to i64 as is done for scalable vectors.
1750 MVT I64VecVT =
1751 MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
1752 if (I64VecVT.isValid() && useRVVForFixedLengthVectorVT(I64VecVT))
1753 setOperationAction(ISD::CLMUL, VT, Custom);
1754 }
1755 }
1756
1757 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1758 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV,
1759 ISD::MASKED_UREM, ISD::MASKED_SREM},
1760 VT, Custom);
1761 }
1762
1763 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1764 // There are no extending loads or truncating stores.
1765 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1766 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1767 setTruncStoreAction(VT, InnerVT, Expand);
1768 }
1769
1770 if (!useRVVForFixedLengthVectorVT(VT))
1771 continue;
1772
1773 // By default everything must be expanded.
1774 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1775 setOperationAction(Op, VT, Expand);
1776
1777 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1778 // expansion to a build_vector of 0s.
1779 setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
1780
1781 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1782 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1783 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1784 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1785 VT, Custom);
1786 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1787 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1788
1789 setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1790 VT, Custom);
1791
1792 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1793 ISD::MGATHER, ISD::MSCATTER},
1794 VT, Custom);
1795 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1796 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1797 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1798 VT, Custom);
1799 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1800
1801 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1802 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1803 Custom);
1804
1805 setOperationAction(ISD::BITCAST, VT, Custom);
1806
1807 if (VT.getVectorElementType() == MVT::f16 &&
1808 !Subtarget.hasVInstructionsF16()) {
1809 setOperationAction(
1810 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1811 Custom);
1812 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1813 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1814 if (Subtarget.hasStdExtZfhmin()) {
1815 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1816 } else {
1817 // We need to custom legalize f16 build vectors if Zfhmin isn't
1818 // available.
1819 setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
1820 }
1821 setOperationAction(ISD::FNEG, VT, Expand);
1822 setOperationAction(ISD::FABS, VT, Expand);
1823 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1824 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1825 // Don't promote f16 vector operations to f32 if f32 vector type is
1826 // not legal.
1827 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1828 // TODO: Support more operations.
1829 if (!isTypeLegal(F32VecVT)) {
1830 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1831 ISD::VECREDUCE_FMAXIMUM,
1832 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1833 VT, Custom);
1834 setOperationAction(ISD::SETCC, VT, Custom);
1835 continue;
1836 }
1837 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1838 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1839 continue;
1840 }
1841
1842 if (VT.getVectorElementType() == MVT::bf16) {
1843 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1844 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1845 if (Subtarget.hasStdExtZvfofp8min())
1846 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
1847 if (Subtarget.hasStdExtZfbfmin()) {
1848 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1849 } else {
1850 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1851 // available.
1852 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
1853 }
1854 if (Subtarget.hasVInstructionsBF16()) {
1855 setOperationAction(ZvfbfaOps, VT, Custom);
1856 setCondCodeAction(VFPCCToExpand, VT, Expand);
1857 }
1858 setOperationAction(
1859 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1860 Custom);
1861 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1862 // Don't promote bf16 vector operations to f32 if f32 vector type is
1863 // not legal.
1864 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1865 // TODO: Support more operations.
1866 if (!isTypeLegal(F32VecVT)) {
1867 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1868 ISD::VECREDUCE_FMAXIMUM,
1869 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1870 VT, Custom);
1871 setOperationAction(ISD::SETCC, VT, Custom);
1872 continue;
1873 }
1874
1875 if (Subtarget.hasVInstructionsBF16())
1876 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1877 else
1878 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1879 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1880 continue;
1881 }
1882
1883 setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1884 Custom);
1885
1886 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1887 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1888 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1889 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1890 ISD::FMAXIMUM, ISD::FMINIMUM},
1891 VT, Custom);
1892
1893 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1894 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1895 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1896 ISD::FNEARBYINT, ISD::FCANONICALIZE},
1897 VT, Custom);
1898
1899 setCondCodeAction(VFPCCToExpand, VT, Expand);
1900
1901 setOperationAction(ISD::SETCC, VT, Custom);
1902 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1903
1904 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1905
1906 setOperationAction(FloatingPointVPOps, VT, Custom);
1907
1908 setOperationAction(
1909 {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1910 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1911 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1912 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1913 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1914 VT, Custom);
1915 }
1916
1917 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1918 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
1919 if (Subtarget.is64Bit())
1920 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1921 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1922 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1923 if (Subtarget.hasStdExtZfbfmin())
1924 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
1925 if (Subtarget.hasStdExtFOrZfinx())
1926 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1927 if (Subtarget.hasStdExtDOrZdinx())
1928 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1929 }
1930 }
1931
1932 if (Subtarget.hasStdExtZaamo())
1933 setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
1934
1935 if (Subtarget.hasForcedAtomics()) {
1936 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1937 setOperationAction(
1938 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1939 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1940 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1941 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1942 XLenVT, LibCall);
1943 }
1944
1945 if (Subtarget.hasVendorXTHeadMemIdx()) {
1946 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1947 setIndexedLoadAction(im, MVT::i8, Legal);
1948 setIndexedStoreAction(im, MVT::i8, Legal);
1949 setIndexedLoadAction(im, MVT::i16, Legal);
1950 setIndexedStoreAction(im, MVT::i16, Legal);
1951 setIndexedLoadAction(im, MVT::i32, Legal);
1952 setIndexedStoreAction(im, MVT::i32, Legal);
1953
1954 if (Subtarget.is64Bit()) {
1955 setIndexedLoadAction(im, MVT::i64, Legal);
1956 setIndexedStoreAction(im, MVT::i64, Legal);
1957 }
1958 }
1959 }
1960
1961 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1962 setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
1963 setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
1964 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1965
1966 setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
1967 setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
1968 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1969 }
1970
1971 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1972 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1973 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1974 ISD::PARTIAL_REDUCE_UMLA,
1975 ISD::PARTIAL_REDUCE_SUMLA};
1976 setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
1977 setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
1978 setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
1979 setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
1980 setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
1981
1982 // An i64 accumulator is handled by performing an i32 vdot4a* and widening
1983 // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
1984 setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom);
1985 setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
1986 setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom);
1987 setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom);
1988
1989 if (Subtarget.useRVVForFixedLengthVectors()) {
1990 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1991 if ((VT.getVectorElementType() != MVT::i32 &&
1992 VT.getVectorElementType() != MVT::i64) ||
1993 !useRVVForFixedLengthVectorVT(VT))
1994 continue;
1995 ElementCount EC = VT.getVectorElementCount();
1996 unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
1997 MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(Scale));
1998 setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
1999 }
2000 }
2001 }
2002
2003 // Customize load and store operation for bf16 if zfh isn't enabled.
2004 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
2005 setOperationAction(ISD::LOAD, MVT::bf16, Custom);
2006 setOperationAction(ISD::STORE, MVT::bf16, Custom);
2007 }
2008
2009 // Function alignments.
2010 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
2011 setMinFunctionAlignment(FunctionAlignment);
2012 // Set preferred alignments.
2013 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
2014 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
2015
2016 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
2017 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
2018 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
2019 ISD::SRA});
2020 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
2021
2022 if (Subtarget.hasStdExtFOrZfinx())
2023 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
2024
2025 // Allow scalar min/max to be combined with vector reductions.
2026 if (Subtarget.hasVInstructions())
2027 setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
2028 if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2029 setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
2030
2031 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
2032 Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2033 setTargetDAGCombine(ISD::TRUNCATE);
2034
2035 if (Subtarget.hasStdExtZbkb())
2036 setTargetDAGCombine(ISD::BITREVERSE);
2037
2038 if (Subtarget.hasStdExtFOrZfinx())
2039 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2040 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
2041 if (Subtarget.hasVInstructions())
2042 setTargetDAGCombine({ISD::FCOPYSIGN,
2043 ISD::VECTOR_MATCH,
2044 ISD::MGATHER,
2045 ISD::MSCATTER,
2046 ISD::MLOAD,
2047 ISD::VP_GATHER,
2048 ISD::VP_SCATTER,
2049 ISD::SRL,
2050 ISD::SHL,
2051 ISD::STORE,
2052 ISD::SPLAT_VECTOR,
2053 ISD::BUILD_VECTOR,
2054 ISD::CONCAT_VECTORS,
2055 ISD::VP_STORE,
2056 ISD::EXPERIMENTAL_VP_REVERSE,
2057 ISD::SDIV,
2058 ISD::UDIV,
2059 ISD::SREM,
2060 ISD::UREM,
2061 ISD::INSERT_VECTOR_ELT,
2062 ISD::ABS,
2063 ISD::ABS_MIN_POISON,
2064 ISD::CTPOP,
2065 ISD::VECTOR_SHUFFLE,
2066 ISD::FMA,
2067 ISD::VSELECT,
2068 ISD::VECREDUCE_ADD,
2069 ISD::VECTOR_SPLICE_RIGHT});
2070
2071 if (Subtarget.hasVendorXTHeadMemPair())
2072 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
2073 if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
2074 setTargetDAGCombine(ISD::BITCAST);
2075
2076 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2077
2078 setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2079
2080 setJumpIsExpensive(Subtarget.isJumpExpensive());
2081
2082 // Disable strict node mutation.
2083 IsStrictFPEnabled = true;
2084 EnableExtLdPromotion = true;
2085
2086 // Let the subtarget decide if a predictable select is more expensive than the
2087 // corresponding branch. This information is used in CGP/SelectOpt to decide
2088 // when to convert selects into branches.
2089 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
2090
2091 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
2092 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
2093
2094 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
2095 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
2096 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
2097
2098 MaxStoresPerMemmoveOptSize =
2099 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
2100 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
2101
2102 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
2103 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
2104}
2105
2106TargetLoweringBase::LegalizeTypeAction
2107RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
2108 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
2109 if (VT == MVT::v2i16 || VT == MVT::v4i8)
2110 return TypeWidenVector;
2111
2112 return TargetLoweringBase::getPreferredVectorAction(VT);
2113}
2114
2115EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
2116 LLVMContext &Context,
2117 EVT VT) const {
2118 if (!VT.isVector())
2119 return getPointerTy(DL);
2120 if (Subtarget.hasVInstructions() &&
2121 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
2122 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
2123 return VT.changeVectorElementTypeToInteger();
2124}
2125
2126TargetLoweringBase::CondMergingParams
2127RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
2128 const Value *LHS,
2129 const Value *RHS,
2130 const Function *F) const {
2131 if (F->hasOptSize())
2132 return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
2133
2134 // Merging conditions eliminates a branch, so the budget we are willing to
2135 // spend eagerly computing the RHS condition should scale with how expensive a
2136 // mispredicted branch is. A branch only costs the full penalty when actually
2137 // mispredicted, so scale it down by an assumed misprediction rate (~25%).
2138 int BaseCost = Subtarget.getMispredictionPenalty() / 4;
2139 if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
2140 BaseCost = BrMergingBaseCostThresh;
2141
2142 return {BaseCost, BrMergingLikelyBias, BrMergingUnlikelyBias};
2143}
2144
2145MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
2146 return Subtarget.getXLenVT();
2147}
2148
2149// Return false if we can lower get_vector_length to a vsetvli intrinsic.
2150bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
2151 unsigned VF,
2152 bool IsScalable) const {
2153 if (!Subtarget.hasVInstructions())
2154 return true;
2155
2156 if (!IsScalable)
2157 return true;
2158
2159 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2160 return true;
2161
2162 // Don't allow VF=1 if those types are't legal.
2163 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2164 return true;
2165
2166 // VLEN=32 support is incomplete.
2167 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2168 return true;
2169
2170 // The maximum VF is for the smallest element width with LMUL=8.
2171 // VF must be a power of 2.
2172 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2173 return VF > MaxVF || !isPowerOf2_32(VF);
2174}
2175
2176void RISCVTargetLowering::getTgtMemIntrinsic(
2177 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2178 MachineFunction &MF, unsigned Intrinsic) const {
2179 IntrinsicInfo Info;
2180 auto &DL = I.getDataLayout();
2181
2182 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2183 bool IsUnitStrided, bool UsePtrVal = false) {
2184 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2185 // We can't use ptrVal if the intrinsic can access memory before the
2186 // pointer. This means we can't use it for strided or indexed intrinsics.
2187 if (UsePtrVal)
2188 Info.ptrVal = I.getArgOperand(PtrOp);
2189 else
2190 Info.fallbackAddressSpace =
2191 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
2192 Type *MemTy;
2193 if (IsStore) {
2194 // Store value is the first operand.
2195 MemTy = I.getArgOperand(0)->getType();
2196 } else {
2197 // Use return type. If it's segment load, return type is a struct.
2198 MemTy = I.getType();
2199 if (MemTy->isStructTy())
2200 MemTy = MemTy->getStructElementType(0);
2201 }
2202 if (!IsUnitStrided)
2203 MemTy = MemTy->getScalarType();
2204
2205 Info.memVT = getValueType(DL, MemTy);
2206 if (MemTy->isTargetExtTy()) {
2207 // RISC-V vector tuple type's alignment type should be its element type.
2208 if (cast<TargetExtType>(MemTy)->getName() == "riscv.vector.tuple")
2209 MemTy = Type::getIntNTy(
2210 MemTy->getContext(),
2211 1 << cast<ConstantInt>(I.getArgOperand(I.arg_size() - 1))
2212 ->getZExtValue());
2213 Info.align = DL.getABITypeAlign(MemTy);
2214 } else {
2215 Info.align = Align(DL.getTypeStoreSize(MemTy->getScalarType()));
2216 }
2217 Info.size = MemoryLocation::UnknownSize;
2218 Info.flags |=
2219 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2220 Infos.push_back(Info);
2221 };
2222
2223 if (I.hasMetadata(LLVMContext::MD_nontemporal))
2224 Info.flags |= MachineMemOperand::MONonTemporal;
2225
2226 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2227 switch (Intrinsic) {
2228 default:
2229 return;
2230 case Intrinsic::riscv_masked_atomicrmw_xchg:
2231 case Intrinsic::riscv_masked_atomicrmw_add:
2232 case Intrinsic::riscv_masked_atomicrmw_sub:
2233 case Intrinsic::riscv_masked_atomicrmw_nand:
2234 case Intrinsic::riscv_masked_atomicrmw_max:
2235 case Intrinsic::riscv_masked_atomicrmw_min:
2236 case Intrinsic::riscv_masked_atomicrmw_umax:
2237 case Intrinsic::riscv_masked_atomicrmw_umin:
2238 case Intrinsic::riscv_masked_cmpxchg:
2239 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2240 // narrow atomic operation. These will be expanded to an LR/SC loop that
2241 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2242 // will be used to modify the appropriate part of the 4 byte data and
2243 // preserve the rest.
2244 Info.opc = ISD::INTRINSIC_W_CHAIN;
2245 Info.memVT = MVT::i32;
2246 Info.ptrVal = I.getArgOperand(0);
2247 Info.offset = 0;
2248 Info.align = Align(4);
2249 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2250 MachineMemOperand::MOVolatile;
2251 Infos.push_back(Info);
2252 return;
2253 case Intrinsic::riscv_seg2_load_mask:
2254 case Intrinsic::riscv_seg3_load_mask:
2255 case Intrinsic::riscv_seg4_load_mask:
2256 case Intrinsic::riscv_seg5_load_mask:
2257 case Intrinsic::riscv_seg6_load_mask:
2258 case Intrinsic::riscv_seg7_load_mask:
2259 case Intrinsic::riscv_seg8_load_mask:
2260 case Intrinsic::riscv_sseg2_load_mask:
2261 case Intrinsic::riscv_sseg3_load_mask:
2262 case Intrinsic::riscv_sseg4_load_mask:
2263 case Intrinsic::riscv_sseg5_load_mask:
2264 case Intrinsic::riscv_sseg6_load_mask:
2265 case Intrinsic::riscv_sseg7_load_mask:
2266 case Intrinsic::riscv_sseg8_load_mask:
2267 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2268 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2269 return;
2270 case Intrinsic::riscv_seg2_store_mask:
2271 case Intrinsic::riscv_seg3_store_mask:
2272 case Intrinsic::riscv_seg4_store_mask:
2273 case Intrinsic::riscv_seg5_store_mask:
2274 case Intrinsic::riscv_seg6_store_mask:
2275 case Intrinsic::riscv_seg7_store_mask:
2276 case Intrinsic::riscv_seg8_store_mask:
2277 // Operands are (vec, ..., vec, ptr, mask, vl)
2278 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2279 /*IsStore*/ true,
2280 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2281 return;
2282 case Intrinsic::riscv_sseg2_store_mask:
2283 case Intrinsic::riscv_sseg3_store_mask:
2284 case Intrinsic::riscv_sseg4_store_mask:
2285 case Intrinsic::riscv_sseg5_store_mask:
2286 case Intrinsic::riscv_sseg6_store_mask:
2287 case Intrinsic::riscv_sseg7_store_mask:
2288 case Intrinsic::riscv_sseg8_store_mask:
2289 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2290 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2291 /*IsStore*/ true,
2292 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2293 return;
2294 case Intrinsic::riscv_vlm:
2295 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2296 /*IsStore*/ false,
2297 /*IsUnitStrided*/ true,
2298 /*UsePtrVal*/ true);
2299 return;
2300 case Intrinsic::riscv_vle:
2301 case Intrinsic::riscv_vle_mask:
2302 case Intrinsic::riscv_vleff:
2303 case Intrinsic::riscv_vleff_mask:
2304 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2305 /*IsStore*/ false,
2306 /*IsUnitStrided*/ true,
2307 /*UsePtrVal*/ true);
2308 return;
2309 case Intrinsic::riscv_vsm:
2310 case Intrinsic::riscv_vse:
2311 case Intrinsic::riscv_vse_mask:
2312 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2313 /*IsStore*/ true,
2314 /*IsUnitStrided*/ true,
2315 /*UsePtrVal*/ true);
2316 return;
2317 case Intrinsic::riscv_vlse:
2318 case Intrinsic::riscv_vlse_mask:
2319 case Intrinsic::riscv_vloxei:
2320 case Intrinsic::riscv_vloxei_mask:
2321 case Intrinsic::riscv_vluxei:
2322 case Intrinsic::riscv_vluxei_mask:
2323 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2324 /*IsStore*/ false,
2325 /*IsUnitStrided*/ false);
2326 return;
2327 case Intrinsic::riscv_vsse:
2328 case Intrinsic::riscv_vsse_mask:
2329 case Intrinsic::riscv_vsoxei:
2330 case Intrinsic::riscv_vsoxei_mask:
2331 case Intrinsic::riscv_vsuxei:
2332 case Intrinsic::riscv_vsuxei_mask:
2333 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2334 /*IsStore*/ true,
2335 /*IsUnitStrided*/ false);
2336 return;
2337 case Intrinsic::riscv_vlseg2:
2338 case Intrinsic::riscv_vlseg3:
2339 case Intrinsic::riscv_vlseg4:
2340 case Intrinsic::riscv_vlseg5:
2341 case Intrinsic::riscv_vlseg6:
2342 case Intrinsic::riscv_vlseg7:
2343 case Intrinsic::riscv_vlseg8:
2344 case Intrinsic::riscv_vlseg2ff:
2345 case Intrinsic::riscv_vlseg3ff:
2346 case Intrinsic::riscv_vlseg4ff:
2347 case Intrinsic::riscv_vlseg5ff:
2348 case Intrinsic::riscv_vlseg6ff:
2349 case Intrinsic::riscv_vlseg7ff:
2350 case Intrinsic::riscv_vlseg8ff:
2351 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2352 /*IsStore*/ false,
2353 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2354 return;
2355 case Intrinsic::riscv_vlseg2_mask:
2356 case Intrinsic::riscv_vlseg3_mask:
2357 case Intrinsic::riscv_vlseg4_mask:
2358 case Intrinsic::riscv_vlseg5_mask:
2359 case Intrinsic::riscv_vlseg6_mask:
2360 case Intrinsic::riscv_vlseg7_mask:
2361 case Intrinsic::riscv_vlseg8_mask:
2362 case Intrinsic::riscv_vlseg2ff_mask:
2363 case Intrinsic::riscv_vlseg3ff_mask:
2364 case Intrinsic::riscv_vlseg4ff_mask:
2365 case Intrinsic::riscv_vlseg5ff_mask:
2366 case Intrinsic::riscv_vlseg6ff_mask:
2367 case Intrinsic::riscv_vlseg7ff_mask:
2368 case Intrinsic::riscv_vlseg8ff_mask:
2369 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2370 /*IsStore*/ false,
2371 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2372 return;
2373 case Intrinsic::riscv_vlsseg2:
2374 case Intrinsic::riscv_vlsseg3:
2375 case Intrinsic::riscv_vlsseg4:
2376 case Intrinsic::riscv_vlsseg5:
2377 case Intrinsic::riscv_vlsseg6:
2378 case Intrinsic::riscv_vlsseg7:
2379 case Intrinsic::riscv_vlsseg8:
2380 case Intrinsic::riscv_vloxseg2:
2381 case Intrinsic::riscv_vloxseg3:
2382 case Intrinsic::riscv_vloxseg4:
2383 case Intrinsic::riscv_vloxseg5:
2384 case Intrinsic::riscv_vloxseg6:
2385 case Intrinsic::riscv_vloxseg7:
2386 case Intrinsic::riscv_vloxseg8:
2387 case Intrinsic::riscv_vluxseg2:
2388 case Intrinsic::riscv_vluxseg3:
2389 case Intrinsic::riscv_vluxseg4:
2390 case Intrinsic::riscv_vluxseg5:
2391 case Intrinsic::riscv_vluxseg6:
2392 case Intrinsic::riscv_vluxseg7:
2393 case Intrinsic::riscv_vluxseg8:
2394 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2395 /*IsStore*/ false,
2396 /*IsUnitStrided*/ false);
2397 return;
2398 case Intrinsic::riscv_vlsseg2_mask:
2399 case Intrinsic::riscv_vlsseg3_mask:
2400 case Intrinsic::riscv_vlsseg4_mask:
2401 case Intrinsic::riscv_vlsseg5_mask:
2402 case Intrinsic::riscv_vlsseg6_mask:
2403 case Intrinsic::riscv_vlsseg7_mask:
2404 case Intrinsic::riscv_vlsseg8_mask:
2405 case Intrinsic::riscv_vloxseg2_mask:
2406 case Intrinsic::riscv_vloxseg3_mask:
2407 case Intrinsic::riscv_vloxseg4_mask:
2408 case Intrinsic::riscv_vloxseg5_mask:
2409 case Intrinsic::riscv_vloxseg6_mask:
2410 case Intrinsic::riscv_vloxseg7_mask:
2411 case Intrinsic::riscv_vloxseg8_mask:
2412 case Intrinsic::riscv_vluxseg2_mask:
2413 case Intrinsic::riscv_vluxseg3_mask:
2414 case Intrinsic::riscv_vluxseg4_mask:
2415 case Intrinsic::riscv_vluxseg5_mask:
2416 case Intrinsic::riscv_vluxseg6_mask:
2417 case Intrinsic::riscv_vluxseg7_mask:
2418 case Intrinsic::riscv_vluxseg8_mask:
2419 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2420 /*IsStore*/ false,
2421 /*IsUnitStrided*/ false);
2422 return;
2423 case Intrinsic::riscv_vsseg2:
2424 case Intrinsic::riscv_vsseg3:
2425 case Intrinsic::riscv_vsseg4:
2426 case Intrinsic::riscv_vsseg5:
2427 case Intrinsic::riscv_vsseg6:
2428 case Intrinsic::riscv_vsseg7:
2429 case Intrinsic::riscv_vsseg8:
2430 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2431 /*IsStore*/ true,
2432 /*IsUnitStrided*/ false);
2433 return;
2434 case Intrinsic::riscv_vsseg2_mask:
2435 case Intrinsic::riscv_vsseg3_mask:
2436 case Intrinsic::riscv_vsseg4_mask:
2437 case Intrinsic::riscv_vsseg5_mask:
2438 case Intrinsic::riscv_vsseg6_mask:
2439 case Intrinsic::riscv_vsseg7_mask:
2440 case Intrinsic::riscv_vsseg8_mask:
2441 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2442 /*IsStore*/ true,
2443 /*IsUnitStrided*/ false);
2444 return;
2445 case Intrinsic::riscv_vssseg2:
2446 case Intrinsic::riscv_vssseg3:
2447 case Intrinsic::riscv_vssseg4:
2448 case Intrinsic::riscv_vssseg5:
2449 case Intrinsic::riscv_vssseg6:
2450 case Intrinsic::riscv_vssseg7:
2451 case Intrinsic::riscv_vssseg8:
2452 case Intrinsic::riscv_vsoxseg2:
2453 case Intrinsic::riscv_vsoxseg3:
2454 case Intrinsic::riscv_vsoxseg4:
2455 case Intrinsic::riscv_vsoxseg5:
2456 case Intrinsic::riscv_vsoxseg6:
2457 case Intrinsic::riscv_vsoxseg7:
2458 case Intrinsic::riscv_vsoxseg8:
2459 case Intrinsic::riscv_vsuxseg2:
2460 case Intrinsic::riscv_vsuxseg3:
2461 case Intrinsic::riscv_vsuxseg4:
2462 case Intrinsic::riscv_vsuxseg5:
2463 case Intrinsic::riscv_vsuxseg6:
2464 case Intrinsic::riscv_vsuxseg7:
2465 case Intrinsic::riscv_vsuxseg8:
2466 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2467 /*IsStore*/ true,
2468 /*IsUnitStrided*/ false);
2469 return;
2470 case Intrinsic::riscv_vssseg2_mask:
2471 case Intrinsic::riscv_vssseg3_mask:
2472 case Intrinsic::riscv_vssseg4_mask:
2473 case Intrinsic::riscv_vssseg5_mask:
2474 case Intrinsic::riscv_vssseg6_mask:
2475 case Intrinsic::riscv_vssseg7_mask:
2476 case Intrinsic::riscv_vssseg8_mask:
2477 case Intrinsic::riscv_vsoxseg2_mask:
2478 case Intrinsic::riscv_vsoxseg3_mask:
2479 case Intrinsic::riscv_vsoxseg4_mask:
2480 case Intrinsic::riscv_vsoxseg5_mask:
2481 case Intrinsic::riscv_vsoxseg6_mask:
2482 case Intrinsic::riscv_vsoxseg7_mask:
2483 case Intrinsic::riscv_vsoxseg8_mask:
2484 case Intrinsic::riscv_vsuxseg2_mask:
2485 case Intrinsic::riscv_vsuxseg3_mask:
2486 case Intrinsic::riscv_vsuxseg4_mask:
2487 case Intrinsic::riscv_vsuxseg5_mask:
2488 case Intrinsic::riscv_vsuxseg6_mask:
2489 case Intrinsic::riscv_vsuxseg7_mask:
2490 case Intrinsic::riscv_vsuxseg8_mask:
2491 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2492 /*IsStore*/ true,
2493 /*IsUnitStrided*/ false);
2494 return;
2495 case Intrinsic::riscv_sf_vlte8:
2496 case Intrinsic::riscv_sf_vlte16:
2497 case Intrinsic::riscv_sf_vlte32:
2498 case Intrinsic::riscv_sf_vlte64:
2499 Info.opc = ISD::INTRINSIC_VOID;
2500 Info.ptrVal = I.getArgOperand(1);
2501 switch (Intrinsic) {
2502 case Intrinsic::riscv_sf_vlte8:
2503 Info.memVT = MVT::i8;
2504 Info.align = Align(1);
2505 break;
2506 case Intrinsic::riscv_sf_vlte16:
2507 Info.memVT = MVT::i16;
2508 Info.align = Align(2);
2509 break;
2510 case Intrinsic::riscv_sf_vlte32:
2511 Info.memVT = MVT::i32;
2512 Info.align = Align(4);
2513 break;
2514 case Intrinsic::riscv_sf_vlte64:
2515 Info.memVT = MVT::i64;
2516 Info.align = Align(8);
2517 break;
2518 }
2519 Info.size = MemoryLocation::UnknownSize;
2520 Info.flags |= MachineMemOperand::MOLoad;
2521 Infos.push_back(Info);
2522 return;
2523 case Intrinsic::riscv_sf_vste8:
2524 case Intrinsic::riscv_sf_vste16:
2525 case Intrinsic::riscv_sf_vste32:
2526 case Intrinsic::riscv_sf_vste64:
2527 Info.opc = ISD::INTRINSIC_VOID;
2528 Info.ptrVal = I.getArgOperand(1);
2529 switch (Intrinsic) {
2530 case Intrinsic::riscv_sf_vste8:
2531 Info.memVT = MVT::i8;
2532 Info.align = Align(1);
2533 break;
2534 case Intrinsic::riscv_sf_vste16:
2535 Info.memVT = MVT::i16;
2536 Info.align = Align(2);
2537 break;
2538 case Intrinsic::riscv_sf_vste32:
2539 Info.memVT = MVT::i32;
2540 Info.align = Align(4);
2541 break;
2542 case Intrinsic::riscv_sf_vste64:
2543 Info.memVT = MVT::i64;
2544 Info.align = Align(8);
2545 break;
2546 }
2547 Info.size = MemoryLocation::UnknownSize;
2548 Info.flags |= MachineMemOperand::MOStore;
2549 Infos.push_back(Info);
2550 return;
2551 }
2552}
2553
2554bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2555 const AddrMode &AM, Type *Ty,
2556 unsigned AS,
2557 Instruction *I) const {
2558 // No global is ever allowed as a base.
2559 if (AM.BaseGV)
2560 return false;
2561
2562 // None of our addressing modes allows a scalable offset
2563 if (AM.ScalableOffset)
2564 return false;
2565
2566 // RVV instructions only support register addressing.
2567 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
2568 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2569
2570 // The Xqcilo extension provides load/store instructions with a 26-bit signed
2571 // offset.
2572 if (Subtarget.hasVendorXqcilo()) {
2573 if (!isInt<26>(AM.BaseOffs))
2574 return false;
2575 } else if (!isInt<12>(AM.BaseOffs)) {
2576 // Otherwise require a 12-bit signed offset.
2577 return false;
2578 }
2579
2580 switch (AM.Scale) {
2581 case 0: // "r+i" or just "i", depending on HasBaseReg.
2582 break;
2583 case 1:
2584 if (!AM.HasBaseReg) // allow "r+i".
2585 break;
2586 return false; // disallow "r+r" or "r+r+i".
2587 default:
2588 return false;
2589 }
2590
2591 return true;
2592}
2593
2594bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2595 return isInt<12>(Imm);
2596}
2597
2598bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2599 // The Xqcilia extension provides add-immediate instructions with a 26-bit
2600 // signed immediate.
2601 if (Subtarget.hasVendorXqcilia())
2602 return isInt<26>(Imm);
2603 return isInt<12>(Imm);
2604}
2605
2606// On RV32, 64-bit integers are split into their high and low parts and held
2607// in two different registers, so the trunc is free since the low register can
2608// just be used.
2609// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2610// isTruncateFree?
2611bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2612 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2613 return false;
2614 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2615 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2616 return (SrcBits == 64 && DestBits == 32);
2617}
2618
2619bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2620 // We consider i64->i32 free on RV64 since we have good selection of W
2621 // instructions that make promoting operations back to i64 free in many cases.
2622 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2623 !DstVT.isInteger())
2624 return false;
2625 unsigned SrcBits = SrcVT.getSizeInBits();
2626 unsigned DestBits = DstVT.getSizeInBits();
2627 return (SrcBits == 64 && DestBits == 32);
2628}
2629
2630bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2631 EVT SrcVT = Val.getValueType();
2632 // free truncate from vnsrl and vnsra
2633 if (Subtarget.hasVInstructions() &&
2634 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2635 SrcVT.isVector() && VT2.isVector()) {
2636 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2637 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2638 if (SrcBits == DestBits * 2) {
2639 return true;
2640 }
2641 }
2642 return TargetLowering::isTruncateFree(Val, VT2);
2643}
2644
2645bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2646 // Zexts are free if they can be combined with a load.
2647 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2648 // poorly with type legalization of compares preferring sext.
2649 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2650 EVT MemVT = LD->getMemoryVT();
2651 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2652 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2653 LD->getExtensionType() == ISD::ZEXTLOAD))
2654 return true;
2655 }
2656
2657 return TargetLowering::isZExtFree(Val, VT2);
2658}
2659
2660bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2661 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2662}
2663
2664bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2665 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
2666}
2667
2668bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2669 return Subtarget.hasCTZLike();
2670}
2671
2672bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2673 return Subtarget.hasCLZLike();
2674}
2675
2676bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2677 const Instruction &AndI) const {
2678 // We expect to be able to match a bit extraction instruction if the Zbs
2679 // extension is supported and the mask is a power of two. However, we
2680 // conservatively return false if the mask would fit in an ANDI instruction,
2681 // on the basis that it's possible the sinking+duplication of the AND in
2682 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2683 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2684 if (!Subtarget.hasBEXTILike())
2685 return false;
2686 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
2687 if (!Mask)
2688 return false;
2689 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
2690}
2691
2692bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2693 EVT VT = Y.getValueType();
2694
2695 if (VT.isVector())
2696 return false;
2697
2698 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2699 (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
2700}
2701
2702bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2703 EVT VT = Y.getValueType();
2704
2705 if (!VT.isVector())
2706 return hasAndNotCompare(Y);
2707
2708 return Subtarget.hasStdExtZvkb();
2709}
2710
2711bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2712 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2713 if (Subtarget.hasStdExtZbs())
2714 return X.getValueType().isScalarInteger();
2715 auto *C = dyn_cast<ConstantSDNode>(Y);
2716 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2717 if (Subtarget.hasVendorXTHeadBs())
2718 return C != nullptr;
2719 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2720 return C && C->getAPIntValue().ule(10);
2721}
2722
2723bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2724 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2725 SDValue Y) const {
2726 if (SelectOpcode != ISD::VSELECT)
2727 return false;
2728
2729 // Only enable for rvv.
2730 if (!VT.isVector() || !Subtarget.hasVInstructions())
2731 return false;
2732
2733 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2734 return false;
2735
2736 return true;
2737}
2738
2739bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2740 Type *Ty) const {
2741 assert(Ty->isIntegerTy());
2742
2743 unsigned BitSize = Ty->getIntegerBitWidth();
2744 if (BitSize > Subtarget.getXLen())
2745 return false;
2746
2747 // Fast path, assume 32-bit immediates are cheap.
2748 int64_t Val = Imm.getSExtValue();
2749 if (isInt<32>(Val))
2750 return true;
2751
2752 // A constant pool entry may be more aligned than the load we're trying to
2753 // replace. If we don't support unaligned scalar mem, prefer the constant
2754 // pool.
2755 // TODO: Can the caller pass down the alignment?
2756 if (!Subtarget.enableUnalignedScalarMem())
2757 return true;
2758
2759 // Prefer to keep the load if it would require many instructions.
2760 // This uses the same threshold we use for constant pools but doesn't
2761 // check useConstantPoolForLargeInts.
2762 // TODO: Should we keep the load only when we're definitely going to emit a
2763 // constant pool?
2764
2765 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget);
2766 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2767}
2768
2769bool RISCVTargetLowering::
2770 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2771 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2772 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2773 SelectionDAG &DAG) const {
2774 // One interesting pattern that we'd want to form is 'bit extract':
2775 // ((1 >> Y) & 1) ==/!= 0
2776 // But we also need to be careful not to try to reverse that fold.
2777
2778 // Is this '((1 >> Y) & 1)'?
2779 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2780 return false; // Keep the 'bit extract' pattern.
2781
2782 // Will this be '((1 >> Y) & 1)' after the transform?
2783 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2784 return true; // Do form the 'bit extract' pattern.
2785
2786 // If 'X' is a constant, and we transform, then we will immediately
2787 // try to undo the fold, thus causing endless combine loop.
2788 // So only do the transform if X is not a constant. This matches the default
2789 // implementation of this function.
2790 return !XC;
2791}
2792
2793bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2794 unsigned Opc = VecOp.getOpcode();
2795
2796 // Assume target opcodes can't be scalarized.
2797 // TODO - do we have any exceptions?
2798 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
2799 return false;
2800
2801 // If the vector op is not supported, try to convert to scalar.
2802 EVT VecVT = VecOp.getValueType();
2803 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
2804 return true;
2805
2806 // If the vector op is supported, but the scalar op is not, the transform may
2807 // not be worthwhile.
2808 // Permit a vector binary operation can be converted to scalar binary
2809 // operation which is custom lowered with illegal type.
2810 EVT ScalarVT = VecVT.getScalarType();
2811 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
2812 isOperationCustom(Opc, ScalarVT);
2813}
2814
2815bool RISCVTargetLowering::isOffsetFoldingLegal(
2816 const GlobalAddressSDNode *GA) const {
2817 // In order to maximise the opportunity for common subexpression elimination,
2818 // keep a separate ADD node for the global address offset instead of folding
2819 // it in the global address node. Later peephole optimisations may choose to
2820 // fold it back in when profitable.
2821 return false;
2822}
2823
2824// Returns 0-31 if the fli instruction is available for the type and this is
2825// legal FP immediate for the type. Returns -1 otherwise.
2826int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2827 if (!Subtarget.hasStdExtZfa())
2828 return -1;
2829
2830 bool IsSupportedVT = false;
2831 if (VT == MVT::f16) {
2832 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2833 } else if (VT == MVT::f32) {
2834 IsSupportedVT = true;
2835 } else if (VT == MVT::f64) {
2836 assert(Subtarget.hasStdExtD() && "Expect D extension");
2837 IsSupportedVT = true;
2838 }
2839
2840 if (!IsSupportedVT)
2841 return -1;
2842
2843 return RISCVLoadFPImm::getLoadFPImm(Imm);
2844}
2845
2846bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2847 bool ForCodeSize) const {
2848 bool IsLegalVT = false;
2849 if (VT == MVT::f16)
2850 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2851 else if (VT == MVT::f32)
2852 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2853 else if (VT == MVT::f64)
2854 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2855 else if (VT == MVT::bf16)
2856 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2857
2858 if (!IsLegalVT)
2859 return false;
2860
2861 if (getLegalZfaFPImm(Imm, VT) >= 0)
2862 return true;
2863
2864 // Some constants can be produced by fli+fneg.
2865 if (Imm.isNegative() && getLegalZfaFPImm(-Imm, VT) >= 0)
2866 return true;
2867
2868 // Cannot create a 64 bit floating-point immediate value for rv32.
2869 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2870 // td can handle +0.0 or -0.0 already.
2871 // -0.0 can be created by fmv + fneg.
2872 return Imm.isZero();
2873 }
2874
2875 // Special case: fmv + fneg
2876 if (Imm.isNegZero())
2877 return true;
2878
2879 // Building an integer and then converting requires a fmv at the end of
2880 // the integer sequence. The fmv is not required for Zfinx.
2881 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2882 const int Cost =
2883 FmvCost + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
2884 Subtarget.getXLen(), Subtarget);
2885 return Cost <= FPImmCost;
2886}
2887
2888// TODO: This is very conservative.
2889TargetLowering::ExtractSubvectorCost
2890RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2891 unsigned Index) const {
2892 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
2893 (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) {
2894 if ((Index % ResVT.getVectorNumElements()) == 0)
2895 return ExtractSubvectorCost::Free;
2896 return ExtractSubvectorCost::Expensive;
2897 }
2898
2899 if (!Subtarget.hasVInstructions())
2900 return ExtractSubvectorCost::Expensive;
2901
2902 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
2903 return ExtractSubvectorCost::Expensive;
2904
2905 // Extracts from index 0 are just subreg extracts.
2906 if (Index == 0)
2907 return ExtractSubvectorCost::Free;
2908
2909 // Only support extracting a fixed from a fixed vector for now.
2910 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2911 return ExtractSubvectorCost::Expensive;
2912
2913 EVT EltVT = ResVT.getVectorElementType();
2914 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2915
2916 // The smallest type we can slide is i8.
2917 if (EltVT == MVT::i1)
2918 return ExtractSubvectorCost::Expensive;
2919
2920 unsigned ResElts = ResVT.getVectorNumElements();
2921 unsigned SrcElts = SrcVT.getVectorNumElements();
2922
2923 unsigned MinVLen = Subtarget.getRealMinVLen();
2924 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2925
2926 // If we're extracting only data from the first VLEN bits of the source
2927 // then we can always do this with an m1 vslidedown.vx. Restricting the
2928 // Index ensures we can use a vslidedown.vi.
2929 // TODO: We can generalize this when the exact VLEN is known.
2930 if (Index + ResElts <= MinVLMAX && Index < 31)
2931 return ExtractSubvectorCost::Free;
2932
2933 // Convervatively only handle extracting half of a vector.
2934 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2935 // the upper half of a vector until we have more test coverage.
2936 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2937 // a cheap extract. However, this case is important in practice for
2938 // shuffled extracts of longer vectors. How resolve?
2939 if ((ResElts * 2) == SrcElts && Index == ResElts)
2940 return ExtractSubvectorCost::Free;
2941 return ExtractSubvectorCost::Expensive;
2942}
2943
2944MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2945 CallingConv::ID CC,
2946 EVT VT) const {
2947 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2948 // We might still end up using a GPR but that will be decided based on ABI.
2949 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2950 !Subtarget.hasStdExtZfhminOrZhinxmin())
2951 return MVT::f32;
2952
2953 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2954}
2955
2956unsigned
2957RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2958 std::optional<MVT> RegisterVT) const {
2959 // Pair inline assembly operand
2960 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2961 *RegisterVT == MVT::Untyped)
2962 return 1;
2963
2964 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2965}
2966
2967unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2968 CallingConv::ID CC,
2969 EVT VT) const {
2970 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2971 // We might still end up using a GPR but that will be decided based on ABI.
2972 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2973 !Subtarget.hasStdExtZfhminOrZhinxmin())
2974 return 1;
2975
2976 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2977}
2978
2979// Changes the condition code and swaps operands if necessary, so the SetCC
2980// operation matches one of the comparisons supported directly by branches
2981// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2982// with 1/-1.
2983static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2984 ISD::CondCode &CC, SelectionDAG &DAG,
2985 const RISCVSubtarget &Subtarget) {
2986 // If this is a single bit test that can't be handled by ANDI, shift the
2987 // bit to be tested to the MSB and perform a signed compare with 0.
2988 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
2989 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2990 isa<ConstantSDNode>(LHS.getOperand(1)) &&
2991 // XAndesPerf supports branch on test bit.
2992 !Subtarget.hasVendorXAndesPerf()) {
2993 uint64_t Mask = LHS.getConstantOperandVal(1);
2994 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
2995 unsigned ShAmt = 0;
2996 if (isPowerOf2_64(Mask)) {
2997 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
2998 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
2999 } else {
3000 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
3001 }
3002
3003 LHS = LHS.getOperand(0);
3004 if (ShAmt != 0)
3005 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
3006 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
3007 return;
3008 }
3009 }
3010
3011 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
3012 int64_t C = RHSC->getSExtValue();
3013 switch (CC) {
3014 default: break;
3015 case ISD::SETGT:
3016 // Convert X > -1 to X >= 0.
3017 if (C == -1) {
3018 RHS = DAG.getConstant(0, DL, RHS.getValueType());
3019 CC = ISD::SETGE;
3020 return;
3021 }
3022 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3023 C != INT64_MAX && isInt<5>(C + 1)) {
3024 // We have a conditional move instruction for SETGE but not SETGT.
3025 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3026 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
3027 CC = ISD::SETGE;
3028 return;
3029 }
3030 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1)) {
3031 // We have a branch immediate instruction for SETGE but not SETGT.
3032 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
3033 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
3034 CC = ISD::SETGE;
3035 return;
3036 }
3037 break;
3038 case ISD::SETLT:
3039 // Convert X < 1 to 0 >= X.
3040 if (C == 1) {
3041 RHS = LHS;
3042 LHS = DAG.getConstant(0, DL, RHS.getValueType());
3043 CC = ISD::SETGE;
3044 return;
3045 }
3046 break;
3047 case ISD::SETUGT:
3048 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3049 C != INT64_MAX && isUInt<5>(C + 1)) {
3050 // We have a conditional move instruction for SETUGE but not SETUGT.
3051 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3052 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
3053 CC = ISD::SETUGE;
3054 return;
3055 }
3056 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
3057 // We have a branch immediate instruction for SETUGE but not SETUGT.
3058 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
3059 // immediate.
3060 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
3061 CC = ISD::SETUGE;
3062 return;
3063 }
3064 break;
3065 }
3066 }
3067
3068 switch (CC) {
3069 default:
3070 break;
3071 case ISD::SETGT:
3072 case ISD::SETLE:
3073 case ISD::SETUGT:
3074 case ISD::SETULE:
3075 CC = ISD::getSetCCSwappedOperands(CC);
3076 std::swap(LHS, RHS);
3077 break;
3078 }
3079}
3080
3081RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
3082 if (VT.isRISCVVectorTuple()) {
3083 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
3084 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
3085 return RISCVVType::LMUL_F8;
3086 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
3087 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
3088 return RISCVVType::LMUL_F4;
3089 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
3090 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
3091 return RISCVVType::LMUL_F2;
3092 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
3093 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
3094 return RISCVVType::LMUL_1;
3095 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
3096 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
3097 return RISCVVType::LMUL_2;
3098 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
3099 return RISCVVType::LMUL_4;
3100 llvm_unreachable("Invalid vector tuple type LMUL.");
3101 }
3102
3103 assert(VT.isScalableVector() && "Expecting a scalable vector type");
3104 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
3105 if (VT.getVectorElementType() == MVT::i1)
3106 KnownSize *= 8;
3107
3108 switch (KnownSize) {
3109 default:
3110 llvm_unreachable("Invalid LMUL.");
3111 case 8:
3112 return RISCVVType::LMUL_F8;
3113 case 16:
3114 return RISCVVType::LMUL_F4;
3115 case 32:
3116 return RISCVVType::LMUL_F2;
3117 case 64:
3118 return RISCVVType::LMUL_1;
3119 case 128:
3120 return RISCVVType::LMUL_2;
3121 case 256:
3122 return RISCVVType::LMUL_4;
3123 case 512:
3124 return RISCVVType::LMUL_8;
3125 }
3126}
3127
3128unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
3129 switch (LMul) {
3130 default:
3131 llvm_unreachable("Invalid LMUL.");
3132 case RISCVVType::LMUL_F8:
3133 case RISCVVType::LMUL_F4:
3134 case RISCVVType::LMUL_F2:
3135 case RISCVVType::LMUL_1:
3136 return RISCV::VRRegClassID;
3137 case RISCVVType::LMUL_2:
3138 return RISCV::VRM2RegClassID;
3139 case RISCVVType::LMUL_4:
3140 return RISCV::VRM4RegClassID;
3141 case RISCVVType::LMUL_8:
3142 return RISCV::VRM8RegClassID;
3143 }
3144}
3145
3146unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
3147 RISCVVType::VLMUL LMUL = getLMUL(VT);
3148 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
3149 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
3150 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
3151 "Unexpected subreg numbering");
3152 return RISCV::sub_vrm1_0 + Index;
3153 }
3154 if (LMUL == RISCVVType::LMUL_2) {
3155 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
3156 "Unexpected subreg numbering");
3157 return RISCV::sub_vrm2_0 + Index;
3158 }
3159 if (LMUL == RISCVVType::LMUL_4) {
3160 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
3161 "Unexpected subreg numbering");
3162 return RISCV::sub_vrm4_0 + Index;
3163 }
3164 llvm_unreachable("Invalid vector type.");
3165}
3166
3167unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
3168 if (VT.isRISCVVectorTuple()) {
3169 unsigned NF = VT.getRISCVVectorTupleNumFields();
3170 unsigned RegsPerField =
3171 std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
3172 (NF * RISCV::RVVBitsPerBlock));
3173 switch (RegsPerField) {
3174 case 1:
3175 if (NF == 2)
3176 return RISCV::VRN2M1RegClassID;
3177 if (NF == 3)
3178 return RISCV::VRN3M1RegClassID;
3179 if (NF == 4)
3180 return RISCV::VRN4M1RegClassID;
3181 if (NF == 5)
3182 return RISCV::VRN5M1RegClassID;
3183 if (NF == 6)
3184 return RISCV::VRN6M1RegClassID;
3185 if (NF == 7)
3186 return RISCV::VRN7M1RegClassID;
3187 if (NF == 8)
3188 return RISCV::VRN8M1RegClassID;
3189 break;
3190 case 2:
3191 if (NF == 2)
3192 return RISCV::VRN2M2RegClassID;
3193 if (NF == 3)
3194 return RISCV::VRN3M2RegClassID;
3195 if (NF == 4)
3196 return RISCV::VRN4M2RegClassID;
3197 break;
3198 case 4:
3199 assert(NF == 2);
3200 return RISCV::VRN2M4RegClassID;
3201 default:
3202 break;
3203 }
3204 llvm_unreachable("Invalid vector tuple type RegClass.");
3205 }
3206
3207 if (VT.getVectorElementType() == MVT::i1)
3208 return RISCV::VRRegClassID;
3209 return getRegClassIDForLMUL(getLMUL(VT));
3210}
3211
3212// Attempt to decompose a subvector insert/extract between VecVT and
3213// SubVecVT via subregister indices. Returns the subregister index that
3214// can perform the subvector insert/extract with the given element index, as
3215// well as the index corresponding to any leftover subvectors that must be
3216// further inserted/extracted within the register class for SubVecVT.
3217std::pair<unsigned, unsigned>
3218RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3219 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3220 const RISCVRegisterInfo *TRI) {
3221 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3222 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3223 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3224 "Register classes not ordered");
3225 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
3226 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
3227
3228 // If VecVT is a vector tuple type, either it's the tuple type with same
3229 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3230 if (VecVT.isRISCVVectorTuple()) {
3231 if (VecRegClassID == SubRegClassID)
3232 return {RISCV::NoSubRegister, 0};
3233
3234 assert(SubVecVT.isScalableVector() &&
3235 "Only allow scalable vector subvector.");
3236 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3237 "Invalid vector tuple insert/extract for vector and subvector with "
3238 "different LMUL.");
3239 return {getSubregIndexByMVT(VecVT, InsertExtractIdx), 0};
3240 }
3241
3242 // Try to compose a subregister index that takes us from the incoming
3243 // LMUL>1 register class down to the outgoing one. At each step we half
3244 // the LMUL:
3245 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3246 // Note that this is not guaranteed to find a subregister index, such as
3247 // when we are extracting from one VR type to another.
3248 unsigned SubRegIdx = RISCV::NoSubRegister;
3249 for (const unsigned RCID :
3250 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3251 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3252 VecVT = VecVT.getHalfNumVectorElementsVT();
3253 bool IsHi =
3254 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3255 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
3256 getSubregIndexByMVT(VecVT, IsHi));
3257 if (IsHi)
3258 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3259 }
3260 return {SubRegIdx, InsertExtractIdx};
3261}
3262
3263// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3264// stores for those types.
3265bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3266 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
3267 (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
3268 return false;
3269
3270 return !Subtarget.useRVVForFixedLengthVectors() ||
3271 VT.isFixedLengthVectorOf(MVT::i1);
3272}
3273
3274bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3275 if (!ScalarTy.isSimple())
3276 return false;
3277 switch (ScalarTy.getSimpleVT().SimpleTy) {
3278 case MVT::iPTR:
3279 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3280 case MVT::i8:
3281 case MVT::i16:
3282 case MVT::i32:
3283 return Subtarget.hasVInstructions();
3284 case MVT::i64:
3285 return Subtarget.hasVInstructionsI64();
3286 case MVT::f16:
3287 return Subtarget.hasVInstructionsF16Minimal();
3288 case MVT::bf16:
3289 return Subtarget.hasVInstructionsBF16Minimal();
3290 case MVT::f32:
3291 return Subtarget.hasVInstructionsF32();
3292 case MVT::f64:
3293 return Subtarget.hasVInstructionsF64();
3294 default:
3295 return false;
3296 }
3297}
3298
3299
3300unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3301 return NumRepeatedDivisors;
3302}
3303
3304static SDValue getVLOperand(SDValue Op) {
3305 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3306 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3307 "Unexpected opcode");
3308 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3309 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
3310 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3311 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
3312 if (!II)
3313 return SDValue();
3314 return Op.getOperand(II->VLOperand + 1 + HasChain);
3315}
3316
3317static bool useRVVForFixedLengthVectorVT(MVT VT,
3318 const RISCVSubtarget &Subtarget) {
3319 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3320 if (!Subtarget.useRVVForFixedLengthVectors())
3321 return false;
3322
3323 // We only support a set of vector types with a consistent maximum fixed size
3324 // across all supported vector element types to avoid legalization issues.
3325 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3326 // fixed-length vector type we support is 1024 bytes.
3327 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3328 return false;
3329
3330 unsigned MinVLen = Subtarget.getRealMinVLen();
3331
3332 MVT EltVT = VT.getVectorElementType();
3333
3334 // Don't use RVV for vectors we cannot scalarize if required.
3335 switch (EltVT.SimpleTy) {
3336 // i1 is supported but has different rules.
3337 default:
3338 return false;
3339 case MVT::i1:
3340 // Masks can only use a single register.
3341 if (VT.getVectorNumElements() > MinVLen)
3342 return false;
3343 MinVLen /= 8;
3344 break;
3345 case MVT::i8:
3346 case MVT::i16:
3347 case MVT::i32:
3348 break;
3349 case MVT::i64:
3350 if (!Subtarget.hasVInstructionsI64())
3351 return false;
3352 break;
3353 case MVT::f16:
3354 if (!Subtarget.hasVInstructionsF16Minimal())
3355 return false;
3356 break;
3357 case MVT::bf16:
3358 if (!Subtarget.hasVInstructionsBF16Minimal())
3359 return false;
3360 break;
3361 case MVT::f32:
3362 if (!Subtarget.hasVInstructionsF32())
3363 return false;
3364 break;
3365 case MVT::f64:
3366 if (!Subtarget.hasVInstructionsF64())
3367 return false;
3368 break;
3369 }
3370
3371 // Reject elements larger than ELEN.
3372 if (EltVT.getSizeInBits() > Subtarget.getELen())
3373 return false;
3374
3375 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
3376 // Don't use RVV for types that don't fit.
3377 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3378 return false;
3379
3380 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3381 // the base fixed length RVV support in place.
3382 if (!VT.isPow2VectorType())
3383 return false;
3384
3385 return true;
3386}
3387
3388bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3389 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3390}
3391
3392// Return the largest legal scalable vector type that matches VT's element type.
3393static MVT getContainerForFixedLengthVector(MVT VT,
3394 const RISCVSubtarget &Subtarget) {
3395 // This may be called before legal types are setup.
3396 assert(((VT.isFixedLengthVector() &&
3397 Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
3398 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3399 "Expected legal fixed length vector!");
3400
3401 unsigned MinVLen = Subtarget.getRealMinVLen();
3402 unsigned MaxELen = Subtarget.getELen();
3403
3404 MVT EltVT = VT.getVectorElementType();
3405 switch (EltVT.SimpleTy) {
3406 default:
3407 llvm_unreachable("unexpected element type for RVV container");
3408 case MVT::i1:
3409 case MVT::i8:
3410 case MVT::i16:
3411 case MVT::i32:
3412 case MVT::i64:
3413 case MVT::bf16:
3414 case MVT::f16:
3415 case MVT::f32:
3416 case MVT::f64: {
3417 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3418 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3419 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3420 unsigned NumElts =
3421 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3422 NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
3423 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3424 return MVT::getScalableVectorVT(EltVT, NumElts);
3425 }
3426 }
3427}
3428
3429MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3430 return ::getContainerForFixedLengthVector(VT, getSubtarget());
3431}
3432
3433// Grow V to consume an entire RVV register.
3434static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3435 const RISCVSubtarget &Subtarget) {
3436 assert(VT.isScalableVector() &&
3437 "Expected to convert into a scalable vector!");
3438 assert(V.getValueType().isFixedLengthVector() &&
3439 "Expected a fixed length vector operand!");
3440 SDLoc DL(V);
3441 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), V, 0);
3442}
3443
3444// Shrink V so it's just big enough to maintain a VT's worth of data.
3445static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3446 const RISCVSubtarget &Subtarget) {
3447 assert(VT.isFixedLengthVector() &&
3448 "Expected to convert into a fixed length vector!");
3449 assert(V.getValueType().isScalableVector() &&
3450 "Expected a scalable vector operand!");
3451 SDLoc DL(V);
3452 return DAG.getExtractSubvector(DL, VT, V, 0);
3453}
3454
3455/// Return the type of the mask type suitable for masking the provided
3456/// vector type. This is simply an i1 element type vector of the same
3457/// (possibly scalable) length.
3458static MVT getMaskTypeFor(MVT VecVT) {
3459 assert(VecVT.isVector());
3460 ElementCount EC = VecVT.getVectorElementCount();
3461 return MVT::getVectorVT(MVT::i1, EC);
3462}
3463
3464/// Creates an all ones mask suitable for masking a vector of type VecTy with
3465/// vector length VL. .
3466static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3467 SelectionDAG &DAG) {
3468 MVT MaskVT = getMaskTypeFor(VecVT);
3469 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
3470}
3471
3472static std::pair<SDValue, SDValue>
3473getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3474 const RISCVSubtarget &Subtarget) {
3475 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3476 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
3477 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3478 return {Mask, VL};
3479}
3480
3481static std::pair<SDValue, SDValue>
3482getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3483 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3484 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3485 SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
3486 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
3487 return {Mask, VL};
3488}
3489
3490// Gets the two common "VL" operands: an all-ones mask and the vector length.
3491// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3492// the vector type that the fixed-length vector is contained in. Otherwise if
3493// VecVT is scalable, then ContainerVT should be the same as VecVT.
3494static std::pair<SDValue, SDValue>
3495getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3496 const RISCVSubtarget &Subtarget) {
3497 if (VecVT.isFixedLengthVector())
3498 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3499 Subtarget);
3500 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3501 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
3502}
3503
3504SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3505 SelectionDAG &DAG) const {
3506 assert(VecVT.isScalableVector() && "Expected scalable vector");
3507 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
3508 VecVT.getVectorElementCount());
3509}
3510
3511std::pair<unsigned, unsigned>
3512RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3513 const RISCVSubtarget &Subtarget) {
3514 assert(VecVT.isScalableVector() && "Expected scalable vector");
3515
3516 unsigned EltSize = VecVT.getScalarSizeInBits();
3517 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3518
3519 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3520 unsigned MaxVLMAX =
3521 RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
3522
3523 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3524 unsigned MinVLMAX =
3525 RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
3526
3527 return std::make_pair(MinVLMAX, MaxVLMAX);
3528}
3529
3530// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3531// of either is (currently) supported. This can get us into an infinite loop
3532// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3533// as a ..., etc.
3534// Until either (or both) of these can reliably lower any node, reporting that
3535// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3536// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3537// which is not desirable.
3538bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3539 EVT VT, unsigned DefinedValues) const {
3540 return false;
3541}
3542
3543InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3544 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3545 // implementation-defined.
3546 if (!VT.isVector())
3547 return InstructionCost::getInvalid();
3548 unsigned DLenFactor = Subtarget.getDLenFactor();
3549 unsigned Cost;
3550 if (VT.isScalableVector()) {
3551 unsigned LMul;
3552 bool Fractional;
3553 std::tie(LMul, Fractional) =
3554 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
3555 if (Fractional)
3556 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3557 else
3558 Cost = (LMul * DLenFactor);
3559 } else {
3560 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
3561 }
3562 return Cost;
3563}
3564
3565
3566/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3567/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3568/// be by default. VRGatherCostModel reflects available options. Note that
3569/// operand (index and possibly mask) are handled separately.
3570InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3571 auto LMULCost = getLMULCost(VT);
3572 bool Log2CostModel =
3573 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3574 if (Log2CostModel && LMULCost.isValid()) {
3575 unsigned Log = Log2_64(LMULCost.getValue());
3576 if (Log > 0)
3577 return LMULCost * Log;
3578 }
3579 return LMULCost * LMULCost;
3580}
3581
3582/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3583/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3584/// or may track the vrgather.vv cost. It is implementation-dependent.
3588
3589/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3590/// for the type VT. (This does not cover the vslide1up or vslide1down
3591/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3592/// or may track the vrgather.vv cost. It is implementation-dependent.
3596
3597/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3598/// for the type VT. (This does not cover the vslide1up or vslide1down
3599/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3600/// or may track the vrgather.vv cost. It is implementation-dependent.
3604
3605static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3606 const RISCVSubtarget &Subtarget) {
3607 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3608 // bf16 conversions are always promoted to f32.
3609 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3610 Op.getValueType() == MVT::bf16) {
3611 bool IsStrict = Op->isStrictFPOpcode();
3612
3613 SDLoc DL(Op);
3614 if (IsStrict) {
3615 SDValue Val = DAG.getNode(Op.getOpcode(), DL, {MVT::f32, MVT::Other},
3616 {Op.getOperand(0), Op.getOperand(1)});
3617 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
3618 {Op.getValueType(), MVT::Other},
3619 {Val.getValue(1), Val.getValue(0),
3620 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
3621 }
3622 return DAG.getNode(
3623 ISD::FP_ROUND, DL, Op.getValueType(),
3624 DAG.getNode(Op.getOpcode(), DL, MVT::f32, Op.getOperand(0)),
3625 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
3626 }
3627
3628 // Other operations are legal.
3629 return Op;
3630}
3631
3632static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3633 const RISCVSubtarget &Subtarget) {
3634 // RISC-V FP-to-int conversions saturate to the destination register size, but
3635 // don't produce 0 for nan. We can use a conversion instruction and fix the
3636 // nan case with a compare and a select.
3637 SDValue Src = Op.getOperand(0);
3638
3639 MVT DstVT = Op.getSimpleValueType();
3640 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3641
3642 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3643
3644 if (!DstVT.isVector()) {
3645 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3646 // the result.
3647 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3648 Src.getValueType() == MVT::bf16) {
3649 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
3650 }
3651
3652 unsigned Opc;
3653 if (SatVT == DstVT)
3654 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3655 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3656 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3657 else
3658 return SDValue();
3659 // FIXME: Support other SatVTs by clamping before or after the conversion.
3660
3661 SDLoc DL(Op);
3662 SDValue FpToInt = DAG.getNode(
3663 Opc, DL, DstVT, Src,
3664 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
3665
3666 if (Opc == RISCVISD::FCVT_WU_RV64)
3667 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
3668
3669 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
3670 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
3671 ISD::CondCode::SETUO);
3672 }
3673
3674 // Vectors.
3675
3676 MVT DstEltVT = DstVT.getVectorElementType();
3677 MVT SrcVT = Src.getSimpleValueType();
3678 MVT SrcEltVT = SrcVT.getVectorElementType();
3679 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3680 unsigned DstEltSize = DstEltVT.getSizeInBits();
3681
3682 // Only handle saturating to the destination type.
3683 if (SatVT != DstEltVT)
3684 return SDValue();
3685
3686 MVT DstContainerVT = DstVT;
3687 MVT SrcContainerVT = SrcVT;
3688 if (DstVT.isFixedLengthVector()) {
3689 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
3690 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
3691 assert(DstContainerVT.getVectorElementCount() ==
3692 SrcContainerVT.getVectorElementCount() &&
3693 "Expected same element count");
3694 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3695 }
3696
3697 SDLoc DL(Op);
3698
3699 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
3700
3701 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
3702 {Src, Src, DAG.getCondCode(ISD::SETNE),
3703 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
3704
3705 // Need to widen by more than 1 step, promote the FP type, then do a widening
3706 // convert.
3707 if (DstEltSize > (2 * SrcEltSize)) {
3708 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3709 MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
3710 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
3711 }
3712
3713 MVT CvtContainerVT = DstContainerVT;
3714 MVT CvtEltVT = DstEltVT;
3715 if (SrcEltSize > (2 * DstEltSize)) {
3716 CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3717 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3718 }
3719
3720 unsigned RVVOpc =
3721 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3722 SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
3723
3724 while (CvtContainerVT != DstContainerVT) {
3725 CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
3726 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3727 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3728 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3729 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3730 Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
3731 }
3732
3733 SDValue SplatZero = DAG.getNode(
3734 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
3735 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
3736 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
3737 Res, DAG.getUNDEF(DstContainerVT), VL);
3738
3739 if (DstVT.isFixedLengthVector())
3740 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
3741
3742 return Res;
3743}
3744
3745static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3746 const RISCVSubtarget &Subtarget) {
3747 bool IsStrict = Op->isStrictFPOpcode();
3748 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3749
3750 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3751 // bf16 conversions are always promoted to f32.
3752 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3753 SrcVal.getValueType() == MVT::bf16) {
3754 SDLoc DL(Op);
3755 if (IsStrict) {
3756 SDValue Ext =
3757 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
3758 {Op.getOperand(0), SrcVal});
3759 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
3760 {Ext.getValue(1), Ext.getValue(0)});
3761 }
3762 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
3763 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
3764 }
3765
3766 // Other operations are legal.
3767 return Op;
3768}
3769
3770static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3771 switch (Opc) {
3772 case ISD::FROUNDEVEN:
3773 case ISD::STRICT_FROUNDEVEN:
3774 return RISCVFPRndMode::RNE;
3775 case ISD::FTRUNC:
3776 case ISD::STRICT_FTRUNC:
3777 return RISCVFPRndMode::RTZ;
3778 case ISD::FFLOOR:
3779 case ISD::STRICT_FFLOOR:
3780 return RISCVFPRndMode::RDN;
3781 case ISD::FCEIL:
3782 case ISD::STRICT_FCEIL:
3783 return RISCVFPRndMode::RUP;
3784 case ISD::FROUND:
3785 case ISD::LROUND:
3786 case ISD::LLROUND:
3787 case ISD::STRICT_FROUND:
3788 case ISD::STRICT_LROUND:
3789 case ISD::STRICT_LLROUND:
3790 return RISCVFPRndMode::RMM;
3791 case ISD::FRINT:
3792 case ISD::LRINT:
3793 case ISD::LLRINT:
3794 case ISD::STRICT_FRINT:
3795 case ISD::STRICT_LRINT:
3796 case ISD::STRICT_LLRINT:
3797 return RISCVFPRndMode::DYN;
3798 }
3799
3800 return RISCVFPRndMode::Invalid;
3801}
3802
3803// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
3804// the integer domain and back. Taking care to avoid converting values that are
3805// nan or already correct.
3806static SDValue
3807lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3808 const RISCVSubtarget &Subtarget) {
3809 MVT VT = Op.getSimpleValueType();
3810 assert(VT.isVector() && "Unexpected type");
3811
3812 SDLoc DL(Op);
3813
3814 SDValue Src = Op.getOperand(0);
3815
3816 // Freeze the source since we are increasing the number of uses.
3817 Src = DAG.getFreeze(Src);
3818
3819 MVT ContainerVT = VT;
3820 if (VT.isFixedLengthVector()) {
3821 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3822 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3823 }
3824
3825 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3826
3827 // We do the conversion on the absolute value and fix the sign at the end.
3828 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3829
3830 // Determine the largest integer that can be represented exactly. This and
3831 // values larger than it don't have any fractional bits so don't need to
3832 // be converted.
3833 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3834 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3835 APFloat MaxVal = APFloat(FltSem);
3836 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3837 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3838 SDValue MaxValNode =
3839 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3840 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3841 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3842
3843 // If abs(Src) was larger than MaxVal or nan, keep it.
3844 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3845 Mask =
3846 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
3847 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3848 Mask, Mask, VL});
3849
3850 // Truncate to integer and convert back to FP.
3851 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3852 MVT XLenVT = Subtarget.getXLenVT();
3853 SDValue Truncated;
3854
3855 switch (Op.getOpcode()) {
3856 default:
3857 llvm_unreachable("Unexpected opcode");
3858 case ISD::FRINT:
3859 case ISD::FCEIL:
3860 case ISD::FFLOOR:
3861 case ISD::FROUND:
3862 case ISD::FROUNDEVEN: {
3863 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3864 assert(FRM != RISCVFPRndMode::Invalid);
3865 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3866 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3867 break;
3868 }
3869 case ISD::FTRUNC:
3870 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3871 Mask, VL);
3872 break;
3873 case ISD::FNEARBYINT:
3874 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3875 Mask, VL);
3876 break;
3877 }
3878
3879 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3880 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3881 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3882 Mask, VL);
3883
3884 // Restore the original sign so that -0.0 is preserved.
3885 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3886 Src, Src, Mask, VL);
3887
3888 if (!VT.isFixedLengthVector())
3889 return Truncated;
3890
3891 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3892}
3893
3894// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3895// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3896// qNan and converting the new source to integer and back to FP.
3897static SDValue
3898lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3899 const RISCVSubtarget &Subtarget) {
3900 SDLoc DL(Op);
3901 MVT VT = Op.getSimpleValueType();
3902 SDValue Chain = Op.getOperand(0);
3903 SDValue Src = Op.getOperand(1);
3904
3905 MVT ContainerVT = VT;
3906 if (VT.isFixedLengthVector()) {
3907 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3908 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3909 }
3910
3911 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3912
3913 // Freeze the source since we are increasing the number of uses.
3914 Src = DAG.getFreeze(Src);
3915
3916 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3917 MVT MaskVT = Mask.getSimpleValueType();
3918 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3919 DAG.getVTList(MaskVT, MVT::Other),
3920 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3921 DAG.getUNDEF(MaskVT), Mask, VL});
3922 Chain = Unorder.getValue(1);
3923 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3924 DAG.getVTList(ContainerVT, MVT::Other),
3925 {Chain, Src, Src, Src, Unorder, VL});
3926 Chain = Src.getValue(1);
3927
3928 // We do the conversion on the absolute value and fix the sign at the end.
3929 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3930
3931 // Determine the largest integer that can be represented exactly. This and
3932 // values larger than it don't have any fractional bits so don't need to
3933 // be converted.
3934 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3935 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3936 APFloat MaxVal = APFloat(FltSem);
3937 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3938 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3939 SDValue MaxValNode =
3940 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3941 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3942 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3943
3944 // If abs(Src) was larger than MaxVal or nan, keep it.
3945 Mask = DAG.getNode(
3946 RISCVISD::SETCC_VL, DL, MaskVT,
3947 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3948
3949 // Truncate to integer and convert back to FP.
3950 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3951 MVT XLenVT = Subtarget.getXLenVT();
3952 SDValue Truncated;
3953
3954 switch (Op.getOpcode()) {
3955 default:
3956 llvm_unreachable("Unexpected opcode");
3957 case ISD::STRICT_FCEIL:
3958 case ISD::STRICT_FFLOOR:
3959 case ISD::STRICT_FROUND:
3960 case ISD::STRICT_FROUNDEVEN: {
3961 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3962 assert(FRM != RISCVFPRndMode::Invalid);
3963 Truncated = DAG.getNode(
3964 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3965 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3966 break;
3967 }
3968 case ISD::STRICT_FTRUNC:
3969 Truncated =
3970 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3971 DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3972 break;
3973 case ISD::STRICT_FNEARBYINT:
3974 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3975 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3976 Mask, VL);
3977 break;
3978 }
3979 Chain = Truncated.getValue(1);
3980
3981 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3982 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3983 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3984 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3985 Truncated, Mask, VL);
3986 Chain = Truncated.getValue(1);
3987 }
3988
3989 // Restore the original sign so that -0.0 is preserved.
3990 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3991 Src, Src, Mask, VL);
3992
3993 if (VT.isFixedLengthVector())
3994 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3995 return DAG.getMergeValues({Truncated, Chain}, DL);
3996}
3997
3998static SDValue
3999lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
4000 const RISCVSubtarget &Subtarget) {
4001 MVT VT = Op.getSimpleValueType();
4002 if (VT.isVector())
4003 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
4004
4005 if (DAG.shouldOptForSize())
4006 return SDValue();
4007
4008 SDLoc DL(Op);
4009 SDValue Src = Op.getOperand(0);
4010
4011 // Create an integer the size of the mantissa with the MSB set. This and all
4012 // values larger than it don't have any fractional bits so don't need to be
4013 // converted.
4014 const fltSemantics &FltSem = VT.getFltSemantics();
4015 unsigned Precision = APFloat::semanticsPrecision(FltSem);
4016 APFloat MaxVal = APFloat(FltSem);
4017 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
4018 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
4019 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
4020
4021 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
4022 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
4023 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
4024}
4025
4026// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
4027static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
4028 const RISCVSubtarget &Subtarget) {
4029 SDLoc DL(Op);
4030 MVT DstVT = Op.getSimpleValueType();
4031 SDValue Src = Op.getOperand(0);
4032 MVT SrcVT = Src.getSimpleValueType();
4033 assert(SrcVT.isVector() && DstVT.isVector() &&
4034 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
4035 "Unexpected type");
4036
4037 MVT DstContainerVT = DstVT;
4038 MVT SrcContainerVT = SrcVT;
4039
4040 if (DstVT.isFixedLengthVector()) {
4041 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
4042 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
4043 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4044 }
4045
4046 auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
4047
4048 // [b]f16 -> f32
4049 MVT SrcElemType = SrcVT.getVectorElementType();
4050 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
4051 MVT F32VT = SrcContainerVT.changeVectorElementType(MVT::f32);
4052 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, F32VT, Src, Mask, VL);
4053 }
4054
4055 SDValue Res =
4056 DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
4057 DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
4058 Subtarget.getXLenVT()),
4059 VL);
4060
4061 if (!DstVT.isFixedLengthVector())
4062 return Res;
4063
4064 return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
4065}
4066
4067static SDValue
4068getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
4069 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
4070 SDValue Offset, SDValue Mask, SDValue VL,
4071 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4072 if (Passthru.isUndef())
4073 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4074 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
4075 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4076 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
4077}
4078
4079static SDValue
4080getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
4081 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
4082 SDValue VL,
4083 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4084 if (Passthru.isUndef())
4085 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4086 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
4087 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4088 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
4089}
4090
4091struct VIDSequence {
4092 int64_t StepNumerator;
4093 unsigned StepDenominator;
4094 int64_t Addend;
4095};
4096
4097static std::optional<APInt> getExactInteger(const APFloat &APF,
4098 uint32_t BitWidth) {
4099 // We will use a SINT_TO_FP to materialize this constant so we should use a
4100 // signed APSInt here.
4101 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
4102 // We use an arbitrary rounding mode here. If a floating-point is an exact
4103 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
4104 // the rounding mode changes the output value, then it is not an exact
4105 // integer.
4106 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
4107 bool IsExact;
4108 // If it is out of signed integer range, it will return an invalid operation.
4109 // If it is not an exact integer, IsExact is false.
4110 if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
4111 APFloatBase::opInvalidOp) ||
4112 !IsExact)
4113 return std::nullopt;
4114 return ValInt.extractBits(BitWidth, 0);
4115}
4116
4117// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
4118// to the (non-zero) step S and start value X. This can be then lowered as the
4119// RVV sequence (VID * S) + X, for example.
4120// The step S is represented as an integer numerator divided by a positive
4121// denominator. Note that the implementation currently only identifies
4122// sequences in which either the numerator is +/- 1 or the denominator is 1. It
4123// cannot detect 2/3, for example.
4124// Note that this method will also match potentially unappealing index
4125// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
4126// determine whether this is worth generating code for.
4127//
4128// EltSizeInBits is the size of the type that the sequence will be calculated
4129// in, i.e. SEW for build_vectors or XLEN for address calculations.
4130static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
4131 unsigned EltSizeInBits) {
4132 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
4133 if (!cast<BuildVectorSDNode>(Op)->isConstant())
4134 return std::nullopt;
4135 bool IsInteger = Op.getValueType().isInteger();
4136
4137 std::optional<unsigned> SeqStepDenom;
4138 std::optional<APInt> SeqStepNum;
4139 std::optional<APInt> SeqAddend;
4140 std::optional<std::pair<APInt, unsigned>> PrevElt;
4141 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
4142
4143 // First extract the ops into a list of constant integer values. This may not
4144 // be possible for floats if they're not all representable as integers.
4145 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
4146 const unsigned OpSize = Op.getScalarValueSizeInBits();
4147 for (auto [Idx, Elt] : enumerate(Op->op_values())) {
4148 if (Elt.isUndef()) {
4149 Elts[Idx] = std::nullopt;
4150 continue;
4151 }
4152 if (IsInteger) {
4153 Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
4154 } else {
4155 auto ExactInteger =
4156 getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
4157 if (!ExactInteger)
4158 return std::nullopt;
4159 Elts[Idx] = *ExactInteger;
4160 }
4161 }
4162
4163 for (auto [Idx, Elt] : enumerate(Elts)) {
4164 // Assume undef elements match the sequence; we just have to be careful
4165 // when interpolating across them.
4166 if (!Elt)
4167 continue;
4168
4169 if (PrevElt) {
4170 // Calculate the step since the last non-undef element, and ensure
4171 // it's consistent across the entire sequence.
4172 unsigned IdxDiff = Idx - PrevElt->second;
4173 APInt ValDiff = *Elt - PrevElt->first;
4174
4175 // A zero-value value difference means that we're somewhere in the middle
4176 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4177 // step change before evaluating the sequence.
4178 if (ValDiff == 0)
4179 continue;
4180
4181 int64_t Remainder = ValDiff.srem(IdxDiff);
4182 // Normalize the step if it's greater than 1.
4183 if (Remainder != ValDiff.getSExtValue()) {
4184 // The difference must cleanly divide the element span.
4185 if (Remainder != 0)
4186 return std::nullopt;
4187 ValDiff = ValDiff.sdiv(IdxDiff);
4188 IdxDiff = 1;
4189 }
4190
4191 if (!SeqStepNum)
4192 SeqStepNum = ValDiff;
4193 else if (ValDiff != SeqStepNum)
4194 return std::nullopt;
4195
4196 if (!SeqStepDenom)
4197 SeqStepDenom = IdxDiff;
4198 else if (IdxDiff != *SeqStepDenom)
4199 return std::nullopt;
4200 }
4201
4202 // Record this non-undef element for later.
4203 if (!PrevElt || PrevElt->first != *Elt)
4204 PrevElt = std::make_pair(*Elt, Idx);
4205 }
4206
4207 // We need to have logged a step for this to count as a legal index sequence.
4208 if (!SeqStepNum || !SeqStepDenom)
4209 return std::nullopt;
4210
4211 // Loop back through the sequence and validate elements we might have skipped
4212 // while waiting for a valid step. While doing this, log any sequence addend.
4213 for (auto [Idx, Elt] : enumerate(Elts)) {
4214 if (!Elt)
4215 continue;
4216 APInt ExpectedVal =
4217 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4218 *SeqStepNum)
4219 .sdiv(*SeqStepDenom);
4220
4221 APInt Addend = *Elt - ExpectedVal;
4222 if (!SeqAddend)
4223 SeqAddend = Addend;
4224 else if (Addend != SeqAddend)
4225 return std::nullopt;
4226 }
4227
4228 assert(SeqAddend && "Must have an addend if we have a step");
4229
4230 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
4231 SeqAddend->getSExtValue()};
4232}
4233
4234// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4235// and lower it as a VRGATHER_VX_VL from the source vector.
4236static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4237 SelectionDAG &DAG,
4238 const RISCVSubtarget &Subtarget) {
4239 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4240 return SDValue();
4241 SDValue Src = SplatVal.getOperand(0);
4242 // Don't perform this optimization for i1 vectors, or if the element types are
4243 // different
4244 // FIXME: Support i1 vectors, maybe by promoting to i8?
4245 MVT EltTy = VT.getVectorElementType();
4246 if (EltTy == MVT::i1 ||
4247 !DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
4248 return SDValue();
4249 MVT SrcVT = Src.getSimpleValueType();
4250 if (EltTy != SrcVT.getVectorElementType())
4251 return SDValue();
4252 SDValue Idx = SplatVal.getOperand(1);
4253 // The index must be a legal type.
4254 if (Idx.getValueType() != Subtarget.getXLenVT())
4255 return SDValue();
4256
4257 // Check that we know Idx lies within VT
4258 if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
4259 auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
4260 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4261 return SDValue();
4262 }
4263
4264 // Convert fixed length vectors to scalable
4265 MVT ContainerVT = VT;
4266 if (VT.isFixedLengthVector())
4267 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4268
4269 MVT SrcContainerVT = SrcVT;
4270 if (SrcVT.isFixedLengthVector()) {
4271 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
4272 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4273 }
4274
4275 // Put Vec in a VT sized vector
4276 if (SrcContainerVT.getVectorMinNumElements() <
4277 ContainerVT.getVectorMinNumElements())
4278 Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Src, 0);
4279 else
4280 Src = DAG.getExtractSubvector(DL, ContainerVT, Src, 0);
4281
4282 // We checked that Idx fits inside VT earlier
4283 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4284 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Src,
4285 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
4286 if (VT.isFixedLengthVector())
4287 Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
4288 return Gather;
4289}
4290
4291static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4292 const RISCVSubtarget &Subtarget) {
4293 MVT VT = Op.getSimpleValueType();
4294 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4295
4296 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4297
4298 SDLoc DL(Op);
4299 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4300
4301 if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
4302 int64_t StepNumerator = SimpleVID->StepNumerator;
4303 unsigned StepDenominator = SimpleVID->StepDenominator;
4304 int64_t Addend = SimpleVID->Addend;
4305
4306 assert(StepNumerator != 0 && "Invalid step");
4307 bool Negate = false;
4308 int64_t SplatStepVal = StepNumerator;
4309 unsigned StepOpcode = ISD::MUL;
4310 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4311 // anyway as the shift of 63 won't fit in uimm5.
4312 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4313 isPowerOf2_64(std::abs(StepNumerator))) {
4314 Negate = StepNumerator < 0;
4315 StepOpcode = ISD::SHL;
4316 SplatStepVal = Log2_64(std::abs(StepNumerator));
4317 }
4318
4319 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4320 // since it's the immediate value many RVV instructions accept. There is
4321 // no vmul.vi instruction so ensure multiply constant can fit in a
4322 // single addi instruction. For the addend, we allow up to 32 bits..
4323 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
4324 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
4325 isPowerOf2_32(StepDenominator) &&
4326 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
4327 MVT VIDVT =
4328 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4329 MVT VIDContainerVT = getContainerForFixedLengthVector(VIDVT, Subtarget);
4330 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
4331 // Convert right out of the scalable type so we can use standard ISD
4332 // nodes for the rest of the computation. If we used scalable types with
4333 // these, we'd lose the fixed-length vector info and generate worse
4334 // vsetvli code.
4335 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
4336 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4337 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4338 SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
4339 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
4340 }
4341 if (StepDenominator != 1) {
4342 SDValue SplatStep =
4343 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
4344 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
4345 }
4346 if (Addend != 0 || Negate) {
4347 SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
4348 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
4349 VID);
4350 }
4351 if (VT.isFloatingPoint()) {
4352 // TODO: Use vfwcvt to reduce register pressure.
4353 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
4354 }
4355 return VID;
4356 }
4357 }
4358
4359 return SDValue();
4360}
4361
4362/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4363/// which constitute a large proportion of the elements. In such cases we can
4364/// splat a vector with the dominant element and make up the shortfall with
4365/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4366/// Note that this includes vectors of 2 elements by association. The
4367/// upper-most element is the "dominant" one, allowing us to use a splat to
4368/// "insert" the upper element, and an insert of the lower element at position
4369/// 0, which improves codegen.
4370static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4371 const RISCVSubtarget &Subtarget) {
4372 MVT VT = Op.getSimpleValueType();
4373 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4374
4375 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4376
4377 SDLoc DL(Op);
4378 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4379
4380 MVT XLenVT = Subtarget.getXLenVT();
4381 unsigned NumElts = Op.getNumOperands();
4382
4383 SDValue DominantValue;
4384 unsigned MostCommonCount = 0;
4385 DenseMap<SDValue, unsigned> ValueCounts;
4386 unsigned NumUndefElts =
4387 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4388
4389 // Track the number of scalar loads we know we'd be inserting, estimated as
4390 // any non-zero floating-point constant. Other kinds of element are either
4391 // already in registers or are materialized on demand. The threshold at which
4392 // a vector load is more desirable than several scalar materializion and
4393 // vector-insertion instructions is not known.
4394 unsigned NumScalarLoads = 0;
4395
4396 for (SDValue V : Op->op_values()) {
4397 if (V.isUndef())
4398 continue;
4399
4400 unsigned &Count = ValueCounts[V];
4401 if (0 == Count)
4402 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
4403 NumScalarLoads += !CFP->isPosZero();
4404
4405 // Is this value dominant? In case of a tie, prefer the highest element as
4406 // it's cheaper to insert near the beginning of a vector than it is at the
4407 // end.
4408 if (++Count >= MostCommonCount) {
4409 DominantValue = V;
4410 MostCommonCount = Count;
4411 }
4412 }
4413
4414 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4415 unsigned NumDefElts = NumElts - NumUndefElts;
4416 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4417
4418 // Don't perform this optimization when optimizing for size, since
4419 // materializing elements and inserting them tends to cause code bloat.
4420 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4421 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
4422 ((MostCommonCount > DominantValueCountThreshold) ||
4423 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
4424 // Start by splatting the most common element.
4425 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
4426
4427 DenseSet<SDValue> Processed{DominantValue};
4428
4429 // We can handle an insert into the last element (of a splat) via
4430 // v(f)slide1down. This is slightly better than the vslideup insert
4431 // lowering as it avoids the need for a vector group temporary. It
4432 // is also better than using vmerge.vx as it avoids the need to
4433 // materialize the mask in a vector register.
4434 if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
4435 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4436 LastOp != DominantValue) {
4437 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4438 auto OpCode =
4439 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4440 if (!VT.isFloatingPoint())
4441 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
4442 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4443 LastOp, Mask, VL);
4444 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
4445 Processed.insert(LastOp);
4446 }
4447
4448 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
4449 for (const auto &OpIdx : enumerate(Op->ops())) {
4450 const SDValue &V = OpIdx.value();
4451 if (V.isUndef() || !Processed.insert(V).second)
4452 continue;
4453 if (ValueCounts[V] == 1) {
4454 Vec = DAG.getInsertVectorElt(DL, Vec, V, OpIdx.index());
4455 } else {
4456 // Blend in all instances of this value using a VSELECT, using a
4457 // mask where each bit signals whether that element is the one
4458 // we're after.
4459 SmallVector<SDValue> Ops;
4460 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
4461 return DAG.getConstant(V == V1, DL, XLenVT);
4462 });
4463 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
4464 DAG.getBuildVector(SelMaskTy, DL, Ops),
4465 DAG.getSplatBuildVector(VT, DL, V), Vec);
4466 }
4467 }
4468
4469 return Vec;
4470 }
4471
4472 return SDValue();
4473}
4474
4475static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4476 const RISCVSubtarget &Subtarget) {
4477 MVT VT = Op.getSimpleValueType();
4478 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4479
4480 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4481
4482 SDLoc DL(Op);
4483 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4484
4485 MVT XLenVT = Subtarget.getXLenVT();
4486 unsigned NumElts = Op.getNumOperands();
4487
4488 if (VT.getVectorElementType() == MVT::i1) {
4489 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
4490 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
4491 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
4492 }
4493
4494 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
4495 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4496 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
4497 }
4498
4499 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4500 // scalar integer chunks whose bit-width depends on the number of mask
4501 // bits and XLEN.
4502 // First, determine the most appropriate scalar integer type to use. This
4503 // is at most XLenVT, but may be shrunk to a smaller vector element type
4504 // according to the size of the final vector - use i8 chunks rather than
4505 // XLenVT if we're producing a v8i1. This results in more consistent
4506 // codegen across RV32 and RV64.
4507 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
4508 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
4509 // If we have to use more than one INSERT_VECTOR_ELT then this
4510 // optimization is likely to increase code size; avoid performing it in
4511 // such a case. We can use a load from a constant pool in this case.
4512 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4513 return SDValue();
4514 // Now we can create our integer vector type. Note that it may be larger
4515 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4516 unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
4517 MVT IntegerViaVecVT =
4518 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
4519 IntegerViaVecElts);
4520
4521 uint64_t Bits = 0;
4522 unsigned BitPos = 0, IntegerEltIdx = 0;
4523 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4524
4525 for (unsigned I = 0; I < NumElts;) {
4526 SDValue V = Op.getOperand(I);
4527 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4528 Bits |= ((uint64_t)BitValue << BitPos);
4529 ++BitPos;
4530 ++I;
4531
4532 // Once we accumulate enough bits to fill our scalar type or process the
4533 // last element, insert into our vector and clear our accumulated data.
4534 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4535 if (NumViaIntegerBits <= 32)
4536 Bits = SignExtend64<32>(Bits);
4537 SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
4538 Elts[IntegerEltIdx] = Elt;
4539 Bits = 0;
4540 BitPos = 0;
4541 IntegerEltIdx++;
4542 }
4543 }
4544
4545 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
4546
4547 if (NumElts < NumViaIntegerBits) {
4548 // If we're producing a smaller vector than our minimum legal integer
4549 // type, bitcast to the equivalent (known-legal) mask type, and extract
4550 // our final mask.
4551 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4552 Vec = DAG.getBitcast(MVT::v8i1, Vec);
4553 Vec = DAG.getExtractSubvector(DL, VT, Vec, 0);
4554 } else {
4555 // Else we must have produced an integer type with the same size as the
4556 // mask type; bitcast for the final result.
4557 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4558 Vec = DAG.getBitcast(VT, Vec);
4559 }
4560
4561 return Vec;
4562 }
4563
4564 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4565 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4566 : RISCVISD::VMV_V_X_VL;
4567 if (!VT.isFloatingPoint())
4568 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4569 Splat =
4570 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4571 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4572 }
4573
4574 // Try and match index sequences, which we can lower to the vid instruction
4575 // with optional modifications. An all-undef vector is matched by
4576 // getSplatValue, above.
4577 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4578 return Res;
4579
4580 // For very small build_vectors, use a single scalar insert of a constant.
4581 // TODO: Base this on constant rematerialization cost, not size.
4582 const unsigned EltBitSize = VT.getScalarSizeInBits();
4583 if (VT.getSizeInBits() <= 32 &&
4584 ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
4585 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
4586 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4587 "Unexpected sequence type");
4588 // If we can use the original VL with the modified element type, this
4589 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4590 // be moved into InsertVSETVLI?
4591 unsigned ViaVecLen =
4592 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4593 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4594
4595 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4596 uint64_t SplatValue = 0;
4597 // Construct the amalgamated value at this larger vector type.
4598 for (const auto &OpIdx : enumerate(Op->op_values())) {
4599 const auto &SeqV = OpIdx.value();
4600 if (!SeqV.isUndef())
4601 SplatValue |=
4602 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4603 }
4604
4605 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4606 // achieve better constant materializion.
4607 // On RV32, we need to sign-extend to use getSignedConstant.
4608 if (ViaIntVT == MVT::i32)
4609 SplatValue = SignExtend64<32>(SplatValue);
4610
4611 SDValue Vec = DAG.getInsertVectorElt(
4612 DL, DAG.getUNDEF(ViaVecVT),
4613 DAG.getSignedConstant(SplatValue, DL, XLenVT), 0);
4614 if (ViaVecLen != 1)
4615 Vec = DAG.getExtractSubvector(DL, MVT::getVectorVT(ViaIntVT, 1), Vec, 0);
4616 return DAG.getBitcast(VT, Vec);
4617 }
4618
4619
4620 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4621 // when re-interpreted as a vector with a larger element type. For example,
4622 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4623 // could be instead splat as
4624 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4625 // TODO: This optimization could also work on non-constant splats, but it
4626 // would require bit-manipulation instructions to construct the splat value.
4627 SmallVector<SDValue> Sequence;
4628 const auto *BV = cast<BuildVectorSDNode>(Op);
4629 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4630 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
4631 BV->getRepeatedSequence(Sequence) &&
4632 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4633 unsigned SeqLen = Sequence.size();
4634 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
4635 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4636 ViaIntVT == MVT::i64) &&
4637 "Unexpected sequence type");
4638
4639 // If we can use the original VL with the modified element type, this
4640 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4641 // be moved into InsertVSETVLI?
4642 const unsigned RequiredVL = NumElts / SeqLen;
4643 const unsigned ViaVecLen =
4644 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4645 NumElts : RequiredVL;
4646 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4647
4648 unsigned EltIdx = 0;
4649 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4650 uint64_t SplatValue = 0;
4651 // Construct the amalgamated value which can be splatted as this larger
4652 // vector type.
4653 for (const auto &SeqV : Sequence) {
4654 if (!SeqV.isUndef())
4655 SplatValue |=
4656 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4657 EltIdx++;
4658 }
4659
4660 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4661 // achieve better constant materializion.
4662 // On RV32, we need to sign-extend to use getSignedConstant.
4663 if (ViaIntVT == MVT::i32)
4664 SplatValue = SignExtend64<32>(SplatValue);
4665
4666 // Since we can't introduce illegal i64 types at this stage, we can only
4667 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4668 // way we can use RVV instructions to splat.
4669 assert((ViaIntVT.bitsLE(XLenVT) ||
4670 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4671 "Unexpected bitcast sequence");
4672 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
4673 SDValue ViaVL =
4674 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
4675 MVT ViaContainerVT =
4676 getContainerForFixedLengthVector(ViaVecVT, Subtarget);
4677 SDValue Splat =
4678 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
4679 DAG.getUNDEF(ViaContainerVT),
4680 DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
4681 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
4682 if (ViaVecLen != RequiredVL)
4683 Splat = DAG.getExtractSubvector(
4684 DL, MVT::getVectorVT(ViaIntVT, RequiredVL), Splat, 0);
4685 return DAG.getBitcast(VT, Splat);
4686 }
4687 }
4688
4689 // If the number of signbits allows, see if we can lower as a <N x i8>.
4690 // Our main goal here is to reduce LMUL (and thus work) required to
4691 // build the constant, but we will also narrow if the resulting
4692 // narrow vector is known to materialize cheaply.
4693 // TODO: We really should be costing the smaller vector. There are
4694 // profitable cases this misses.
4695 if (EltBitSize > 8 && VT.isInteger() &&
4696 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4697 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4698 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
4699 DL, Op->ops());
4700 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
4701 Source, DAG, Subtarget);
4702 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
4703 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4704 }
4705
4706 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4707 return Res;
4708
4709 // For constant vectors, use generic constant pool lowering. Otherwise,
4710 // we'd have to materialize constants in GPRs just to move them into the
4711 // vector.
4712 return SDValue();
4713}
4714
4715static unsigned getPACKOpcode(unsigned DestBW,
4716 const RISCVSubtarget &Subtarget) {
4717 switch (DestBW) {
4718 default:
4719 llvm_unreachable("Unsupported pack size");
4720 case 16:
4721 return RISCV::PACKH;
4722 case 32:
4723 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4724 case 64:
4725 assert(Subtarget.is64Bit());
4726 return RISCV::PACK;
4727 }
4728}
4729
4730/// Double the element size of the build vector to reduce the number
4731/// of vslide1down in the build vector chain. In the worst case, this
4732/// trades three scalar operations for 1 vector operation. Scalar
4733/// operations are generally lower latency, and for out-of-order cores
4734/// we also benefit from additional parallelism.
4735static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4736 const RISCVSubtarget &Subtarget) {
4737 SDLoc DL(Op);
4738 MVT VT = Op.getSimpleValueType();
4739 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4740 MVT ElemVT = VT.getVectorElementType();
4741 if (!ElemVT.isInteger())
4742 return SDValue();
4743
4744 // TODO: Relax these architectural restrictions, possibly with costing
4745 // of the actual instructions required.
4746 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4747 return SDValue();
4748
4749 unsigned NumElts = VT.getVectorNumElements();
4750 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4751 if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
4752 NumElts % 2 != 0)
4753 return SDValue();
4754
4755 // Produce [B,A] packed into a type twice as wide. Note that all
4756 // scalars are XLenVT, possibly masked (see below).
4757 MVT XLenVT = Subtarget.getXLenVT();
4758 SDValue Mask = DAG.getConstant(
4759 APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
4760 auto pack = [&](SDValue A, SDValue B) {
4761 // Bias the scheduling of the inserted operations to near the
4762 // definition of the element - this tends to reduce register
4763 // pressure overall.
4764 SDLoc ElemDL(B);
4765 if (Subtarget.hasStdExtZbkb())
4766 // Note that we're relying on the high bits of the result being
4767 // don't care. For PACKW, the result is *sign* extended.
4768 return SDValue(
4769 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
4770 ElemDL, XLenVT, A, B),
4771 0);
4772
4773 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
4774 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
4775 SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
4776 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
4777 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
4778 SDNodeFlags::Disjoint);
4779 };
4780
4781 SmallVector<SDValue> NewOperands;
4782 NewOperands.reserve(NumElts / 2);
4783 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4784 NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
4785 assert(NumElts == NewOperands.size() * 2);
4786 MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
4787 MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
4788 return DAG.getNode(ISD::BITCAST, DL, VT,
4789 DAG.getBuildVector(WideVecVT, DL, NewOperands));
4790}
4791
4792static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4793 const RISCVSubtarget &Subtarget) {
4794 MVT VT = Op.getSimpleValueType();
4795 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4796
4797 MVT EltVT = VT.getVectorElementType();
4798 MVT XLenVT = Subtarget.getXLenVT();
4799
4800 SDLoc DL(Op);
4801
4802 if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
4803 if (VT == MVT::v2i16) {
4804 SDValue Lo = DAG.getBitcast(
4805 MVT::v2i16,
4806 DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32));
4807 SDValue Hi = DAG.getBitcast(
4808 MVT::v2i16,
4809 DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32));
4810 return DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16, Lo, Hi);
4811 }
4812
4813 if (VT == MVT::v4i8) {
4814 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
4815 // d))
4816 SDValue Val0 =
4817 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(0));
4818 SDValue Val1 =
4819 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(1));
4820 SDValue Val2 =
4821 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(2));
4822 SDValue Val3 =
4823 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(3));
4824 SDValue Concat1 =
4825 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val0, Val2);
4826 SDValue Concat2 =
4827 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val1, Val3);
4828 SDValue PPairE =
4829 DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v8i8, Concat1, Concat2);
4830
4831 SDValue Lo = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 0);
4832 SDValue Hi = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 4);
4833
4834 return DAG.getBitcast(MVT::v4i8,
4835 DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16,
4836 DAG.getBitcast(MVT::v2i16, Lo),
4837 DAG.getBitcast(MVT::v2i16, Hi)));
4838 }
4839
4840 llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
4841 }
4842
4843 // Proper support for f16 requires Zvfh. bf16 always requires special
4844 // handling. We need to cast the scalar to integer and create an integer
4845 // build_vector.
4846 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
4847 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4848 MVT IVT = VT.changeVectorElementType(MVT::i16);
4849 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4850 for (const auto &[I, U] : enumerate(Op->ops())) {
4851 SDValue Elem = U.get();
4852 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4853 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4854 // Called by LegalizeDAG, we need to use XLenVT operations since we
4855 // can't create illegal types.
4856 if (auto *C = dyn_cast<ConstantFPSDNode>(Elem)) {
4857 // Manually constant fold so the integer build_vector can be lowered
4858 // better. Waiting for DAGCombine will be too late.
4859 APInt V =
4860 C->getValueAPF().bitcastToAPInt().sext(XLenVT.getSizeInBits());
4861 NewOps[I] = DAG.getConstant(V, DL, XLenVT);
4862 } else {
4863 NewOps[I] = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Elem);
4864 }
4865 } else {
4866 // Called by scalar type legalizer, we can use i16.
4867 NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
4868 }
4869 }
4870 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, IVT, NewOps);
4871 return DAG.getBitcast(VT, Res);
4872 }
4873
4874 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
4875 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
4876 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4877
4878 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4879
4880 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4881
4882 if (VT.getVectorElementType() == MVT::i1) {
4883 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4884 // vector type, we have a legal equivalently-sized i8 type, so we can use
4885 // that.
4886 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
4887 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
4888
4889 SDValue WideVec;
4890 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4891 // For a splat, perform a scalar truncate before creating the wider
4892 // vector.
4893 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4894 DAG.getConstant(1, DL, Splat.getValueType()));
4895 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
4896 } else {
4897 SmallVector<SDValue, 8> Ops(Op->op_values());
4898 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
4899 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
4900 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4901 }
4902
4903 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4904 }
4905
4906 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4907 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4908 return Gather;
4909
4910 if (!VT.isFloatingPoint())
4911 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4912
4913 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4914 // pressure at high LMUL.
4915 bool IsScalar = all_of(Op->ops().drop_front(),
4916 [](const SDUse &U) { return U.get().isUndef(); });
4917 unsigned Opc =
4918 VT.isFloatingPoint()
4919 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4920 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4921 Splat =
4922 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4923 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4924 }
4925
4926 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4927 return Res;
4928
4929 // If we're compiling for an exact VLEN value, we can split our work per
4930 // register in the register group.
4931 if (const auto VLen = Subtarget.getRealVLen();
4932 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4933 MVT ElemVT = VT.getVectorElementType();
4934 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4935 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4936 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
4937 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
4938 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4939
4940 // The following semantically builds up a fixed length concat_vector
4941 // of the component build_vectors. We eagerly lower to scalable and
4942 // insert_subvector here to avoid DAG combining it back to a large
4943 // build_vector.
4944 SmallVector<SDValue> BuildVectorOps(Op->ops());
4945 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4946 SDValue Vec = DAG.getUNDEF(ContainerVT);
4947 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4948 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
4949 SDValue SubBV =
4950 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4951 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
4952 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4953 Vec = DAG.getInsertSubvector(DL, Vec, SubBV, InsertIdx);
4954 }
4955 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4956 }
4957
4958 // If we're about to resort to vslide1down (or stack usage), pack our
4959 // elements into the widest scalar type we can. This will force a VL/VTYPE
4960 // toggle, but reduces the critical path, the number of vslide1down ops
4961 // required, and possibly enables scalar folds of the values.
4962 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4963 return Res;
4964
4965 // For m1 vectors, if we have non-undef values in both halves of our vector,
4966 // split the vector into low and high halves, build them separately, then
4967 // use a vselect to combine them. For long vectors, this cuts the critical
4968 // path of the vslide1down sequence in half, and gives us an opportunity
4969 // to special case each half independently. Note that we don't change the
4970 // length of the sub-vectors here, so if both fallback to the generic
4971 // vslide1down path, we should be able to fold the vselect into the final
4972 // vslidedown (for the undef tail) for the first half w/ masking.
4973 unsigned NumElts = VT.getVectorNumElements();
4974 unsigned NumUndefElts =
4975 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4976 unsigned NumDefElts = NumElts - NumUndefElts;
4977 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4978 ContainerVT.bitsLE(RISCVTargetLowering::getM1VT(ContainerVT))) {
4979 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4980 SmallVector<SDValue> MaskVals;
4981 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4982 SubVecAOps.reserve(NumElts);
4983 SubVecBOps.reserve(NumElts);
4984 for (const auto &[Idx, U] : enumerate(Op->ops())) {
4985 SDValue Elem = U.get();
4986 if (Idx < NumElts / 2) {
4987 SubVecAOps.push_back(Elem);
4988 SubVecBOps.push_back(UndefElem);
4989 } else {
4990 SubVecAOps.push_back(UndefElem);
4991 SubVecBOps.push_back(Elem);
4992 }
4993 bool SelectMaskVal = (Idx < NumElts / 2);
4994 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4995 }
4996 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
4997 MaskVals.size() == NumElts);
4998
4999 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
5000 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
5001 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
5002 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5003 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
5004 }
5005
5006 // Cap the cost at a value linear to the number of elements in the vector.
5007 // The default lowering is to use the stack. The vector store + scalar loads
5008 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
5009 // being (at least) linear in LMUL. As a result, using the vslidedown
5010 // lowering for every element ends up being VL*LMUL..
5011 // TODO: Should we be directly costing the stack alternative? Doing so might
5012 // give us a more accurate upper bound.
5013 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
5014
5015 // TODO: unify with TTI getSlideCost.
5016 InstructionCost PerSlideCost = 1;
5017 switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
5018 default: break;
5019 case RISCVVType::LMUL_2:
5020 PerSlideCost = 2;
5021 break;
5022 case RISCVVType::LMUL_4:
5023 PerSlideCost = 4;
5024 break;
5025 case RISCVVType::LMUL_8:
5026 PerSlideCost = 8;
5027 break;
5028 }
5029
5030 // TODO: Should we be using the build instseq then cost + evaluate scheme
5031 // we use for integer constants here?
5032 unsigned UndefCount = 0;
5033 for (const SDValue &V : Op->ops()) {
5034 if (V.isUndef()) {
5035 UndefCount++;
5036 continue;
5037 }
5038 if (UndefCount) {
5039 LinearBudget -= PerSlideCost;
5040 UndefCount = 0;
5041 }
5042 LinearBudget -= PerSlideCost;
5043 }
5044 if (UndefCount) {
5045 LinearBudget -= PerSlideCost;
5046 }
5047
5048 if (LinearBudget < 0)
5049 return SDValue();
5050
5051 assert((!VT.isFloatingPoint() ||
5052 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
5053 "Illegal type which will result in reserved encoding");
5054
5055 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
5056
5057 // General case: splat the first operand and slide other operands down one
5058 // by one to form a vector. Alternatively, if every operand is an
5059 // extraction from element 0 of a vector, we use that vector from the last
5060 // extraction as the start value and slide up instead of slide down. Such that
5061 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
5062 // vslideup of 1 later and eliminate the vector to scalar movement, which is
5063 // something we cannot do with vslide1down/vslidedown.
5064 // Of course, using vslide1up/vslideup might increase the register pressure,
5065 // and that's why we conservatively limit to cases where every operand is an
5066 // extraction from the first element.
5067 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
5068 SDValue EVec;
5069 bool SlideUp = false;
5070 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
5071 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
5072 if (SlideUp)
5073 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
5074 Mask, VL, Policy);
5075 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
5076 Mask, VL, Policy);
5077 };
5078
5079 // The reason we don't use all_of here is because we're also capturing EVec
5080 // from the last non-undef operand. If the std::execution_policy of the
5081 // underlying std::all_of is anything but std::sequenced_policy we might
5082 // capture the wrong EVec.
5083 for (SDValue V : Operands) {
5084 using namespace SDPatternMatch;
5085 SlideUp = V.isUndef() || sd_match(V, m_ExtractElt(m_Value(EVec), m_Zero()));
5086 if (!SlideUp)
5087 break;
5088 }
5089
5090 // Do not slideup if the element type of EVec is different.
5091 if (SlideUp) {
5092 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
5093 MVT ContainerEltVT = ContainerVT.getVectorElementType();
5094 if (EVecEltVT != ContainerEltVT)
5095 SlideUp = false;
5096 }
5097
5098 if (SlideUp) {
5099 MVT EVecContainerVT = EVec.getSimpleValueType();
5100 // Make sure the original vector has scalable vector type.
5101 if (EVecContainerVT.isFixedLengthVector()) {
5102 EVecContainerVT =
5103 getContainerForFixedLengthVector(EVecContainerVT, Subtarget);
5104 EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget);
5105 }
5106
5107 // Adapt EVec's type into ContainerVT.
5108 if (EVecContainerVT.getVectorMinNumElements() <
5109 ContainerVT.getVectorMinNumElements())
5110 EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0);
5111 else
5112 EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0);
5113
5114 // Reverse the elements as we're going to slide up from the last element.
5115 std::reverse(Operands.begin(), Operands.end());
5116 }
5117
5118 SDValue Vec;
5119 UndefCount = 0;
5120 for (SDValue V : Operands) {
5121 if (V.isUndef()) {
5122 UndefCount++;
5123 continue;
5124 }
5125
5126 // Start our sequence with either a TA splat or extract source in the
5127 // hopes that hardware is able to recognize there's no dependency on the
5128 // prior value of our temporary register.
5129 if (!Vec) {
5130 if (SlideUp) {
5131 Vec = EVec;
5132 } else {
5133 Vec = DAG.getSplatVector(VT, DL, V);
5134 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5135 }
5136
5137 UndefCount = 0;
5138 continue;
5139 }
5140
5141 if (UndefCount) {
5142 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5143 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5144 VL);
5145 UndefCount = 0;
5146 }
5147
5148 unsigned Opcode;
5149 if (VT.isFloatingPoint())
5150 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
5151 else
5152 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
5153
5154 if (!VT.isFloatingPoint())
5155 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
5156 Vec = DAG.getNode(Opcode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
5157 V, Mask, VL);
5158 }
5159 if (UndefCount) {
5160 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5161 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5162 VL);
5163 }
5164 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5165}
5166
5167static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5168 SDValue Lo, SDValue Hi, SDValue VL,
5169 SelectionDAG &DAG) {
5170 if (!Passthru)
5171 Passthru = DAG.getUNDEF(VT);
5172 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
5173 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
5174 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
5175 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5176 // node in order to try and match RVV vector/scalar instructions.
5177 if ((LoC >> 31) == HiC)
5178 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5179
5180 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5181 // VL. This can temporarily increase VL if VL less than VLMAX.
5182 if (LoC == HiC) {
5183 SDValue NewVL;
5184 if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
5185 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
5186 else
5187 NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
5188 MVT InterVT =
5189 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
5190 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
5191 DAG.getUNDEF(InterVT), Lo, NewVL);
5192 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
5193 }
5194 }
5195
5196 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5197 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
5198 isa<ConstantSDNode>(Hi.getOperand(1)) &&
5199 Hi.getConstantOperandVal(1) == 31)
5200 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5201
5202 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5203 // even if it might be sign extended.
5204 if (Hi.isUndef())
5205 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5206
5207 // Fall back to a stack store and stride x0 vector load.
5208 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
5209 Hi, VL);
5210}
5211
5212// Called by type legalization to handle splat of i64 on RV32.
5213// FIXME: We can optimize this when the type has sign or zero bits in one
5214// of the halves.
5215static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5216 SDValue Scalar, SDValue VL,
5217 SelectionDAG &DAG) {
5218 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5219 SDValue Lo, Hi;
5220 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
5221 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5222}
5223
5224// This function lowers a splat of a scalar operand Splat with the vector
5225// length VL. It ensures the final sequence is type legal, which is useful when
5226// lowering a splat after type legalization.
5227static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5228 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5229 const RISCVSubtarget &Subtarget) {
5230 bool HasPassthru = Passthru && !Passthru.isUndef();
5231 if (!HasPassthru && !Passthru)
5232 Passthru = DAG.getUNDEF(VT);
5233
5234 MVT EltVT = VT.getVectorElementType();
5235 MVT XLenVT = Subtarget.getXLenVT();
5236
5237 if (VT.isFloatingPoint()) {
5238 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
5239 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5240 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5241 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5242 Scalar = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Scalar);
5243 else
5244 Scalar = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Scalar);
5245 MVT IVT = VT.changeVectorElementType(MVT::i16);
5246 Passthru = DAG.getNode(ISD::BITCAST, DL, IVT, Passthru);
5247 SDValue Splat =
5248 lowerScalarSplat(Passthru, Scalar, VL, IVT, DL, DAG, Subtarget);
5249 return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
5250 }
5251 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
5252 }
5253
5254 // Simplest case is that the operand needs to be promoted to XLenVT.
5255 if (Scalar.getValueType().bitsLE(XLenVT)) {
5256 // If the operand is a constant, sign extend to increase our chances
5257 // of being able to use a .vi instruction. ANY_EXTEND would become a
5258 // a zero extend and the simm5 check in isel would fail.
5259 // FIXME: Should we ignore the upper bits in isel instead?
5260 unsigned ExtOpc =
5261 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5262 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5263 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
5264 }
5265
5266 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5267 "Unexpected scalar for splat lowering!");
5268
5269 if (isOneConstant(VL) && isNullConstant(Scalar))
5270 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
5271 DAG.getConstant(0, DL, XLenVT), VL);
5272
5273 // Otherwise use the more complicated splatting algorithm.
5274 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5275}
5276
5277// This function lowers an insert of a scalar operand Scalar into lane
5278// 0 of the vector regardless of the value of VL. The contents of the
5279// remaining lanes of the result vector are unspecified. VL is assumed
5280// to be non-zero.
5281static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5282 const SDLoc &DL, SelectionDAG &DAG,
5283 const RISCVSubtarget &Subtarget) {
5284 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5285
5286 const MVT XLenVT = Subtarget.getXLenVT();
5287 SDValue Passthru = DAG.getUNDEF(VT);
5288
5289 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5290 isNullConstant(Scalar.getOperand(1))) {
5291 SDValue ExtractedVal = Scalar.getOperand(0);
5292 // The element types must be the same.
5293 if (ExtractedVal.getValueType().getVectorElementType() ==
5294 VT.getVectorElementType()) {
5295 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5296 MVT ExtractedContainerVT = ExtractedVT;
5297 if (ExtractedContainerVT.isFixedLengthVector()) {
5298 ExtractedContainerVT =
5299 getContainerForFixedLengthVector(ExtractedContainerVT, Subtarget);
5300 ExtractedVal = convertToScalableVector(ExtractedContainerVT,
5301 ExtractedVal, DAG, Subtarget);
5302 }
5303 if (ExtractedContainerVT.bitsLE(VT))
5304 return DAG.getInsertSubvector(DL, Passthru, ExtractedVal, 0);
5305 return DAG.getExtractSubvector(DL, VT, ExtractedVal, 0);
5306 }
5307 }
5308
5309 if (VT.isFloatingPoint())
5310 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5311 VL);
5312
5313 // Avoid the tricky legalization cases by falling back to using the
5314 // splat code which already handles it gracefully.
5315 if (!Scalar.getValueType().bitsLE(XLenVT))
5316 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
5317 DAG.getConstant(1, DL, XLenVT),
5318 VT, DL, DAG, Subtarget);
5319
5320 // If the operand is a constant, sign extend to increase our chances
5321 // of being able to use a .vi instruction. ANY_EXTEND would become a
5322 // a zero extend and the simm5 check in isel would fail.
5323 // FIXME: Should we ignore the upper bits in isel instead?
5324 unsigned ExtOpc =
5325 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5326 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5327 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5328 VL);
5329}
5330
5331/// If concat_vector(V1,V2) could be folded away to some existing
5332/// vector source, return it. Note that the source may be larger
5333/// than the requested concat_vector (i.e. a extract_subvector
5334/// might be required.)
5335static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5336 EVT VT = V1.getValueType();
5337 assert(VT == V2.getValueType() && "argument types must match");
5338 // Both input must be extracts.
5339 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5340 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5341 return SDValue();
5342
5343 // Extracting from the same source.
5344 SDValue Src = V1.getOperand(0);
5345 if (Src != V2.getOperand(0) ||
5346 VT.isScalableVector() != Src.getValueType().isScalableVector())
5347 return SDValue();
5348
5349 // The extracts must extract the two halves of the source.
5350 if (V1.getConstantOperandVal(1) != 0 ||
5351 V2.getConstantOperandVal(1) != VT.getVectorMinNumElements())
5352 return SDValue();
5353
5354 return Src;
5355}
5356
5357// Can this shuffle be performed on exactly one (possibly larger) input?
5358static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5359
5360 if (V2.isUndef())
5361 return V1;
5362
5363 unsigned NumElts = VT.getVectorNumElements();
5364 // Src needs to have twice the number of elements.
5365 // TODO: Update shuffle lowering to add the extract subvector
5366 if (SDValue Src = foldConcatVector(V1, V2);
5367 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5368 return Src;
5369
5370 return SDValue();
5371}
5372
5373static bool isLegalVTForZvzipOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5374 MVT ContainerVT = VT;
5375 if (VT.isFixedLengthVector())
5376 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5377 // Determine LMUL of the container vector.
5378 return RISCVTargetLowering::getLMUL(ContainerVT) != RISCVVType::LMUL_8;
5379}
5380
5381/// Is this shuffle interleaving contiguous elements from one vector into the
5382/// even elements and contiguous elements from another vector into the odd
5383/// elements. \p EvenSrc will contain the element that should be in the first
5384/// even element. \p OddSrc will contain the element that should be in the first
5385/// odd element. These can be the first element in a source or the element half
5386/// way through the source.
5387static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5388 int &OddSrc, const RISCVSubtarget &Subtarget) {
5389 // We need to be able to widen elements to the next larger integer type or
5390 // use the vzip instruction at e64.
5391 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
5392 if (!Subtarget.hasStdExtZvzip())
5393 return false;
5394 if (!isLegalVTForZvzipOperand(VT, Subtarget))
5395 return false;
5396 }
5397
5398 int Size = Mask.size();
5399 int NumElts = VT.getVectorNumElements();
5400 assert(Size == (int)NumElts && "Unexpected mask size");
5401
5402 SmallVector<unsigned, 2> StartIndexes;
5403 if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
5404 return false;
5405
5406 EvenSrc = StartIndexes[0];
5407 OddSrc = StartIndexes[1];
5408
5409 // One source should be low half of first vector.
5410 if (EvenSrc != 0 && OddSrc != 0)
5411 return false;
5412
5413 // Subvectors will be subtracted from either at the start of the two input
5414 // vectors, or at the start and middle of the first vector if it's an unary
5415 // interleave.
5416 // In both cases, HalfNumElts will be extracted.
5417 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5418 // we'll create an illegal extract_subvector.
5419 // FIXME: We could support other values using a slidedown first.
5420 int HalfNumElts = NumElts / 2;
5421 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5422}
5423
5424/// Is this mask representing a masked combination of two slides?
5425static bool isMaskedSlidePair(ArrayRef<int> Mask,
5426 std::array<std::pair<int, int>, 2> &SrcInfo) {
5427 if (!llvm::isMaskedSlidePair(Mask, Mask.size(), SrcInfo))
5428 return false;
5429
5430 // Avoid matching vselect idioms
5431 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5432 return false;
5433 // Prefer vslideup as the second instruction, and identity
5434 // only as the initial instruction.
5435 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5436 SrcInfo[1].second == 0)
5437 std::swap(SrcInfo[0], SrcInfo[1]);
5438 assert(SrcInfo[0].first != -1 && "Must find one slide");
5439 return true;
5440}
5441
5442// Exactly matches the semantics of a previously existing custom matcher
5443// to allow migration to new matcher without changing output.
5444static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5445 unsigned NumElts) {
5446 if (SrcInfo[1].first == -1)
5447 return true;
5448 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5449 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5450}
5451
5452static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5453 ArrayRef<int> Mask, unsigned Factor,
5454 bool RequiredPolarity) {
5455 int NumElts = Mask.size();
5456 for (const auto &[Idx, M] : enumerate(Mask)) {
5457 if (M < 0)
5458 continue;
5459 int Src = M >= NumElts;
5460 int Diff = (int)Idx - (M % NumElts);
5461 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5462 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5463 "Must match exactly one of the two slides");
5464 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5465 return false;
5466 }
5467 return true;
5468}
5469
5470/// Given a shuffle which can be represented as a pair of two slides,
5471/// see if it is a pair-even idiom.
5472/// Pair-even is:
5473/// vs2: a0 a1 a2 a3
5474/// vs1: b0 b1 b2 b3
5475/// vd: a0 b0 a2 b2
5476static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5477 ArrayRef<int> Mask, unsigned &Factor) {
5478 Factor = SrcInfo[1].second;
5479 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5480 Mask.size() % Factor == 0 &&
5481 isAlternating(SrcInfo, Mask, Factor, true);
5482}
5483
5484/// Given a shuffle which can be represented as a pair of two slides,
5485/// see if it is a pair-odd idiom.
5486/// Pair-odd is:
5487/// vs2: a0 a1 a2 a3
5488/// vs1: b0 b1 b2 b3
5489/// vd: a1 b1 a3 b3
5490/// Note that the operand order is swapped due to the way we canonicalize
5491/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5492static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5493 ArrayRef<int> Mask, unsigned &Factor) {
5494 Factor = -SrcInfo[1].second;
5495 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5496 Mask.size() % Factor == 0 &&
5497 isAlternating(SrcInfo, Mask, Factor, false);
5498}
5499
5500// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5501// 2, 4, 8 and the integer type Factor-times larger than VT's
5502// element type must be a legal element type.
5503// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5504// -> [p, q, r, s] (Factor=2, Index=1)
5505static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5506 SDValue Src, unsigned Factor,
5507 unsigned Index, SelectionDAG &DAG) {
5508 unsigned EltBits = VT.getScalarSizeInBits();
5509 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5510 MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
5511 SrcEC.divideCoefficientBy(Factor));
5512 MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
5513 SrcEC.divideCoefficientBy(Factor));
5514 Src = DAG.getBitcast(WideSrcVT, Src);
5515
5516 unsigned Shift = Index * EltBits;
5517 SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
5518 DAG.getConstant(Shift, DL, WideSrcVT));
5519 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
5520 MVT CastVT = ResVT.changeVectorElementType(VT.getVectorElementType());
5521 Res = DAG.getBitcast(CastVT, Res);
5522 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
5523}
5524
5525/// Match a single source shuffle which is an identity except that some
5526/// particular element is repeated. This can be lowered as a masked
5527/// vrgather.vi/vx. Note that the two source form of this is handled
5528/// by the recursive splitting logic and doesn't need special handling.
5529static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5530 const RISCVSubtarget &Subtarget,
5531 SelectionDAG &DAG) {
5532
5533 SDLoc DL(SVN);
5534 MVT VT = SVN->getSimpleValueType(0);
5535 SDValue V1 = SVN->getOperand(0);
5536 assert(SVN->getOperand(1).isUndef());
5537 ArrayRef<int> Mask = SVN->getMask();
5538 const unsigned NumElts = VT.getVectorNumElements();
5539 MVT XLenVT = Subtarget.getXLenVT();
5540
5541 std::optional<int> SplatIdx;
5542 for (auto [I, M] : enumerate(Mask)) {
5543 if (M == -1 || I == (unsigned)M)
5544 continue;
5545 if (SplatIdx && *SplatIdx != M)
5546 return SDValue();
5547 SplatIdx = M;
5548 }
5549
5550 if (!SplatIdx)
5551 return SDValue();
5552
5553 SmallVector<SDValue> MaskVals;
5554 for (int MaskIndex : Mask) {
5555 bool SelectMaskVal = MaskIndex == *SplatIdx;
5556 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5557 }
5558 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5559 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
5560 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5561 SDValue Splat = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT),
5562 SmallVector<int>(NumElts, *SplatIdx));
5563 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, Splat, V1);
5564}
5565
5566// Lower the following shuffle to vslidedown.
5567// a)
5568// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5569// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5570// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5571// b)
5572// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5573// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5574// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5575// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5576// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5577// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5578static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5579 SDValue V1, SDValue V2,
5580 ArrayRef<int> Mask,
5581 const RISCVSubtarget &Subtarget,
5582 SelectionDAG &DAG) {
5583 auto findNonEXTRACT_SUBVECTORParent =
5584 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5585 uint64_t Offset = 0;
5586 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5587 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5588 // a scalable vector. But we don't want to match the case.
5589 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
5590 Offset += Parent.getConstantOperandVal(1);
5591 Parent = Parent.getOperand(0);
5592 }
5593 return std::make_pair(Parent, Offset);
5594 };
5595
5596 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5597 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5598
5599 // Extracting from the same source.
5600 SDValue Src = V1Src;
5601 if (Src != V2Src)
5602 return SDValue();
5603
5604 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5605 SmallVector<int, 16> NewMask(Mask);
5606 for (size_t i = 0; i != NewMask.size(); ++i) {
5607 if (NewMask[i] == -1)
5608 continue;
5609
5610 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5611 NewMask[i] = NewMask[i] + V1IndexOffset;
5612 } else {
5613 // Minus NewMask.size() is needed. Otherwise, the b case would be
5614 // <5,6,7,12> instead of <5,6,7,8>.
5615 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5616 }
5617 }
5618
5619 // First index must be known and non-zero. It will be used as the slidedown
5620 // amount.
5621 if (NewMask[0] <= 0)
5622 return SDValue();
5623
5624 // NewMask is also continuous.
5625 for (unsigned i = 1; i != NewMask.size(); ++i)
5626 if (NewMask[i - 1] + 1 != NewMask[i])
5627 return SDValue();
5628
5629 MVT XLenVT = Subtarget.getXLenVT();
5630 MVT SrcVT = Src.getSimpleValueType();
5631 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
5632 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
5633 SDValue Slidedown =
5634 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
5635 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
5636 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
5637 return DAG.getExtractSubvector(
5638 DL, VT, convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), 0);
5639}
5640
5641// Because vslideup leaves the destination elements at the start intact, we can
5642// use it to perform shuffles that insert subvectors:
5643//
5644// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5645// ->
5646// vsetvli zero, 8, e8, mf2, ta, ma
5647// vslideup.vi v8, v9, 4
5648//
5649// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5650// ->
5651// vsetvli zero, 5, e8, mf2, tu, ma
5652// vslideup.v1 v8, v9, 2
5653static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5654 SDValue V1, SDValue V2,
5655 ArrayRef<int> Mask,
5656 const RISCVSubtarget &Subtarget,
5657 SelectionDAG &DAG) {
5658 unsigned NumElts = VT.getVectorNumElements();
5659 int NumSubElts, Index;
5660 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumElts, NumSubElts,
5661 Index))
5662 return SDValue();
5663
5664 bool OpsSwapped = Mask[Index] < (int)NumElts;
5665 SDValue InPlace = OpsSwapped ? V2 : V1;
5666 SDValue ToInsert = OpsSwapped ? V1 : V2;
5667
5668 MVT XLenVT = Subtarget.getXLenVT();
5669 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5670 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
5671 // We slide up by the index that the subvector is being inserted at, and set
5672 // VL to the index + the number of elements being inserted.
5673 unsigned Policy =
5674 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5675 // If the we're adding a suffix to the in place vector, i.e. inserting right
5676 // up to the very end of it, then we don't actually care about the tail.
5677 if (NumSubElts + Index >= (int)NumElts)
5678 Policy |= RISCVVType::TAIL_AGNOSTIC;
5679
5680 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
5681 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
5682 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
5683
5684 SDValue Res;
5685 // If we're inserting into the lowest elements, use a tail undisturbed
5686 // vmv.v.v.
5687 if (Index == 0)
5688 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
5689 VL);
5690 else
5691 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
5692 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
5693 return convertFromScalableVector(VT, Res, DAG, Subtarget);
5694}
5695
5696// A shuffle of shuffles where the final data only is drawn from 2 input ops
5697// can be compressed into a single shuffle
5698static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5699 const RISCVSubtarget &Subtarget,
5700 SelectionDAG &DAG) {
5701 SDValue V1 = SVN->getOperand(0);
5702 SDValue V2 = SVN->getOperand(1);
5703
5704 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5705 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5706 return SDValue();
5707
5708 if (!V1.hasOneUse() || !V2.hasOneUse())
5709 return SDValue();
5710
5711 ArrayRef<int> Mask = SVN->getMask();
5712 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(V1.getNode())->getMask();
5713 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(V2.getNode())->getMask();
5714 unsigned NumElts = Mask.size();
5715 SmallVector<int> NewMask(NumElts, -1);
5716 for (unsigned Idx : seq<unsigned>(NumElts)) {
5717 int Lane = Mask[Idx];
5718 // Don't assign if poison
5719 if (Lane == -1)
5720 continue;
5721 int OrigLane;
5722 bool SecondOp = false;
5723 if ((unsigned)Lane < NumElts) {
5724 OrigLane = V1Mask[Lane];
5725 } else {
5726 OrigLane = V2Mask[Lane - NumElts];
5727 SecondOp = true;
5728 }
5729 if (OrigLane == -1)
5730 continue;
5731 // Don't handle if shuffling from a second operand
5732 if ((unsigned)OrigLane >= NumElts)
5733 return SDValue();
5734 if (SecondOp)
5735 OrigLane += NumElts;
5736 NewMask[Idx] = OrigLane;
5737 }
5738
5739 EVT VT = SVN->getValueType(0);
5740 SDLoc DL(SVN);
5741
5742 return DAG.getVectorShuffle(VT, DL, V1->getOperand(0), V2->getOperand(0),
5743 NewMask);
5744}
5745
5746/// Match v(f)slide1up/down idioms. These operations involve sliding
5747/// N-1 elements to make room for an inserted scalar at one end.
5748static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5749 SDValue V1, SDValue V2,
5750 ArrayRef<int> Mask,
5751 const RISCVSubtarget &Subtarget,
5752 SelectionDAG &DAG) {
5753 bool OpsSwapped = false;
5754 if (!isa<BuildVectorSDNode>(V1)) {
5755 if (!isa<BuildVectorSDNode>(V2))
5756 return SDValue();
5757 std::swap(V1, V2);
5758 OpsSwapped = true;
5759 }
5760 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
5761 if (!Splat)
5762 return SDValue();
5763
5764 // Return true if the mask could describe a slide of Mask.size() - 1
5765 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5766 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5767 const unsigned S = (Offset > 0) ? 0 : -Offset;
5768 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5769 for (unsigned i = S; i != E; ++i)
5770 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5771 return false;
5772 return true;
5773 };
5774
5775 const unsigned NumElts = VT.getVectorNumElements();
5776 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5777 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5778 return SDValue();
5779
5780 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5781 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5782 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5783 return SDValue();
5784
5785 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5786 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5787
5788 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5789 // vslide1{down,up}.vx instead.
5790 if ((VT.getVectorElementType() == MVT::bf16 &&
5791 !Subtarget.hasVInstructionsBF16()) ||
5792 (VT.getVectorElementType() == MVT::f16 &&
5793 !Subtarget.hasVInstructionsF16())) {
5794 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5795 Splat =
5796 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Splat);
5797 V2 = DAG.getBitcast(
5798 IntVT, convertToScalableVector(ContainerVT, V2, DAG, Subtarget));
5799 SDValue Vec = DAG.getNode(
5800 IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5801 IntVT, DAG.getUNDEF(IntVT), V2, Splat, TrueMask, VL);
5802 Vec = DAG.getBitcast(ContainerVT, Vec);
5803 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5804 }
5805
5806 auto OpCode = IsVSlidedown ?
5807 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5808 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5809 if (!VT.isFloatingPoint())
5810 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
5811 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
5812 DAG.getUNDEF(ContainerVT),
5813 convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
5814 Splat, TrueMask, VL);
5815 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5816}
5817
5818/// Match a mask which "spreads" the leading elements of a vector evenly
5819/// across the result. Factor is the spread amount, and Index is the
5820/// offset applied. (on success, Index < Factor) This is the inverse
5821/// of a deinterleave with the same Factor and Index. This is analogous
5822/// to an interleave, except that all but one lane is undef.
5823bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5824 unsigned &Index) {
5825 SmallVector<bool> LaneIsUndef(Factor, true);
5826 for (unsigned i = 0; i < Mask.size(); i++)
5827 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5828
5829 bool Found = false;
5830 for (unsigned i = 0; i < Factor; i++) {
5831 if (LaneIsUndef[i])
5832 continue;
5833 if (Found)
5834 return false;
5835 Index = i;
5836 Found = true;
5837 }
5838 if (!Found)
5839 return false;
5840
5841 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5842 unsigned j = i * Factor + Index;
5843 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5844 return false;
5845 }
5846 return true;
5847}
5848
5849static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
5850 const SDLoc &DL, SelectionDAG &DAG,
5851 const RISCVSubtarget &Subtarget) {
5852 assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
5853 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5854
5855 MVT VT = Op0.getSimpleValueType();
5856 MVT IntVT = VT.changeVectorElementTypeToInteger();
5857 Op0 = DAG.getBitcast(IntVT, Op0);
5858 Op1 = DAG.getBitcast(IntVT, Op1);
5859
5860 MVT ContainerVT = IntVT;
5861 if (VT.isFixedLengthVector()) {
5862 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5863 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5864 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5865 }
5866
5867 MVT InnerVT = ContainerVT;
5868 auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
5869
5870 SDValue Passthru = DAG.getUNDEF(InnerVT);
5871 SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
5872 if (IntVT.isFixedLengthVector())
5873 Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
5874 Res = DAG.getBitcast(VT, Res);
5875 return Res;
5876}
5877
5878static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
5879 SelectionDAG &DAG,
5880 const RISCVSubtarget &Subtarget) {
5881 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5882 MVT VT = Op0.getSimpleValueType();
5883 MVT IntVT = VT.changeVectorElementTypeToInteger();
5884 Op0 = DAG.getBitcast(IntVT, Op0);
5885 Op1 = DAG.getBitcast(IntVT, Op1);
5886 MVT ContainerVT = IntVT;
5887 if (VT.isFixedLengthVector()) {
5888 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5889 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5890 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5891 }
5892 MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
5893 auto [Mask, VL] = getDefaultVLOps(IntVT, ContainerVT, DL, DAG, Subtarget);
5894 SDValue Passthru = DAG.getUNDEF(ResVT);
5895 SDValue Res =
5896 DAG.getNode(RISCVISD::VZIP_VL, DL, ResVT, Op0, Op1, Passthru, Mask, VL);
5897 if (IntVT.isFixedLengthVector())
5898 Res = convertFromScalableVector(IntVT.getDoubleNumVectorElementsVT(), Res,
5899 DAG, Subtarget);
5900 Res = DAG.getBitcast(VT.getDoubleNumVectorElementsVT(), Res);
5901 return Res;
5902}
5903
5904static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
5905 SelectionDAG &DAG,
5906 const RISCVSubtarget &Subtarget) {
5907 assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
5908 MVT VT = Op.getSimpleValueType();
5909 assert(VT.getVectorMinNumElements() >= 2);
5910
5911 MVT IntVT = VT.changeVectorElementTypeToInteger();
5912 Op = DAG.getBitcast(IntVT, Op);
5913 MVT ContainerVT = IntVT;
5914 if (VT.isFixedLengthVector()) {
5915 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5916 // For E64 with LMUL <= 1, we can't represent a smaller fractional LMUL for
5917 // the result (LMUL <= 1/2 is not valid for E64). We must widen the input
5918 // container to at least LMUL=2 so the result can be LMUL=1.
5919 if (ContainerVT.getVectorElementType() == MVT::i64 &&
5920 RISCVTargetLowering::getLMUL(ContainerVT) == RISCVVType::LMUL_1) {
5921 ContainerVT = MVT::getScalableVectorVT(MVT::i64, 2);
5922 }
5923 Op = convertToScalableVector(ContainerVT, Op, DAG, Subtarget);
5924 }
5925
5926 MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
5927 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5928 MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
5929 auto [Mask, VL] = getDefaultVLOps(ResVT, ResVT, DL, DAG, Subtarget);
5930 if (VT.isFixedLengthVector())
5931 VL = DAG.getConstant(VT.getVectorNumElements() / 2, DL,
5932 Subtarget.getXLenVT());
5933 SDValue Passthru = DAG.getUNDEF(ResVT);
5934 SDValue Res = DAG.getNode(Opc, DL, ResVT, Op, Passthru, Mask, VL);
5935 if (HalfIntVT.isFixedLengthVector())
5936 Res = convertFromScalableVector(HalfIntVT, Res, DAG, Subtarget);
5937 Res = DAG.getBitcast(HalfVT, Res);
5938 return Res;
5939}
5940
5941// Given a vector a, b, c, d return a vector Factor times longer
5942// with Factor-1 undef's between elements. Ex:
5943// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5944// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5945static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5946 const SDLoc &DL, SelectionDAG &DAG) {
5947
5948 MVT VT = V.getSimpleValueType();
5949 unsigned EltBits = VT.getScalarSizeInBits();
5950 ElementCount EC = VT.getVectorElementCount();
5951 V = DAG.getBitcast(VT.changeTypeToInteger(), V);
5952
5953 MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
5954
5955 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
5956 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5957 // allow the SHL to fold away if Index is 0.
5958 if (Index != 0)
5959 Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
5960 DAG.getConstant(EltBits * Index, DL, WideVT));
5961 // Make sure to use original element type
5962 MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
5963 EC.multiplyCoefficientBy(Factor));
5964 return DAG.getBitcast(ResultVT, Result);
5965}
5966
5967// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
5968// to create an interleaved vector of <[vscale x] n*2 x ty>.
5969// This requires that the size of ty is less than the subtarget's maximum ELEN.
5970static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
5971 const SDLoc &DL, SelectionDAG &DAG,
5972 const RISCVSubtarget &Subtarget) {
5973
5974 // FIXME: Not only does this optimize the code, it fixes some correctness
5975 // issues because MIR does not have freeze.
5976 if (EvenV.isUndef())
5977 return getWideningSpread(OddV, 2, 1, DL, DAG);
5978 if (OddV.isUndef())
5979 return getWideningSpread(EvenV, 2, 0, DL, DAG);
5980
5981 MVT VecVT = EvenV.getSimpleValueType();
5982 MVT VecContainerVT = VecVT; // <vscale x n x ty>
5983 // Convert fixed vectors to scalable if needed
5984 if (VecContainerVT.isFixedLengthVector()) {
5985 VecContainerVT = getContainerForFixedLengthVector(VecVT, Subtarget);
5986 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
5987 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
5988 }
5989
5990 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
5991
5992 // We're working with a vector of the same size as the resulting
5993 // interleaved vector, but with half the number of elements and
5994 // twice the SEW (Hence the restriction on not using the maximum
5995 // ELEN)
5996 MVT WideVT =
5997 MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
5998 VecVT.getVectorElementCount());
5999 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
6000 if (WideContainerVT.isFixedLengthVector())
6001 WideContainerVT = getContainerForFixedLengthVector(WideVT, Subtarget);
6002
6003 // Bitcast the input vectors to integers in case they are FP
6004 VecContainerVT = VecContainerVT.changeTypeToInteger();
6005 EvenV = DAG.getBitcast(VecContainerVT, EvenV);
6006 OddV = DAG.getBitcast(VecContainerVT, OddV);
6007
6008 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
6009 SDValue Passthru = DAG.getUNDEF(WideContainerVT);
6010
6011 SDValue Interleaved;
6012 if (Subtarget.hasStdExtZvbb()) {
6013 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
6014 SDValue OffsetVec =
6015 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
6016 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
6017 OffsetVec, Passthru, Mask, VL);
6018 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
6019 Interleaved, EvenV, Passthru, Mask, VL);
6020 } else {
6021 // FIXME: We should freeze the odd vector here. We already handled the case
6022 // of provably undef/poison above.
6023
6024 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
6025 // vwaddu.vv
6026 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
6027 OddV, Passthru, Mask, VL);
6028
6029 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
6030 SDValue AllOnesVec = DAG.getSplatVector(
6031 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
6032 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
6033 OddV, AllOnesVec, Passthru, Mask, VL);
6034
6035 // Add the two together so we get
6036 // (OddV * 0xff...ff) + (OddV + EvenV)
6037 // = (OddV * 0x100...00) + EvenV
6038 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
6039 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
6040 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
6041 Interleaved, OddsMul, Passthru, Mask, VL);
6042 }
6043
6044 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
6045 MVT ResultContainerVT = MVT::getVectorVT(
6046 VecVT.getVectorElementType(), // Make sure to use original type
6047 VecContainerVT.getVectorElementCount().multiplyCoefficientBy(2));
6048 Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
6049
6050 // Convert back to a fixed vector if needed
6051 MVT ResultVT =
6052 MVT::getVectorVT(VecVT.getVectorElementType(),
6053 VecVT.getVectorElementCount().multiplyCoefficientBy(2));
6054 if (ResultVT.isFixedLengthVector())
6055 Interleaved =
6056 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
6057
6058 return Interleaved;
6059}
6060
6061// If we have a vector of bits that we want to reverse, we can use a vbrev on a
6062// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
6063static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
6064 SelectionDAG &DAG,
6065 const RISCVSubtarget &Subtarget) {
6066 SDLoc DL(SVN);
6067 MVT VT = SVN->getSimpleValueType(0);
6068 SDValue V = SVN->getOperand(0);
6069 unsigned NumElts = VT.getVectorNumElements();
6070
6071 assert(VT.getVectorElementType() == MVT::i1);
6072
6073 if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
6074 SVN->getMask().size()) ||
6075 !SVN->getOperand(1).isUndef())
6076 return SDValue();
6077
6078 unsigned ViaEltSize = std::max((uint64_t)8, PowerOf2Ceil(NumElts));
6079 EVT ViaVT = EVT::getVectorVT(
6080 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
6081 EVT ViaBitVT =
6082 EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
6083
6084 // If we don't have zvbb or the larger element type > ELEN, the operation will
6085 // be illegal.
6086 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
6087 ViaVT) ||
6088 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
6089 return SDValue();
6090
6091 // If the bit vector doesn't fit exactly into the larger element type, we need
6092 // to insert it into the larger vector and then shift up the reversed bits
6093 // afterwards to get rid of the gap introduced.
6094 if (ViaEltSize > NumElts)
6095 V = DAG.getInsertSubvector(DL, DAG.getUNDEF(ViaBitVT), V, 0);
6096
6097 SDValue Res =
6098 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
6099
6100 // Shift up the reversed bits if the vector didn't exactly fit into the larger
6101 // element type.
6102 if (ViaEltSize > NumElts)
6103 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
6104 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
6105
6106 Res = DAG.getBitcast(ViaBitVT, Res);
6107
6108 if (ViaEltSize > NumElts)
6109 Res = DAG.getExtractSubvector(DL, VT, Res, 0);
6110 return Res;
6111}
6112
6113static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
6114 const RISCVSubtarget &Subtarget,
6115 MVT &RotateVT, unsigned &RotateAmt) {
6116 unsigned NumElts = VT.getVectorNumElements();
6117 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6118 unsigned NumSubElts;
6119 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, 2,
6120 NumElts, NumSubElts, RotateAmt))
6121 return false;
6122 RotateVT = MVT::getVectorVT(MVT::getIntegerVT(EltSizeInBits * NumSubElts),
6123 NumElts / NumSubElts);
6124
6125 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
6126 return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
6127}
6128
6129// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
6130// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
6131// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
6132static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
6133 SelectionDAG &DAG,
6134 const RISCVSubtarget &Subtarget) {
6135 SDLoc DL(SVN);
6136
6137 EVT VT = SVN->getValueType(0);
6138 unsigned RotateAmt;
6139 MVT RotateVT;
6140 if (!isLegalBitRotate(SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
6141 return SDValue();
6142
6143 SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
6144
6145 SDValue Rotate;
6146 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
6147 // so canonicalize to vrev8.
6148 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
6149 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
6150 else
6151 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
6152 DAG.getConstant(RotateAmt, DL, RotateVT));
6153
6154 return DAG.getBitcast(VT, Rotate);
6155}
6156
6157// If compiling with an exactly known VLEN, see if we can split a
6158// shuffle on m2 or larger into a small number of m1 sized shuffles
6159// which write each destination registers exactly once.
6160static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
6161 SelectionDAG &DAG,
6162 const RISCVSubtarget &Subtarget) {
6163 SDLoc DL(SVN);
6164 MVT VT = SVN->getSimpleValueType(0);
6165 SDValue V1 = SVN->getOperand(0);
6166 SDValue V2 = SVN->getOperand(1);
6167 ArrayRef<int> Mask = SVN->getMask();
6168
6169 // If we don't know exact data layout, not much we can do. If this
6170 // is already m1 or smaller, no point in splitting further.
6171 const auto VLen = Subtarget.getRealVLen();
6172 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
6173 return SDValue();
6174
6175 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
6176 // expansion for.
6177 unsigned RotateAmt;
6178 MVT RotateVT;
6179 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
6180 return SDValue();
6181
6182 MVT ElemVT = VT.getVectorElementType();
6183 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
6184
6185 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
6186 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
6187 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
6188 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
6189 unsigned NumOpElts = M1VT.getVectorMinNumElements();
6190 unsigned NumElts = ContainerVT.getVectorMinNumElements();
6191 unsigned NumOfSrcRegs = NumElts / NumOpElts;
6192 unsigned NumOfDestRegs = NumElts / NumOpElts;
6193 // The following semantically builds up a fixed length concat_vector
6194 // of the component shuffle_vectors. We eagerly lower to scalable here
6195 // to avoid DAG combining it back to a large shuffle_vector again.
6196 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6197 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
6198 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
6199 Operands;
6200 processShuffleMasks(
6201 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
6202 [&]() { Operands.emplace_back(); },
6203 [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
6204 Operands.emplace_back().emplace_back(SrcVecIdx, UINT_MAX,
6205 SmallVector<int>(SrcSubMask));
6206 },
6207 [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
6208 if (NewReg)
6209 Operands.emplace_back();
6210 Operands.back().emplace_back(Idx1, Idx2, SmallVector<int>(SrcSubMask));
6211 });
6212 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
6213 // Note: check that we do not emit too many shuffles here to prevent code
6214 // size explosion.
6215 // TODO: investigate, if it can be improved by extra analysis of the masks to
6216 // check if the code is more profitable.
6217 unsigned NumShuffles = std::accumulate(
6218 Operands.begin(), Operands.end(), 0u,
6219 [&](unsigned N,
6220 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
6221 if (Data.empty())
6222 return N;
6223 N += Data.size();
6224 for (const auto &P : Data) {
6225 unsigned Idx2 = std::get<1>(P);
6226 ArrayRef<int> Mask = std::get<2>(P);
6227 if (Idx2 != UINT_MAX)
6228 ++N;
6229 else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
6230 --N;
6231 }
6232 return N;
6233 });
6234 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6235 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6236 return SDValue();
6237 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6238 SDValue SubVec = DAG.getExtractSubvector(DL, M1VT, SrcVec, ExtractIdx);
6239 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
6240 return SubVec;
6241 };
6242 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6243 ArrayRef<int> Mask) {
6244 SDValue SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec1, SubVec2, Mask);
6245 return SubVec;
6246 };
6247 SDValue Vec = DAG.getUNDEF(ContainerVT);
6248 for (auto [I, Data] : enumerate(Operands)) {
6249 if (Data.empty())
6250 continue;
6251 SmallDenseMap<unsigned, SDValue, 4> Values;
6252 for (unsigned I : seq<unsigned>(Data.size())) {
6253 const auto &[Idx1, Idx2, _] = Data[I];
6254 // If the shuffle contains permutation of odd number of elements,
6255 // Idx1 might be used already in the first iteration.
6256 //
6257 // Idx1 = shuffle Idx1, Idx2
6258 // Idx1 = shuffle Idx1, Idx3
6259 SDValue &V = Values.try_emplace(Idx1).first->getSecond();
6260 if (!V)
6261 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6262 (Idx1 % NumOfSrcRegs) * NumOpElts);
6263 if (Idx2 != UINT_MAX) {
6264 SDValue &V = Values.try_emplace(Idx2).first->getSecond();
6265 if (!V)
6266 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6267 (Idx2 % NumOfSrcRegs) * NumOpElts);
6268 }
6269 }
6270 SDValue V;
6271 for (const auto &[Idx1, Idx2, Mask] : Data) {
6272 SDValue V1 = Values.at(Idx1);
6273 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Idx2);
6274 V = PerformShuffle(V1, V2, Mask);
6275 Values[Idx1] = V;
6276 }
6277
6278 unsigned InsertIdx = I * NumOpElts;
6279 V = convertToScalableVector(M1VT, V, DAG, Subtarget);
6280 Vec = DAG.getInsertSubvector(DL, Vec, V, InsertIdx);
6281 }
6282 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
6283}
6284
6285// Matches a subset of compress masks with a contiguous prefix of output
6286// elements. This could be extended to allow gaps by deciding which
6287// source elements to spuriously demand.
6288static bool isCompressMask(ArrayRef<int> Mask) {
6289 int Last = -1;
6290 bool SawUndef = false;
6291 for (const auto &[Idx, M] : enumerate(Mask)) {
6292 if (M == -1) {
6293 SawUndef = true;
6294 continue;
6295 }
6296 if (SawUndef)
6297 return false;
6298 if (Idx > (unsigned)M)
6299 return false;
6300 if (M <= Last)
6301 return false;
6302 Last = M;
6303 }
6304 return true;
6305}
6306
6307/// Given a shuffle where the indices are disjoint between the two sources,
6308/// e.g.:
6309///
6310/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6311///
6312/// Merge the two sources into one and do a single source shuffle:
6313///
6314/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6315/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6316///
6317/// A vselect will either be merged into a masked instruction or be lowered as a
6318/// vmerge.vvm, which is cheaper than a vrgather.vv.
6319static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6320 SelectionDAG &DAG,
6321 const RISCVSubtarget &Subtarget) {
6322 MVT VT = SVN->getSimpleValueType(0);
6323 MVT XLenVT = Subtarget.getXLenVT();
6324 SDLoc DL(SVN);
6325
6326 const ArrayRef<int> Mask = SVN->getMask();
6327
6328 // Work out which source each lane will come from.
6329 SmallVector<int, 16> Srcs(Mask.size(), -1);
6330
6331 for (int Idx : Mask) {
6332 if (Idx == -1)
6333 continue;
6334 unsigned SrcIdx = Idx % Mask.size();
6335 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6336 if (Srcs[SrcIdx] == -1)
6337 // Mark this source as using this lane.
6338 Srcs[SrcIdx] = Src;
6339 else if (Srcs[SrcIdx] != Src)
6340 // The other source is using this lane: not disjoint.
6341 return SDValue();
6342 }
6343
6344 SmallVector<SDValue> SelectMaskVals;
6345 for (int Lane : Srcs) {
6346 if (Lane == -1)
6347 SelectMaskVals.push_back(DAG.getUNDEF(XLenVT));
6348 else
6349 SelectMaskVals.push_back(DAG.getConstant(Lane ? 0 : 1, DL, XLenVT));
6350 }
6351 MVT MaskVT = VT.changeVectorElementType(MVT::i1);
6352 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, SelectMaskVals);
6353 SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
6354 SVN->getOperand(0), SVN->getOperand(1));
6355
6356 // Move all indices relative to the first source.
6357 SmallVector<int> NewMask(Mask.size());
6358 for (unsigned I = 0; I < Mask.size(); I++) {
6359 if (Mask[I] == -1)
6360 NewMask[I] = -1;
6361 else
6362 NewMask[I] = Mask[I] % Mask.size();
6363 }
6364
6365 return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
6366}
6367
6368/// Is this mask local (i.e. elements only move within their local span), and
6369/// repeating (that is, the same rearrangement is being done within each span)?
6370static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6371 // Require a prefix from the original mask until the consumer code
6372 // is adjusted to rewrite the mask instead of just taking a prefix.
6373 for (auto [I, M] : enumerate(Mask)) {
6374 if (M == -1)
6375 continue;
6376 if ((M / Span) != (int)(I / Span))
6377 return false;
6378 int SpanIdx = I % Span;
6379 int Expected = M % Span;
6380 if (Mask[SpanIdx] != Expected)
6381 return false;
6382 }
6383 return true;
6384}
6385
6386/// Is this mask only using elements from the first span of the input?
6387static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6388 return all_of(Mask, [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6389}
6390
6391/// Return true for a mask which performs an arbitrary shuffle within the first
6392/// span, and then repeats that same result across all remaining spans. Note
6393/// that this doesn't check if all the inputs come from a single span!
6394static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6395 // Require a prefix from the original mask until the consumer code
6396 // is adjusted to rewrite the mask instead of just taking a prefix.
6397 for (auto [I, M] : enumerate(Mask)) {
6398 if (M == -1)
6399 continue;
6400 int SpanIdx = I % Span;
6401 if (Mask[SpanIdx] != M)
6402 return false;
6403 }
6404 return true;
6405}
6406
6407/// Try to widen element type to get a new mask value for a better permutation
6408/// sequence. This doesn't try to inspect the widened mask for profitability;
6409/// we speculate the widened form is equal or better. This has the effect of
6410/// reducing mask constant sizes - allowing cheaper materialization sequences
6411/// - and index sequence sizes - reducing register pressure and materialization
6412/// cost, at the cost of (possibly) an extra VTYPE toggle.
6413static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6414 SDLoc DL(Op);
6415 MVT VT = Op.getSimpleValueType();
6416 MVT ScalarVT = VT.getVectorElementType();
6417 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6418 SDValue V0 = Op.getOperand(0);
6419 SDValue V1 = Op.getOperand(1);
6420 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
6421
6422 // Avoid wasted work leading to isTypeLegal check failing below
6423 if (ElementSize > 32)
6424 return SDValue();
6425
6426 SmallVector<int, 8> NewMask;
6427 if (!widenShuffleMaskElts(Mask, NewMask))
6428 return SDValue();
6429
6430 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
6431 : MVT::getIntegerVT(ElementSize * 2);
6432 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
6433 if (!DAG.getTargetLoweringInfo().isTypeLegal(NewVT))
6434 return SDValue();
6435 V0 = DAG.getBitcast(NewVT, V0);
6436 V1 = DAG.getBitcast(NewVT, V1);
6437 return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
6438}
6439
6440// Match an interleave shuffle that forms a P-extension packed zip:
6441// <a0, b0, a1, b1, ...> -> zip*p/wzip*p
6442static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
6443 const RISCVSubtarget &Subtarget,
6444 SelectionDAG &DAG) {
6445 SDValue V1 = SVN->getOperand(0);
6446 SDValue V2 = SVN->getOperand(1);
6447 SDLoc DL(SVN);
6448 MVT VT = SVN->getSimpleValueType(0);
6449 unsigned NumElts = VT.getVectorNumElements();
6450 ArrayRef<int> Mask = SVN->getMask();
6451
6452 if (VT != MVT::v8i8 && VT != MVT::v4i16)
6453 return SDValue();
6454
6455 SmallVector<unsigned, 2> StartIndexes;
6456 if (!V2.isUndef() &&
6457 ShuffleVectorInst::isInterleaveMask(Mask, 2, NumElts * 2, StartIndexes)) {
6458 unsigned EvenSrc = StartIndexes[0];
6459 unsigned OddSrc = StartIndexes[1];
6460 if (EvenSrc == 0 && OddSrc == NumElts) {
6461 if (Subtarget.is64Bit())
6462 return DAG.getNode(RISCVISD::PZIP, DL, VT, V1, V2);
6463 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6464 V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
6465 V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
6466 return DAG.getNode(RISCVISD::PWZIP, DL, VT, V1, V2);
6467 }
6468 if (EvenSrc == NumElts && OddSrc == 0) {
6469 if (Subtarget.is64Bit())
6470 return DAG.getNode(RISCVISD::PZIP, DL, VT, V2, V1);
6471 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6472 V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
6473 V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
6474 return DAG.getNode(RISCVISD::PWZIP, DL, VT, V2, V1);
6475 }
6476 }
6477
6478 return SDValue();
6479}
6480
6481// Match a deinterleave shuffle that forms a P-extension packed unzip:
6482// <a0, a2, ..., b0, b2, ...> -> unzip*p
6483// <a1, a3, ..., b1, b3, ...> -> unzip*hp
6484static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
6485 SelectionDAG &DAG, bool IsRV64) {
6486 MVT VT = SVN->getSimpleValueType(0);
6487 if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
6488 return SDValue();
6489
6490 SDValue V1 = SVN->getOperand(0);
6491 SDValue V2 = SVN->getOperand(1);
6492 SDLoc DL(SVN);
6493 ArrayRef<int> Mask = SVN->getMask();
6494
6495 unsigned Index = 0;
6496 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index))
6497 return SDValue();
6498
6499 unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
6500 return DAG.getNode(Opc, DL, VT, V1, V2);
6501}
6502
6503// Match the packed zero-extend shuffle mask <0, N, 2, N+2, ...>: even result
6504// lanes keep operand 0's even lanes and odd result lanes come from operand 1.
6505// The odd lanes may select any element of operand 1, which is looser than a
6506// strict pair-even mask; DAGCombiner::XformToShuffleWithZero forms exactly this
6507// from a packed zero-extend `and`, leaving each zeroed lane at its own
6508// position, and lowerVECTOR_SHUFFLEAsPPair then forms the PPAIRE. Undef lanes
6509// always match.
6510static bool isPackedZExtShuffleMask(ArrayRef<int> Mask) {
6511 unsigned NumElts = Mask.size();
6512 if (NumElts % 2 != 0)
6513 return false;
6514 for (unsigned I = 0; I != NumElts / 2; ++I)
6515 if ((Mask[2 * I] >= 0 && Mask[2 * I] != (int)(2 * I)) ||
6516 (Mask[2 * I + 1] >= 0 && Mask[2 * I + 1] < (int)NumElts))
6517 return false;
6518 return true;
6519}
6520
6521// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
6522// it to an RV32 P narrowing shift on the concatenated source.
6523static SDValue
6524lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
6525 const RISCVSubtarget &Subtarget,
6526 SelectionDAG &DAG) {
6527 MVT VT = SVN->getSimpleValueType(0);
6528 if (Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
6529 return SDValue();
6530
6531 SDValue V1 = SVN->getOperand(0);
6532 SDValue V2 = SVN->getOperand(1);
6533 SDLoc DL(SVN);
6534 unsigned NumElts = VT.getVectorNumElements();
6535
6536 SDValue Src = foldConcatVector(V1, V2);
6537 if (!Src) {
6538 MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
6539 Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, V1, V2);
6540 }
6541
6542 // The source vector should be twice the size.
6543 if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
6544 return SDValue();
6545
6546 unsigned Index = 0;
6547 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(SVN->getMask(), 2, Index))
6548 return SDValue();
6549
6550 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
6551 return DAG.getNode(RISCVISD::PNSRL, DL, VT, Src,
6552 DAG.getConstant(Index * EltBits, DL, MVT::i32));
6553}
6554
6555// Match a strided-interleave shuffle that forms a P-extension packed pair:
6556// <a0, b0, a2, b2, ...> -> ppaire.*
6557// <a0, b1, a2, b3, ...> -> ppaireo.*
6558// <a1, b0, a3, b2, ...> -> ppairoe.*
6559// <a1, b1, a3, b3, ...> -> ppairo.*
6560static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
6561 SelectionDAG &DAG) {
6562 MVT VT = SVN->getSimpleValueType(0);
6563 if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v4i16)
6564 return SDValue();
6565
6566 SDValue V1 = SVN->getOperand(0);
6567 SDValue V2 = SVN->getOperand(1);
6568 SDLoc DL(SVN);
6569 unsigned NumElts = VT.getVectorNumElements();
6570 ArrayRef<int> Mask = SVN->getMask();
6571
6572 // A splat operand's lanes are all equal, so a lane selecting from it matches
6573 // any of its positions. This covers the zero operand XformToShuffleWithZero
6574 // forms for a packed zero-extend, which keeps each zeroed lane at its own
6575 // position rather than the strided one.
6576 bool V1IsSplat = DAG.isSplatValue(V1);
6577 bool V2IsSplat = DAG.isSplatValue(V2);
6578
6579 // Walk the mask once, tracking the operand feeding the destination's even
6580 // lanes (index 0) and the operand feeding its odd lanes (index 1) — either
6581 // may turn out to be V1 or V2 — along with whether each pulls the even or
6582 // odd element out of its pair. All even (resp. odd) lanes must agree on
6583 // both the operand and the parity used; a splat operand's lanes are all
6584 // equal so it never constrains the parity.
6585 SDValue Src[2];
6586 std::optional<bool> Parity[2];
6587 for (unsigned I = 0; I != NumElts; ++I) {
6588 int M = Mask[I];
6589 if (M < 0)
6590 continue;
6591 unsigned Lane = I % 2;
6592 bool FromV1 = (unsigned)M < NumElts;
6593 SDValue Cand = FromV1 ? V1 : V2;
6594 unsigned Local = (unsigned)M % NumElts;
6595 if (!Src[Lane])
6596 Src[Lane] = Cand;
6597 else if (Src[Lane] != Cand)
6598 return SDValue();
6599
6600 // Splats don't constrain parity.
6601 if (FromV1 ? V1IsSplat : V2IsSplat)
6602 continue;
6603
6604 // The index must be from the even/odd element of its pair.
6605 if (Local / 2 != I / 2)
6606 return SDValue();
6607
6608 bool P = Local % 2;
6609 if (!Parity[Lane])
6610 Parity[Lane] = P;
6611 else if (*Parity[Lane] != P)
6612 return SDValue();
6613 }
6614
6615 // Make sure we have a source for both lanes.
6616 if (!Src[0] || !Src[1])
6617 return SDValue();
6618
6619 bool EvenIsOdd = Parity[0].value_or(false);
6620 bool OddIsOdd = Parity[1].value_or(false);
6621 unsigned Opc;
6622 if (!EvenIsOdd && !OddIsOdd)
6623 Opc = RISCVISD::PPAIRE;
6624 else if (EvenIsOdd && OddIsOdd)
6625 Opc = RISCVISD::PPAIRO;
6626 else if (!EvenIsOdd && OddIsOdd)
6627 Opc = RISCVISD::PPAIREO;
6628 else
6629 Opc = RISCVISD::PPAIROE;
6630
6631 return DAG.getNode(Opc, DL, VT, Src[0], Src[1]);
6632}
6633
6634SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6635 SelectionDAG &DAG) const {
6636 SDValue V1 = Op.getOperand(0);
6637 SDValue V2 = Op.getOperand(1);
6638 SDLoc DL(Op);
6639 MVT XLenVT = Subtarget.getXLenVT();
6640 MVT VT = Op.getSimpleValueType();
6641 unsigned NumElts = VT.getVectorNumElements();
6642 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
6643
6644 // Select RVP-specific packed shuffles before falling back to the generic
6645 // fixed/scalable-vector lowering below.
6646 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
6647 ArrayRef<int> Mask = SVN->getMask();
6648
6649 // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
6650 // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
6651 // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
6652 // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
6653 auto IsLowReverse = [&](unsigned L) {
6654 return V2.isUndef() &&
6655 ShuffleVectorInst::isReverseMask(Mask.take_front(L), L) &&
6656 all_of(Mask.drop_front(L), [](int M) { return M < 0; });
6657 };
6658 if (IsLowReverse(NumElts))
6659 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6660 if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
6661 return DAG.getNode(RISCVISD::PPAIROE_H, DL, VT, V1, V1);
6662 // Widened: reversing sends the low-half lanes to the top half, so shift
6663 // them back down by half the register. Only the 64-bit packed types are
6664 // legal here, so the register is XLen (i64).
6665 if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
6666 IsLowReverse(NumElts / 2)) {
6667 SDValue Rev = DAG.getBitcast(
6668 MVT::i64, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1));
6669 SDValue Srl =
6670 DAG.getNode(ISD::SRL, DL, MVT::i64, Rev,
6671 DAG.getConstant(VT.getSizeInBits() / 2, DL, MVT::i64));
6672 return DAG.getBitcast(VT, Srl);
6673 }
6674
6675 if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, Subtarget.is64Bit()))
6676 return V;
6677 if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
6678 return V;
6679 if (SDValue V =
6680 lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
6681 return V;
6682 if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
6683 return V;
6684 return SDValue();
6685 }
6686
6687 if (VT.getVectorElementType() == MVT::i1) {
6688 // Lower to a vror.vi of a larger element type if possible before we promote
6689 // i1s to i8s.
6690 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6691 return V;
6692 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6693 return V;
6694
6695 // Promote i1 shuffle to i8 shuffle.
6696 MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
6697 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
6698 V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
6699 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
6700 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
6701 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
6702 ISD::SETNE);
6703 }
6704
6705 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6706
6707 // Store the return value in a single variable instead of structured bindings
6708 // so that we can pass it to GetSlide below, which cannot capture structured
6709 // bindings until C++20.
6710 auto TrueMaskVL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6711 auto [TrueMask, VL] = TrueMaskVL;
6712
6713 if (SVN->isSplat()) {
6714 const int Lane = SVN->getSplatIndex();
6715 if (Lane >= 0) {
6716 MVT SVT = VT.getVectorElementType();
6717
6718 // Turn splatted vector load into a strided load with an X0 stride.
6719 SDValue V = V1;
6720 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6721 // with undef.
6722 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6723 int Offset = Lane;
6724 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6725 int OpElements =
6726 V.getOperand(0).getSimpleValueType().getVectorNumElements();
6727 V = V.getOperand(Offset / OpElements);
6728 Offset %= OpElements;
6729 }
6730
6731 // We need to ensure the load isn't atomic or volatile.
6732 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
6733 auto *Ld = cast<LoadSDNode>(V);
6734 Offset *= SVT.getStoreSize();
6735 SDValue NewAddr = DAG.getMemBasePlusOffset(
6736 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
6737
6738 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6739 if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
6740 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
6741 SDValue IntID =
6742 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
6743 SDValue Ops[] = {Ld->getChain(),
6744 IntID,
6745 DAG.getUNDEF(ContainerVT),
6746 NewAddr,
6747 DAG.getRegister(RISCV::X0, XLenVT),
6748 VL};
6749 SDValue NewLoad = DAG.getMemIntrinsicNode(
6750 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
6751 DAG.getMachineFunction().getMachineMemOperand(
6752 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
6753 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
6754 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
6755 }
6756
6757 MVT SplatVT = ContainerVT;
6758
6759 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6760 if (SVT == MVT::bf16 ||
6761 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6762 SVT = MVT::i16;
6763 SplatVT = ContainerVT.changeVectorElementType(SVT);
6764 }
6765
6766 // Otherwise use a scalar load and splat. This will give the best
6767 // opportunity to fold a splat into the operation. ISel can turn it into
6768 // the x0 strided load if we aren't able to fold away the select.
6769 if (SVT.isFloatingPoint())
6770 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
6771 Ld->getPointerInfo().getWithOffset(Offset),
6772 Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
6773 else
6774 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
6775 Ld->getPointerInfo().getWithOffset(Offset), SVT,
6776 Ld->getBaseAlign(),
6777 Ld->getMemOperand()->getFlags());
6778 DAG.makeEquivalentMemoryOrdering(Ld, V);
6779
6780 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6781 : RISCVISD::VMV_V_X_VL;
6782 SDValue Splat =
6783 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
6784 Splat = DAG.getBitcast(ContainerVT, Splat);
6785 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
6786 }
6787
6788 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6789 assert(Lane < (int)NumElts && "Unexpected lane!");
6790 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
6791 V1, DAG.getConstant(Lane, DL, XLenVT),
6792 DAG.getUNDEF(ContainerVT), TrueMask, VL);
6793 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6794 }
6795 }
6796
6797 // For exact VLEN m2 or greater, try to split to m1 operations if we
6798 // can split cleanly.
6799 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6800 return V;
6801
6802 ArrayRef<int> Mask = SVN->getMask();
6803
6804 if (SDValue V =
6805 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6806 return V;
6807
6808 if (SDValue V =
6809 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6810 return V;
6811
6812 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6813 // available.
6814 if (Subtarget.hasStdExtZvkb())
6815 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6816 return V;
6817
6818 if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef() &&
6819 NumElts != 2)
6820 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6821
6822 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6823 // use shift and truncate to perform the shuffle.
6824 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6825 // shift-and-trunc reducing total cost for everything except an mf8 result.
6826 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6827 // to do the entire operation.
6828 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6829 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6830 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6831 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6832 unsigned Index = 0;
6833 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6834 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6835 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6836 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6837 if (1 < count_if(Mask,
6838 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6839 1 < count_if(Mask, [&Mask](int Idx) {
6840 return Idx >= (int)Mask.size();
6841 })) {
6842 // Narrow each source and concatenate them.
6843 // FIXME: For small LMUL it is better to concatenate first.
6844 MVT EltVT = VT.getVectorElementType();
6845 auto EltCnt = VT.getVectorElementCount();
6846 MVT SubVT =
6847 MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor));
6848
6849 SDValue Lo =
6850 getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG);
6851 SDValue Hi =
6852 getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG);
6853
6854 SDValue Concat =
6855 DAG.getNode(ISD::CONCAT_VECTORS, DL,
6856 SubVT.getDoubleNumVectorElementsVT(), Lo, Hi);
6857 if (Factor == 2)
6858 return Concat;
6859
6860 SDValue Vec = DAG.getUNDEF(VT);
6861 return DAG.getInsertSubvector(DL, Vec, Concat, 0);
6862 }
6863 }
6864 }
6865 }
6866
6867 // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
6868 // e64 which can't match above.
6869 unsigned Index = 0;
6870 if (Subtarget.hasStdExtZvzip() &&
6871 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
6872 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6873 bool UsesBothSources =
6874 1 < count_if(Mask,
6875 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6876 1 < count_if(Mask,
6877 [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
6878
6879 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
6880 unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
6881 MVT NewVT = VT.getDoubleNumVectorElementsVT();
6882 if (isTypeLegal(NewVT)) {
6883 SDValue Op;
6884 if (V2.isUndef()) {
6885 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6886 } else if (auto VLEN = Subtarget.getRealVLen();
6887 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
6888 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6889 } else if (SDValue Src = foldConcatVector(V1, V2)) {
6890 Op = DAG.getExtractSubvector(DL, NewVT, Src, 0);
6891 }
6892 if (Op)
6893 return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
6894 }
6895
6896 if (UsesBothSources &&
6897 V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
6898 V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
6899 SDValue Lo = lowerZvzipVUNZIP(Opc, V1, DL, DAG, Subtarget);
6900 SDValue Hi = lowerZvzipVUNZIP(Opc, V2, DL, DAG, Subtarget);
6901 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6902 }
6903 }
6904 }
6905
6906 if (SDValue V =
6907 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6908 return V;
6909
6910 // Detect an interleave shuffle and lower to
6911 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6912 int EvenSrc, OddSrc;
6913 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6914 !(NumElts == 2 &&
6915 ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))) {
6916 // Extract the halves of the vectors.
6917 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6918
6919 // Recognize if one half is actually undef; the matching above will
6920 // otherwise reuse the even stream for the undef one. This improves
6921 // spread(2) shuffles.
6922 bool LaneIsUndef[2] = { true, true};
6923 for (const auto &[Idx, M] : enumerate(Mask))
6924 LaneIsUndef[Idx % 2] &= (M == -1);
6925
6926 int Size = Mask.size();
6927 SDValue EvenV, OddV;
6928 if (LaneIsUndef[0]) {
6929 EvenV = DAG.getUNDEF(HalfVT);
6930 } else {
6931 assert(EvenSrc >= 0 && "Undef source?");
6932 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6933 EvenV = DAG.getExtractSubvector(DL, HalfVT, EvenV, EvenSrc % Size);
6934 }
6935
6936 if (LaneIsUndef[1]) {
6937 OddV = DAG.getUNDEF(HalfVT);
6938 } else {
6939 assert(OddSrc >= 0 && "Undef source?");
6940 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6941 OddV = DAG.getExtractSubvector(DL, HalfVT, OddV, OddSrc % Size);
6942 }
6943
6944 // Prefer vzip if available.
6945 // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
6946 if (Subtarget.hasStdExtZvzip() && isLegalVTForZvzipOperand(VT, Subtarget))
6947 return lowerZvzipVZIP(EvenV, OddV, DL, DAG, Subtarget);
6948 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6949 }
6950
6951 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6952 // instructions (in any combination) with masking on the second instruction.
6953 // Also handles masked slides into an identity source, and single slides
6954 // without masking. Avoid matching bit rotates (which are not also element
6955 // rotates) as slide pairs. This is a performance heuristic, not a
6956 // functional check.
6957 std::array<std::pair<int, int>, 2> SrcInfo;
6958 unsigned RotateAmt;
6959 MVT RotateVT;
6960 if (::isMaskedSlidePair(Mask, SrcInfo) &&
6961 (isElementRotate(SrcInfo, NumElts) ||
6962 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
6963 SDValue Sources[2];
6964 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
6965 int SrcIdx = Info.first;
6966 assert(SrcIdx == 0 || SrcIdx == 1);
6967 SDValue &Src = Sources[SrcIdx];
6968 if (!Src) {
6969 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
6970 Src = convertToScalableVector(ContainerVT, SrcV, DAG, Subtarget);
6971 }
6972 return Src;
6973 };
6974 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
6975 SDValue Passthru) {
6976 auto [TrueMask, VL] = TrueMaskVL;
6977 SDValue SrcV = GetSourceFor(Src);
6978 int SlideAmt = Src.second;
6979 if (SlideAmt == 0) {
6980 // Should never be second operation
6981 assert(Mask == TrueMask);
6982 return SrcV;
6983 }
6984 if (SlideAmt < 0)
6985 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6986 DAG.getConstant(-SlideAmt, DL, XLenVT), Mask, VL,
6987 RISCVVType::TAIL_AGNOSTIC);
6988 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6989 DAG.getConstant(SlideAmt, DL, XLenVT), Mask, VL,
6990 RISCVVType::TAIL_AGNOSTIC);
6991 };
6992
6993 if (SrcInfo[1].first == -1) {
6994 SDValue Res = DAG.getUNDEF(ContainerVT);
6995 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6996 return convertFromScalableVector(VT, Res, DAG, Subtarget);
6997 }
6998
6999 if (Subtarget.hasStdExtZvzip()) {
7000 bool TryWiden = false;
7001 unsigned Factor;
7002 if (isPairEven(SrcInfo, Mask, Factor)) {
7003 if (Factor == 1) {
7004 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
7005 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
7006 return lowerZvzipVPAIR(RISCVISD::VPAIRE_VL, Src1, Src2, DL, DAG,
7007 Subtarget);
7008 }
7009 TryWiden = true;
7010 }
7011 if (isPairOdd(SrcInfo, Mask, Factor)) {
7012 if (Factor == 1) {
7013 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
7014 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
7015 return lowerZvzipVPAIR(RISCVISD::VPAIRO_VL, Src1, Src2, DL, DAG,
7016 Subtarget);
7017 }
7018 TryWiden = true;
7019 }
7020 // If we found a widening oppurtunity which would let us form a
7021 // pair-even or pair-odd, use the generic code to widen the shuffle
7022 // and recurse through this logic.
7023 if (TryWiden)
7024 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7025 return V;
7026 }
7027
7028 // Build the mask. Note that vslideup unconditionally preserves elements
7029 // below the slide amount in the destination, and thus those elements are
7030 // undefined in the mask. If the mask ends up all true (or undef), it
7031 // will be folded away by general logic.
7032 SmallVector<SDValue> MaskVals;
7033 for (const auto &[Idx, M] : enumerate(Mask)) {
7034 if (M < 0 ||
7035 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
7036 MaskVals.push_back(DAG.getUNDEF(XLenVT));
7037 continue;
7038 }
7039 int Src = M >= (int)NumElts;
7040 int Diff = (int)Idx - (M % NumElts);
7041 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
7042 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
7043 "Must match exactly one of the two slides");
7044 MaskVals.push_back(DAG.getConstant(C, DL, XLenVT));
7045 }
7046 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7047 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7048 SDValue SelectMask = convertToScalableVector(
7049 ContainerVT.changeVectorElementType(MVT::i1),
7050 DAG.getBuildVector(MaskVT, DL, MaskVals), DAG, Subtarget);
7051
7052 SDValue Res = DAG.getUNDEF(ContainerVT);
7053 Res = GetSlide(SrcInfo[0], TrueMask, Res);
7054 Res = GetSlide(SrcInfo[1], SelectMask, Res);
7055 return convertFromScalableVector(VT, Res, DAG, Subtarget);
7056 }
7057
7058 // Handle any remaining single source shuffles
7059 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
7060 if (V2.isUndef()) {
7061 // We might be able to express the shuffle as a bitrotate. But even if we
7062 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
7063 // shifts and a vor will have a higher throughput than a vrgather.
7064 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
7065 return V;
7066
7067 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
7068 return V;
7069
7070 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
7071 // is fully covered in interleave(2) above, so it is ignored here.
7072 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
7073 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
7074 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
7075 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
7076 unsigned Index;
7077 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
7078 MVT NarrowVT =
7079 MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
7080 SDValue Src = DAG.getExtractSubvector(DL, NarrowVT, V1, 0);
7081 return getWideningSpread(Src, Factor, Index, DL, DAG);
7082 }
7083 }
7084 }
7085
7086 // If only a prefix of the source elements influence a prefix of the
7087 // destination elements, try to see if we can reduce the required LMUL
7088 unsigned MinVLen = Subtarget.getRealMinVLen();
7089 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
7090 if (NumElts > MinVLMAX) {
7091 unsigned MaxIdx = 0;
7092 for (auto [I, M] : enumerate(Mask)) {
7093 if (M == -1)
7094 continue;
7095 MaxIdx = std::max({(unsigned)I, (unsigned)M, MaxIdx});
7096 }
7097 unsigned NewNumElts =
7098 std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
7099 if (NewNumElts != NumElts) {
7100 MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
7101 V1 = DAG.getExtractSubvector(DL, NewVT, V1, 0);
7102 SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
7103 Mask.take_front(NewNumElts));
7104 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
7105 }
7106 }
7107
7108 // Before hitting generic lowering fallbacks, try to widen the mask
7109 // to a wider SEW.
7110 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7111 return V;
7112
7113 // Can we generate a vcompress instead of a vrgather? These scale better
7114 // at high LMUL, at the cost of not being able to fold a following select
7115 // into them. The mask constants are also smaller than the index vector
7116 // constants, and thus easier to materialize.
7117 if (isCompressMask(Mask)) {
7118 SmallVector<SDValue> MaskVals(NumElts,
7119 DAG.getConstant(false, DL, XLenVT));
7120 for (auto Idx : Mask) {
7121 if (Idx == -1)
7122 break;
7123 assert(Idx >= 0 && (unsigned)Idx < NumElts);
7124 MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
7125 }
7126 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7127 SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
7128 return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
7129 DAG.getUNDEF(VT));
7130 }
7131
7132 if (VT.getScalarSizeInBits() == 8 &&
7133 any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
7134 // On such a vector we're unable to use i8 as the index type.
7135 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
7136 // may involve vector splitting if we're already at LMUL=8, or our
7137 // user-supplied maximum fixed-length LMUL.
7138 return SDValue();
7139 }
7140
7141 // Base case for the two operand recursion below - handle the worst case
7142 // single source shuffle.
7143 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
7144 MVT IndexVT = VT.changeTypeToInteger();
7145 // Since we can't introduce illegal index types at this stage, use i16 and
7146 // vrgatherei16 if the corresponding index type for plain vrgather is greater
7147 // than XLenVT.
7148 if (IndexVT.getScalarType().bitsGT(XLenVT)) {
7149 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7150 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
7151 }
7152
7153 // If the mask allows, we can do all the index computation in 16 bits. This
7154 // requires less work and less register pressure at high LMUL, and creates
7155 // smaller constants which may be cheaper to materialize.
7156 if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
7157 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
7158 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7159 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
7160 }
7161
7162 MVT IndexContainerVT =
7163 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
7164
7165 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
7166 SmallVector<SDValue> GatherIndicesLHS;
7167 for (int MaskIndex : Mask) {
7168 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
7169 GatherIndicesLHS.push_back(IsLHSIndex
7170 ? DAG.getConstant(MaskIndex, DL, XLenVT)
7171 : DAG.getUNDEF(XLenVT));
7172 }
7173 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
7174 LHSIndices =
7175 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
7176 // At m1 and less, there's no point trying any of the high LMUL splitting
7177 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
7178 if (NumElts <= MinVLMAX) {
7179 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
7180 DAG.getUNDEF(ContainerVT), TrueMask, VL);
7181 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7182 }
7183
7184 const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
7185 EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
7186 auto [InnerTrueMask, InnerVL] =
7187 getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
7188 int N =
7189 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
7190 assert(isPowerOf2_32(N) && N <= 8);
7191
7192 // If we have a locally repeating mask, then we can reuse the first
7193 // register in the index register group for all registers within the
7194 // source register group. TODO: This generalizes to m2, and m4.
7195 if (isLocalRepeatingShuffle(Mask, MinVLMAX)) {
7196 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7197 SDValue Gather = DAG.getUNDEF(ContainerVT);
7198 for (int i = 0; i < N; i++) {
7199 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
7200 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, SubIdx);
7201 SDValue SubVec =
7202 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7203 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7204 Gather = DAG.getInsertSubvector(DL, Gather, SubVec, SubIdx);
7205 }
7206 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7207 }
7208
7209 // If we have a shuffle which only uses the first register in our source
7210 // register group, and repeats the same index across all spans, we can
7211 // use a single vrgather (and possibly some register moves).
7212 // TODO: This can be generalized for m2 or m4, or for any shuffle for
7213 // which we can do a linear number of shuffles to form an m1 which
7214 // contains all the output elements.
7215 if (isLowSourceShuffle(Mask, MinVLMAX) &&
7216 isSpanSplatShuffle(Mask, MinVLMAX)) {
7217 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
7218 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7219 SDValue SubVec = DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7220 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7221 SDValue Gather = DAG.getUNDEF(ContainerVT);
7222 for (int i = 0; i < N; i++)
7223 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
7224 M1VT.getVectorMinNumElements() * i);
7225 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7226 }
7227
7228 // If we have a shuffle which only uses the first register in our
7229 // source register group, we can do a linear number of m1 vrgathers
7230 // reusing the same source register (but with different indices)
7231 // TODO: This can be generalized for m2 or m4, or for any shuffle
7232 // for which we can do a vslidedown followed by this expansion.
7233 if (isLowSourceShuffle(Mask, MinVLMAX)) {
7234 SDValue SlideAmt =
7235 DAG.getElementCount(DL, XLenVT, M1VT.getVectorElementCount());
7236 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
7237 SDValue Gather = DAG.getUNDEF(ContainerVT);
7238 for (int i = 0; i < N; i++) {
7239 if (i != 0)
7240 LHSIndices = getVSlidedown(DAG, Subtarget, DL, IndexContainerVT,
7241 DAG.getUNDEF(IndexContainerVT), LHSIndices,
7242 SlideAmt, TrueMask, VL);
7243 SDValue SubIndex =
7244 DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7245 SDValue SubVec =
7246 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7247 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7248 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
7249 M1VT.getVectorMinNumElements() * i);
7250 }
7251 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7252 }
7253
7254 // Fallback to generic vrgather if we can't find anything better.
7255 // On many machines, this will be O(LMUL^2)
7256 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
7257 DAG.getUNDEF(ContainerVT), TrueMask, VL);
7258 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7259 }
7260
7261 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
7262 // merged with a second vrgather.
7263 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
7264
7265 // Now construct the mask that will be used by the blended vrgather operation.
7266 // Construct the appropriate indices into each vector.
7267 for (int MaskIndex : Mask) {
7268 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
7269 ShuffleMaskLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
7270 ? MaskIndex : -1);
7271 ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
7272 }
7273
7274 // If the mask indices are disjoint between the two sources, we can lower it
7275 // as a vselect + a single source vrgather.vv. Don't do this if we think the
7276 // operands may end up being lowered to something cheaper than a vrgather.vv.
7277 if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
7278 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
7279 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
7280 !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
7281 !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
7282 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
7283 return V;
7284
7285 // Before hitting generic lowering fallbacks, try to widen the mask
7286 // to a wider SEW.
7287 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7288 return V;
7289
7290 // Try to pick a profitable operand order.
7291 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
7292 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts);
7293
7294 // Recursively invoke lowering for each operand if we had two
7295 // independent single source shuffles, and then combine the result via a
7296 // vselect. Note that the vselect will likely be folded back into the
7297 // second permute (vrgather, or other) by the post-isel combine.
7298 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
7299 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
7300
7301 SmallVector<SDValue> MaskVals;
7302 for (int MaskIndex : Mask) {
7303 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
7304 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
7305 }
7306
7307 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7308 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7309 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
7310
7311 if (SwapOps)
7312 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
7313 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
7314}
7315
7316bool RISCVTargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
7317 EVT VT) const {
7318 // Enable DAGCombiner::XformToShuffleWithZero to rewrite a packed zero-extend
7319 // `and` into shuffle(src, zero, ...), which lowerVECTOR_SHUFFLE turns into
7320 // RISCVISD::PPAIRE. Accept the packed byte/halfword views that lowering
7321 // handles; the 32-bit views are illegal on RV64 but reachable before type
7322 // legalization widens them to the legal 64-bit view.
7323 if (!Subtarget.hasStdExtP() || !VT.isSimple())
7324 return false;
7325 MVT SVT = VT.getSimpleVT();
7326 return (SVT == MVT::v4i8 || SVT == MVT::v8i8 || SVT == MVT::v4i16) &&
7327 isPackedZExtShuffleMask(M);
7328}
7329
7330bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7331 // Only support legal VTs for other shuffles for now.
7332 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
7333 return false;
7334
7335 // Support splats for any type. These should type legalize well.
7336 if (ShuffleVectorSDNode::isSplatMask(M))
7337 return true;
7338
7339 const unsigned NumElts = M.size();
7340 MVT SVT = VT.getSimpleVT();
7341
7342 // Not for i1 vectors.
7343 if (SVT.getScalarType() == MVT::i1)
7344 return false;
7345
7346 std::array<std::pair<int, int>, 2> SrcInfo;
7347 int Dummy1, Dummy2;
7348 return ShuffleVectorInst::isReverseMask(M, NumElts) ||
7349 (::isMaskedSlidePair(M, SrcInfo) &&
7350 isElementRotate(SrcInfo, NumElts)) ||
7351 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
7352}
7353
7354// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
7355// the exponent.
7356SDValue
7357RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
7358 SelectionDAG &DAG) const {
7359 MVT VT = Op.getSimpleValueType();
7360 unsigned EltSize = VT.getScalarSizeInBits();
7361 SDValue Src = Op.getOperand(0);
7362 SDLoc DL(Op);
7363 MVT ContainerVT = VT;
7364
7365 // We choose FP type that can represent the value if possible. Otherwise, we
7366 // use rounding to zero conversion for correct exponent of the result.
7367 // TODO: Use f16 for i8 when possible?
7368 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
7369 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
7370 FloatEltVT = MVT::f32;
7371 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
7372
7373 // Legal types should have been checked in the RISCVTargetLowering
7374 // constructor.
7375 // TODO: Splitting may make sense in some cases.
7376 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
7377 "Expected legal float type!");
7378
7379 // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
7380 // The trailing zero count is equal to log2 of this single bit value.
7381 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
7382 SDValue Neg = DAG.getNegative(Src, DL, VT);
7383 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
7384 }
7385
7386 // We have a legal FP type, convert to it.
7387 SDValue FloatVal;
7388 if (FloatVT.bitsGT(VT)) {
7389 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
7390 } else {
7391 // Use RTZ to avoid rounding influencing exponent of FloatVal.
7392 if (VT.isFixedLengthVector()) {
7393 ContainerVT = getContainerForFixedLengthVector(VT);
7394 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
7395 }
7396 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7397 SDValue RTZRM =
7398 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
7399 MVT ContainerFloatVT =
7400 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
7401 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
7402 Src, Mask, RTZRM, VL);
7403 if (VT.isFixedLengthVector())
7404 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
7405 }
7406 // Bitcast to integer and shift the exponent to the LSB.
7407 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
7408 SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
7409 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
7410
7411 // Restore back to original type. Truncation after SRL is to generate vnsrl.
7412 SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
7413 DAG.getConstant(ShiftAmt, DL, IntVT));
7414 if (IntVT.bitsLT(VT))
7415 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
7416 else if (IntVT.bitsGT(VT))
7417 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
7418
7419 // The exponent contains log2 of the value in biased form.
7420 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
7421 // For trailing zeros, we just need to subtract the bias.
7422 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
7423 return DAG.getNode(ISD::SUB, DL, VT, Exp,
7424 DAG.getConstant(ExponentBias, DL, VT));
7425
7426 // For leading zeros, we need to remove the bias and convert from log2 to
7427 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
7428 unsigned Adjust = ExponentBias + (EltSize - 1);
7429 SDValue Res =
7430 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
7431
7432 // The above result with zero input equals to Adjust which is greater than
7433 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
7434 if (Op.getOpcode() == ISD::CTLZ)
7435 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
7436
7437 return Res;
7438}
7439
7440SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7441 SelectionDAG &DAG) const {
7442 SDLoc DL(Op);
7443 MVT XLenVT = Subtarget.getXLenVT();
7444 SDValue Source = Op->getOperand(0);
7445 MVT SrcVT = Source.getSimpleValueType();
7446 SDValue Mask = Op->getOperand(1);
7447 SDValue EVL = Op->getOperand(2);
7448
7449 if (SrcVT.isFixedLengthVector()) {
7450 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
7451 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
7452 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
7453 Subtarget);
7454 SrcVT = ContainerVT;
7455 }
7456
7457 // Convert to boolean vector.
7458 if (SrcVT.getScalarType() != MVT::i1) {
7459 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
7460 SrcVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorElementCount());
7461 Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
7462 {Source, AllZero, DAG.getCondCode(ISD::SETNE),
7463 DAG.getUNDEF(SrcVT), Mask, EVL});
7464 }
7465
7466 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
7467 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
7468 // In this case, we can interpret poison as -1, so nothing to do further.
7469 return Res;
7470
7471 // Convert -1 to VL.
7472 SDValue SetCC =
7473 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
7474 Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
7475 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
7476}
7477
7478// While RVV has alignment restrictions, we should always be able to load as a
7479// legal equivalently-sized byte-typed vector instead. This method is
7480// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7481// the load is already correctly-aligned, it returns SDValue().
7482SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7483 SelectionDAG &DAG) const {
7484 auto *Load = cast<LoadSDNode>(Op);
7485 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7486
7487 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7488 Load->getMemoryVT(),
7489 *Load->getMemOperand()))
7490 return SDValue();
7491
7492 SDLoc DL(Op);
7493 MVT VT = Op.getSimpleValueType();
7494 unsigned EltSizeBits = VT.getScalarSizeInBits();
7495 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7496 "Unexpected unaligned RVV load type");
7497 MVT NewVT =
7498 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7499 assert(NewVT.isValid() &&
7500 "Expecting equally-sized RVV vector types to be legal");
7501 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7502 Load->getPointerInfo(), Load->getBaseAlign(),
7503 Load->getMemOperand()->getFlags());
7504 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7505}
7506
7507// While RVV has alignment restrictions, we should always be able to store as a
7508// legal equivalently-sized byte-typed vector instead. This method is
7509// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7510// returns SDValue() if the store is already correctly aligned.
7511SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7512 SelectionDAG &DAG) const {
7513 auto *Store = cast<StoreSDNode>(Op);
7514 assert(Store && Store->getValue().getValueType().isVector() &&
7515 "Expected vector store");
7516
7517 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7518 Store->getMemoryVT(),
7519 *Store->getMemOperand()))
7520 return SDValue();
7521
7522 SDLoc DL(Op);
7523 SDValue StoredVal = Store->getValue();
7524 MVT VT = StoredVal.getSimpleValueType();
7525 unsigned EltSizeBits = VT.getScalarSizeInBits();
7526 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7527 "Unexpected unaligned RVV store type");
7528 MVT NewVT =
7529 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7530 assert(NewVT.isValid() &&
7531 "Expecting equally-sized RVV vector types to be legal");
7532 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7533 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7534 Store->getPointerInfo(), Store->getBaseAlign(),
7535 Store->getMemOperand()->getFlags());
7536}
7537
7538// While RVV has alignment restrictions, we should always be able to load as a
7539// legal equivalently-sized byte-typed vector instead. This method is
7540// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7541// the load is already correctly-aligned, it returns SDValue().
7542SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7543 SelectionDAG &DAG) const {
7544 auto *Load = cast<VPLoadSDNode>(Op);
7545 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7546
7547 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7548 Load->getMemoryVT(),
7549 *Load->getMemOperand()))
7550 return SDValue();
7551
7552 SDValue Mask = Load->getMask();
7553
7554 // FIXME: Handled masked loads somehow.
7555 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7556 return SDValue();
7557
7558 SDLoc DL(Op);
7559 MVT VT = Op.getSimpleValueType();
7560 unsigned EltSizeBits = VT.getScalarSizeInBits();
7561 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7562 "Unexpected unaligned RVV load type");
7563 MVT NewVT =
7564 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7565 assert(NewVT.isValid() &&
7566 "Expecting equally-sized RVV vector types to be legal");
7567
7568 SDValue VL = Load->getVectorLength();
7569 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7570 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7571
7572 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7573 SDValue L = DAG.getLoadVP(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7574 DAG.getAllOnesConstant(DL, MaskVT), VL,
7575 Load->getPointerInfo(), Load->getBaseAlign(),
7576 Load->getMemOperand()->getFlags(), AAMDNodes());
7577 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7578}
7579
7580// While RVV has alignment restrictions, we should always be able to store as a
7581// legal equivalently-sized byte-typed vector instead. This method is
7582// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7583// It returns SDValue() if the store is already correctly aligned.
7584SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7585 SelectionDAG &DAG) const {
7586 auto *Store = cast<VPStoreSDNode>(Op);
7587 assert(Store && Store->getValue().getValueType().isVector() &&
7588 "Expected vector store");
7589
7590 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7591 Store->getMemoryVT(),
7592 *Store->getMemOperand()))
7593 return SDValue();
7594
7595 SDValue Mask = Store->getMask();
7596
7597 // FIXME: Handled masked stores somehow.
7598 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7599 return SDValue();
7600
7601 SDLoc DL(Op);
7602 SDValue StoredVal = Store->getValue();
7603 MVT VT = StoredVal.getSimpleValueType();
7604 unsigned EltSizeBits = VT.getScalarSizeInBits();
7605 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7606 "Unexpected unaligned RVV store type");
7607 MVT NewVT =
7608 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7609 assert(NewVT.isValid() &&
7610 "Expecting equally-sized RVV vector types to be legal");
7611
7612 SDValue VL = Store->getVectorLength();
7613 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7614 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7615
7616 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7617
7618 LocationSize Size = LocationSize::precise(NewVT.getStoreSize());
7619 MachineFunction &MF = DAG.getMachineFunction();
7620 MachineMemOperand *MMO = MF.getMachineMemOperand(
7621 Store->getPointerInfo(), Store->getMemOperand()->getFlags(), Size,
7622 Store->getBaseAlign());
7623
7624 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7625 return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7626 DAG.getPOISON(Store->getBasePtr().getValueType()),
7627 DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
7628 ISD::UNINDEXED);
7629}
7630
7631static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7632 const RISCVSubtarget &Subtarget) {
7633 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7634
7635 int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
7636
7637 // All simm32 constants should be handled by isel.
7638 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7639 // this check redundant, but small immediates are common so this check
7640 // should have better compile time.
7641 if (isInt<32>(Imm))
7642 return Op;
7643
7644 // We only need to cost the immediate, if constant pool lowering is enabled.
7645 if (!Subtarget.useConstantPoolForLargeInts())
7646 return Op;
7647
7648 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget);
7649 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7650 return Op;
7651
7652 // Optimizations below are disabled for opt size. If we're optimizing for
7653 // size, use a constant pool.
7654 if (DAG.shouldOptForSize())
7655 return SDValue();
7656
7657 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7658 // that if it will avoid a constant pool.
7659 // It will require an extra temporary register though.
7660 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7661 // low and high 32 bits are the same and bit 31 and 63 are set.
7662 unsigned ShiftAmt, AddOpc;
7663 RISCVMatInt::InstSeq SeqLo =
7664 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
7665 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7666 return Op;
7667
7668 return SDValue();
7669}
7670
7671SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7672 SelectionDAG &DAG) const {
7673 MVT VT = Op.getSimpleValueType();
7674 const APFloat &Imm = cast<ConstantFPSDNode>(Op)->getValueAPF();
7675
7676 // Can this constant be selected by a Zfa FLI instruction?
7677 bool Negate = false;
7678 int Index = getLegalZfaFPImm(Imm, VT);
7679
7680 // If the constant is negative, try negating.
7681 if (Index < 0 && Imm.isNegative()) {
7682 Index = getLegalZfaFPImm(-Imm, VT);
7683 Negate = true;
7684 }
7685
7686 // If we couldn't find a FLI lowering, fall back to generic code.
7687 if (Index < 0)
7688 return SDValue();
7689
7690 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7691 SDLoc DL(Op);
7692 SDValue Const =
7693 DAG.getNode(RISCVISD::FLI, DL, VT,
7694 DAG.getTargetConstant(Index, DL, Subtarget.getXLenVT()));
7695 if (!Negate)
7696 return Const;
7697
7698 return DAG.getNode(ISD::FNEG, DL, VT, Const);
7699}
7700
7701static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7702 SelectionDAG &DAG) {
7703
7704 unsigned IsData = Op.getConstantOperandVal(4);
7705
7706 // mips-p8700 we support data prefetch for now.
7707 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7708 return Op.getOperand(0);
7709 return Op;
7710}
7711
7712static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7713 const RISCVSubtarget &Subtarget) {
7714 SDLoc dl(Op);
7715 AtomicOrdering FenceOrdering =
7716 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
7717 SyncScope::ID FenceSSID =
7718 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
7719
7720 if (Subtarget.hasStdExtZtso()) {
7721 // The only fence that needs an instruction is a sequentially-consistent
7722 // cross-thread fence.
7723 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7724 FenceSSID == SyncScope::System)
7725 return Op;
7726
7727 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7728 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7729 }
7730
7731 // singlethread fences only synchronize with signal handlers on the same
7732 // thread and thus only need to preserve instruction order, not actually
7733 // enforce memory ordering.
7734 if (FenceSSID == SyncScope::SingleThread)
7735 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7736 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7737
7738 return Op;
7739}
7740
7741SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7742 SelectionDAG &DAG) const {
7743 SDLoc DL(Op);
7744 MVT VT = Op.getSimpleValueType();
7745 MVT XLenVT = Subtarget.getXLenVT();
7746 unsigned Check = Op.getConstantOperandVal(1);
7747 unsigned TDCMask = 0;
7748 if (Check & fcSNan)
7749 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7750 if (Check & fcQNan)
7751 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7752 if (Check & fcPosInf)
7753 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7754 if (Check & fcNegInf)
7755 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7756 if (Check & fcPosNormal)
7757 TDCMask |= RISCV::FPMASK_Positive_Normal;
7758 if (Check & fcNegNormal)
7759 TDCMask |= RISCV::FPMASK_Negative_Normal;
7760 if (Check & fcPosSubnormal)
7761 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7762 if (Check & fcNegSubnormal)
7763 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7764 if (Check & fcPosZero)
7765 TDCMask |= RISCV::FPMASK_Positive_Zero;
7766 if (Check & fcNegZero)
7767 TDCMask |= RISCV::FPMASK_Negative_Zero;
7768
7769 bool IsOneBitMask = isPowerOf2_32(TDCMask);
7770
7771 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
7772
7773 if (VT.isVector()) {
7774 SDValue Op0 = Op.getOperand(0);
7775 MVT VT0 = Op.getOperand(0).getSimpleValueType();
7776
7777 if (VT.isScalableVector()) {
7778 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7779 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
7780 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
7781 VL, Op->getFlags());
7782 if (IsOneBitMask)
7783 return DAG.getSetCC(DL, VT, FPCLASS,
7784 DAG.getConstant(TDCMask, DL, DstVT),
7785 ISD::CondCode::SETEQ);
7786 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
7787 DAG.getConstant(TDCMask, DL, DstVT));
7788 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
7789 ISD::SETNE);
7790 }
7791
7792 MVT ContainerVT0 = getContainerForFixedLengthVector(VT0);
7793 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7794 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7795 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
7796 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
7797
7798 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
7799 Mask, VL, Op->getFlags());
7800
7801 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7802 DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
7803 if (IsOneBitMask) {
7804 SDValue VMSEQ =
7805 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7806 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
7807 DAG.getUNDEF(ContainerVT), Mask, VL});
7808 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
7809 }
7810 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
7811 TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
7812
7813 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
7814 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7815 DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
7816
7817 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7818 {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
7819 DAG.getUNDEF(ContainerVT), Mask, VL});
7820 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
7821 }
7822
7823 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
7824 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
7825 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
7826 ISD::CondCode::SETNE);
7827 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
7828}
7829
7830// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7831// operations propagate nans.
7832static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7833 const RISCVSubtarget &Subtarget) {
7834 SDLoc DL(Op);
7835 MVT VT = Op.getSimpleValueType();
7836
7837 SDValue X = Op.getOperand(0);
7838 SDValue Y = Op.getOperand(1);
7839
7840 if (!VT.isVector()) {
7841 MVT XLenVT = Subtarget.getXLenVT();
7842
7843 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7844 // ensures that when one input is a nan, the other will also be a nan
7845 // allowing the nan to propagate. If both inputs are nan, this will swap the
7846 // inputs which is harmless.
7847
7848 SDValue NewY = Y;
7849 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
7850 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
7851 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
7852 }
7853
7854 SDValue NewX = X;
7855 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
7856 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
7857 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
7858 }
7859
7860 unsigned Opc =
7861 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7862 return DAG.getNode(Opc, DL, VT, NewX, NewY);
7863 }
7864
7865 // Check no NaNs before converting to fixed vector scalable.
7866 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
7867 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
7868
7869 MVT ContainerVT = VT;
7870 if (VT.isFixedLengthVector()) {
7871 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
7872 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
7873 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
7874 }
7875
7876 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7877
7878 SDValue NewY = Y;
7879 if (!XIsNeverNan) {
7880 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7881 {X, X, DAG.getCondCode(ISD::SETOEQ),
7882 DAG.getUNDEF(ContainerVT), Mask, VL});
7883 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
7884 DAG.getUNDEF(ContainerVT), VL);
7885 }
7886
7887 SDValue NewX = X;
7888 if (!YIsNeverNan) {
7889 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7890 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
7891 DAG.getUNDEF(ContainerVT), Mask, VL});
7892 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
7893 DAG.getUNDEF(ContainerVT), VL);
7894 }
7895
7896 unsigned Opc =
7897 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
7898 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
7899 DAG.getUNDEF(ContainerVT), Mask, VL);
7900 if (VT.isFixedLengthVector())
7901 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
7902 return Res;
7903}
7904
7905static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7906 const RISCVSubtarget &Subtarget) {
7907 bool IsFABS = Op.getOpcode() == ISD::FABS;
7908 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7909 "Wrong opcode for lowering FABS or FNEG.");
7910
7911 MVT XLenVT = Subtarget.getXLenVT();
7912 MVT VT = Op.getSimpleValueType();
7913 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7914
7915 SDLoc DL(Op);
7916 SDValue Fmv =
7917 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op.getOperand(0));
7918
7919 APInt Mask = IsFABS ? APInt::getSignedMaxValue(16) : APInt::getSignMask(16);
7920 Mask = Mask.sext(Subtarget.getXLen());
7921
7922 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7923 SDValue Logic =
7924 DAG.getNode(LogicOpc, DL, XLenVT, Fmv, DAG.getConstant(Mask, DL, XLenVT));
7925 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
7926}
7927
7928static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7929 const RISCVSubtarget &Subtarget) {
7930 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7931
7932 MVT XLenVT = Subtarget.getXLenVT();
7933 MVT VT = Op.getSimpleValueType();
7934 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7935
7936 SDValue Mag = Op.getOperand(0);
7937 SDValue Sign = Op.getOperand(1);
7938
7939 SDLoc DL(Op);
7940
7941 // Get sign bit into an integer value.
7942 unsigned SignSize = Sign.getValueSizeInBits();
7943 SDValue SignAsInt = [&]() {
7944 if (SignSize == Subtarget.getXLen())
7945 return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
7946 switch (SignSize) {
7947 case 16:
7948 return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
7949 case 32:
7950 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
7951 case 64: {
7952 assert(XLenVT == MVT::i32 && "Unexpected type");
7953 // Copy the upper word to integer.
7954 SignSize = 32;
7955 return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
7956 .getValue(1);
7957 }
7958 default:
7959 llvm_unreachable("Unexpected sign size");
7960 }
7961 }();
7962
7963 // Get the signbit at the right position for MagAsInt.
7964 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
7965 SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
7966 SignAsInt,
7967 DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
7968
7969 // Mask the sign bit and any bits above it. The extra bits will be dropped
7970 // when we convert back to FP.
7971 SDValue SignMask = DAG.getConstant(
7972 APInt::getSignMask(16).sext(Subtarget.getXLen()), DL, XLenVT);
7973 SDValue SignBit = DAG.getNode(ISD::AND, DL, XLenVT, SignAsInt, SignMask);
7974
7975 // Transform Mag value to integer, and clear the sign bit.
7976 SDValue MagAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Mag);
7977 SDValue ClearSignMask = DAG.getConstant(
7978 APInt::getSignedMaxValue(16).sext(Subtarget.getXLen()), DL, XLenVT);
7979 SDValue ClearedSign =
7980 DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
7981
7982 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
7983 SDNodeFlags::Disjoint);
7984
7985 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
7986}
7987
7988/// Get a RISC-V target specified VL op for a given SDNode.
7989static unsigned getRISCVVLOp(SDValue Op) {
7990#define OP_CASE(NODE) \
7991 case ISD::NODE: \
7992 return RISCVISD::NODE##_VL;
7993#define VP_CASE(NODE) \
7994 case ISD::VP_##NODE: \
7995 return RISCVISD::NODE##_VL;
7996 // clang-format off
7997 switch (Op.getOpcode()) {
7998 default:
7999 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
8000 OP_CASE(ADD)
8001 OP_CASE(SUB)
8002 OP_CASE(MUL)
8003 OP_CASE(MULHS)
8004 OP_CASE(MULHU)
8005 OP_CASE(SDIV)
8006 OP_CASE(SREM)
8007 OP_CASE(UDIV)
8008 OP_CASE(UREM)
8009 OP_CASE(SHL)
8010 OP_CASE(SRA)
8011 OP_CASE(SRL)
8012 OP_CASE(ROTL)
8013 OP_CASE(ROTR)
8014 OP_CASE(BSWAP)
8015 OP_CASE(CTTZ)
8016 OP_CASE(CTLZ)
8017 OP_CASE(CTPOP)
8018 OP_CASE(BITREVERSE)
8019 OP_CASE(CLMUL)
8020 OP_CASE(CLMULH)
8021 OP_CASE(SADDSAT)
8022 OP_CASE(UADDSAT)
8023 OP_CASE(SSUBSAT)
8024 OP_CASE(USUBSAT)
8025 OP_CASE(AVGFLOORS)
8026 OP_CASE(AVGFLOORU)
8027 OP_CASE(AVGCEILS)
8028 OP_CASE(AVGCEILU)
8029 OP_CASE(FADD)
8030 OP_CASE(FSUB)
8031 OP_CASE(FMUL)
8032 OP_CASE(FDIV)
8033 OP_CASE(FNEG)
8034 OP_CASE(FABS)
8035 OP_CASE(FCOPYSIGN)
8036 OP_CASE(FSQRT)
8037 OP_CASE(SMIN)
8038 OP_CASE(SMAX)
8039 OP_CASE(UMIN)
8040 OP_CASE(UMAX)
8041 OP_CASE(ABDS)
8042 OP_CASE(ABDU)
8043 OP_CASE(STRICT_FADD)
8044 OP_CASE(STRICT_FSUB)
8045 OP_CASE(STRICT_FMUL)
8046 OP_CASE(STRICT_FDIV)
8047 OP_CASE(STRICT_FSQRT)
8048 VP_CASE(SDIV) // VP_SDIV
8049 VP_CASE(SREM) // VP_SREM
8050 VP_CASE(UDIV) // VP_UDIV
8051 VP_CASE(UREM) // VP_UREM
8052 case ISD::CTLZ_ZERO_POISON:
8053 return RISCVISD::CTLZ_VL;
8054 case ISD::CTTZ_ZERO_POISON:
8055 return RISCVISD::CTTZ_VL;
8056 case ISD::FMA:
8057 return RISCVISD::VFMADD_VL;
8058 case ISD::STRICT_FMA:
8059 return RISCVISD::STRICT_VFMADD_VL;
8060 case ISD::AND:
8061 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8062 return RISCVISD::VMAND_VL;
8063 return RISCVISD::AND_VL;
8064 case ISD::OR:
8065 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8066 return RISCVISD::VMOR_VL;
8067 return RISCVISD::OR_VL;
8068 case ISD::XOR:
8069 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8070 return RISCVISD::VMXOR_VL;
8071 return RISCVISD::XOR_VL;
8072 case ISD::ANY_EXTEND:
8073 case ISD::ZERO_EXTEND:
8074 return RISCVISD::VZEXT_VL;
8075 case ISD::SIGN_EXTEND:
8076 return RISCVISD::VSEXT_VL;
8077 case ISD::SETCC:
8078 return RISCVISD::SETCC_VL;
8079 case ISD::VSELECT:
8080 return RISCVISD::VMERGE_VL;
8081 case ISD::VP_MERGE:
8082 return RISCVISD::VMERGE_VL;
8083 case ISD::FMINNUM:
8084 case ISD::FMINIMUMNUM:
8085 return RISCVISD::VFMIN_VL;
8086 case ISD::FMAXNUM:
8087 case ISD::FMAXIMUMNUM:
8088 return RISCVISD::VFMAX_VL;
8089 case ISD::LRINT:
8090 case ISD::LLRINT:
8091 return RISCVISD::VFCVT_RM_X_F_VL;
8092 case ISD::MASKED_UDIV:
8093 return RISCVISD::UDIV_VL;
8094 case ISD::MASKED_UREM:
8095 return RISCVISD::UREM_VL;
8096 case ISD::MASKED_SDIV:
8097 return RISCVISD::SDIV_VL;
8098 case ISD::MASKED_SREM:
8099 return RISCVISD::SREM_VL;
8100 }
8101 // clang-format on
8102#undef OP_CASE
8103#undef VP_CASE
8104}
8105
8106static bool isPromotedOpNeedingSplit(SDValue Op,
8107 const RISCVSubtarget &Subtarget,
8108 const TargetLowering &TLI) {
8109 MVT OpVT = Op.getSimpleValueType();
8110 if (!OpVT.isVector())
8111 return false;
8112 MVT EltVT = OpVT.getVectorElementType();
8113 if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
8114 !Subtarget.hasVInstructionsF16()) &&
8115 !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
8116 (!Subtarget.hasVInstructionsBF16() ||
8117 !llvm::is_contained(ZvfbfaOps, Op.getOpcode()))))
8118 return false;
8119 // Need to split when the same width f32 vector type isn't legal.
8120 return !TLI.isTypeLegal(
8121 MVT::getVectorVT(MVT::f32, OpVT.getVectorElementCount()));
8122}
8123
8124static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
8125 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
8126 SDLoc DL(Op);
8127
8128 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8129 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8130
8131 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8132 if (!Op.getOperand(j).getValueType().isVector()) {
8133 LoOperands[j] = Op.getOperand(j);
8134 HiOperands[j] = Op.getOperand(j);
8135 continue;
8136 }
8137 std::tie(LoOperands[j], HiOperands[j]) =
8138 DAG.SplitVector(Op.getOperand(j), DL);
8139 }
8140
8141 SDValue LoRes =
8142 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
8143 SDValue HiRes =
8144 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
8145
8146 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
8147}
8148
8149static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
8150 bool IsVP) {
8151 SDLoc DL(Op);
8152
8153 if (IsVP) {
8154 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
8155 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
8156 auto [EVLLo, EVLHi] =
8157 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
8158
8159 SDValue ResLo =
8160 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8161 {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
8162 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8163 {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
8164 }
8165
8166 unsigned Opcode = Op.getOpcode();
8167 unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
8168
8169 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(OpNo), DL);
8170 if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
8171 SDValue ResLo = DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8172 Op.getOperand(0), Lo, Op->getFlags());
8173 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), ResLo, Hi,
8174 Op->getFlags());
8175 }
8176
8177 SDValue ResLo =
8178 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Lo, Op->getFlags());
8179 SDValue ResHi =
8180 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Hi, Op->getFlags());
8181 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
8182 return DAG.getNode(BaseOpc, DL, Op.getValueType(), ResLo, ResHi,
8183 Op->getFlags());
8184}
8185
8186static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
8187
8188 assert(Op->isStrictFPOpcode());
8189
8190 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
8191
8192 SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
8193 SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
8194
8195 SDLoc DL(Op);
8196
8197 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8198 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8199
8200 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8201 if (!Op.getOperand(j).getValueType().isVector()) {
8202 LoOperands[j] = Op.getOperand(j);
8203 HiOperands[j] = Op.getOperand(j);
8204 continue;
8205 }
8206 std::tie(LoOperands[j], HiOperands[j]) =
8207 DAG.SplitVector(Op.getOperand(j), DL);
8208 }
8209
8210 SDValue LoRes =
8211 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
8212 HiOperands[0] = LoRes.getValue(1);
8213 SDValue HiRes =
8214 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
8215
8216 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
8217 LoRes.getValue(0), HiRes.getValue(0));
8218 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
8219}
8220
8221SDValue
8222RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
8223 SelectionDAG &DAG) const {
8224 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8225 "Unexpected bfloat16 load lowering");
8226
8227 SDLoc DL(Op);
8228 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
8229 EVT MemVT = LD->getMemoryVT();
8230 SDValue Load = DAG.getExtLoad(
8231 ISD::ZEXTLOAD, DL, Subtarget.getXLenVT(), LD->getChain(),
8232 LD->getBasePtr(),
8233 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
8234 LD->getMemOperand());
8235 // Using mask to make bf16 nan-boxing valid when we don't have flh
8236 // instruction. -65536 would be treat as a small number and thus it can be
8237 // directly used lui to get the constant.
8238 SDValue mask = DAG.getSignedConstant(-65536, DL, Subtarget.getXLenVT());
8239 SDValue OrSixteenOne =
8240 DAG.getNode(ISD::OR, DL, Load.getValueType(), {Load, mask});
8241 SDValue ConvertedResult =
8242 DAG.getNode(RISCVISD::NDS_FMV_BF16_X, DL, MVT::bf16, OrSixteenOne);
8243 return DAG.getMergeValues({ConvertedResult, Load.getValue(1)}, DL);
8244}
8245
8246SDValue
8247RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
8248 SelectionDAG &DAG) const {
8249 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8250 "Unexpected bfloat16 store lowering");
8251
8252 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
8253 SDLoc DL(Op);
8254 SDValue FMV = DAG.getNode(RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
8255 Subtarget.getXLenVT(), ST->getValue());
8256 return DAG.getTruncStore(
8257 ST->getChain(), DL, FMV, ST->getBasePtr(),
8258 EVT::getIntegerVT(*DAG.getContext(), ST->getMemoryVT().getSizeInBits()),
8259 ST->getMemOperand());
8260}
8261
8262static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
8263 const RISCVSubtarget &Subtarget);
8264
8265static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
8266 const RISCVSubtarget &Subtarget);
8267
8268SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
8269 SelectionDAG &DAG) const {
8270 switch (Op.getOpcode()) {
8271 default:
8272 reportFatalInternalError(
8273 "Unimplemented RISCVTargetLowering::LowerOperation Case");
8274 case ISD::PREFETCH:
8275 return LowerPREFETCH(Op, Subtarget, DAG);
8276 case ISD::ATOMIC_FENCE:
8277 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
8278 case ISD::GlobalAddress:
8279 return lowerGlobalAddress(Op, DAG);
8280 case ISD::BlockAddress:
8281 return lowerBlockAddress(Op, DAG);
8282 case ISD::ConstantPool:
8283 return lowerConstantPool(Op, DAG);
8284 case ISD::JumpTable:
8285 return lowerJumpTable(Op, DAG);
8286 case ISD::GlobalTLSAddress:
8287 return lowerGlobalTLSAddress(Op, DAG);
8288 case ISD::Constant:
8289 return lowerConstant(Op, DAG, Subtarget);
8290 case ISD::ConstantFP:
8291 return lowerConstantFP(Op, DAG);
8292 case ISD::SELECT:
8293 return lowerSELECT(Op, DAG);
8294 case ISD::BRCOND:
8295 return lowerBRCOND(Op, DAG);
8296 case ISD::VASTART:
8297 return lowerVASTART(Op, DAG);
8298 case ISD::FRAMEADDR:
8299 return lowerFRAMEADDR(Op, DAG);
8300 case ISD::RETURNADDR:
8301 return lowerRETURNADDR(Op, DAG);
8302 case ISD::SHL_PARTS:
8303 return lowerShiftLeftParts(Op, DAG);
8304 case ISD::SRA_PARTS:
8305 return lowerShiftRightParts(Op, D