LLVM 24.0.0git
RISCVISelLowering.cpp
Go to the documentation of this file.
1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineJumpTableInfo.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SDPatternMatch.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsRISCV.h"
44#include "llvm/MC/MCCodeEmitter.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/InstructionCost.h"
50#include "llvm/Support/KnownBits.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "riscv-lower"
58
59STATISTIC(NumTailCalls, "Number of tail calls");
60
61static cl::opt<unsigned> ExtensionMaxWebSize(
62 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
63 cl::desc("Give the maximum size (in number of nodes) of the web of "
64 "instructions that we will consider for VW expansion"),
65 cl::init(18));
66
67static cl::opt<bool>
68 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
69 cl::desc("Allow the formation of VW_W operations (e.g., "
70 "VWADD_W) with splat constants"),
71 cl::init(false));
72
73static cl::opt<unsigned> NumRepeatedDivisors(
74 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
75 cl::desc("Set the minimum number of repetitions of a divisor to allow "
76 "transformation to multiplications by the reciprocal"),
77 cl::init(2));
78
79static cl::opt<int>
80 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
81 cl::desc("Give the maximum number of instructions that we will "
82 "use for creating a floating-point immediate value"),
83 cl::init(3));
84
85static cl::opt<bool>
86 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
87 cl::desc("Swap add and addi in cases where the add may "
88 "be combined with a shift"),
89 cl::init(true));
90
91static cl::opt<int> BrMergingBaseCostThresh(
92 "riscv-br-merging-base-cost", cl::init(2),
93 cl::desc(
94 "Sets the cost threshold for when multiple conditionals will be merged "
95 "into one branch versus be split in multiple branches. Merging "
96 "conditionals saves branches at the cost of additional instructions. "
97 "This value sets the instruction cost limit, below which conditionals "
98 "will be merged, and above which conditionals will be split. Set to -1 "
99 "to never merge branches."),
100 cl::Hidden);
101
102static cl::opt<int> BrMergingLikelyBias(
103 "riscv-br-merging-likely-bias", cl::init(0),
104 cl::desc(
105 "Increases 'riscv-br-merging-base-cost' in cases that it is "
106 "likely that all conditionals will be executed. For example for "
107 "merging the conditionals (a == b && c > d), if its known that "
108 "a == b is likely, then it is likely that if the conditionals are "
109 "split both sides will be executed, so it may be desirable to "
110 "increase the instruction cost threshold. Set to -1 to never merge "
111 "likely branches."),
112 cl::Hidden);
113
114static cl::opt<int> BrMergingUnlikelyBias(
115 "riscv-br-merging-unlikely-bias", cl::init(-1),
116 cl::desc(
117 "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
118 "that all conditionals will be executed. For example for merging "
119 "the conditionals (a == b && c > d), if its known that a == b is "
120 "unlikely, then it is unlikely that if the conditionals are split "
121 "both sides will be executed, so it may be desirable to decrease "
122 "the instruction cost threshold. Set to -1 to never merge unlikely "
123 "branches."),
124 cl::Hidden);
125
126// TODO: Support more ops
133
134RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
135 const RISCVSubtarget &STI)
136 : TargetLowering(TM, STI), Subtarget(STI) {
137
138 RISCVABI::ABI ABI = Subtarget.getTargetABI();
139 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
140
141 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
142 !Subtarget.hasStdExtF()) {
143 errs() << "Hard-float 'f' ABI can't be used for a target that "
144 "doesn't support the F instruction set extension (ignoring "
145 "target-abi)\n";
146 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
147 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
148 !Subtarget.hasStdExtD()) {
149 errs() << "Hard-float 'd' ABI can't be used for a target that "
150 "doesn't support the D instruction set extension (ignoring "
151 "target-abi)\n";
152 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
153 }
154
155 switch (ABI) {
156 default:
157 reportFatalUsageError("Don't know how to lower this ABI");
158 case RISCVABI::ABI_ILP32:
159 case RISCVABI::ABI_ILP32E:
160 case RISCVABI::ABI_LP64E:
161 case RISCVABI::ABI_ILP32F:
162 case RISCVABI::ABI_ILP32D:
163 case RISCVABI::ABI_LP64:
164 case RISCVABI::ABI_LP64F:
165 case RISCVABI::ABI_LP64D:
166 break;
167 }
168
169 MVT XLenVT = Subtarget.getXLenVT();
170
171 // Set up the register classes.
172 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
173
174 if (Subtarget.hasStdExtZfhmin())
175 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
176 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
177 addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
178 if (Subtarget.hasStdExtF())
179 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
180 if (Subtarget.hasStdExtD())
181 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
182 if (Subtarget.hasStdExtZhinxmin())
183 addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
184 if (Subtarget.hasStdExtZfinx())
185 addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
186 if (Subtarget.hasStdExtZdinx()) {
187 if (Subtarget.is64Bit())
188 addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
189 else
190 addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
191 }
192
193 static const MVT::SimpleValueType BoolVecVTs[] = {
194 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
195 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
196 static const MVT::SimpleValueType IntVecVTs[] = {
197 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
198 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
199 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
200 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
201 MVT::nxv4i64, MVT::nxv8i64};
202 static const MVT::SimpleValueType F16VecVTs[] = {
203 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
204 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
205 static const MVT::SimpleValueType BF16VecVTs[] = {
206 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
207 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
208 static const MVT::SimpleValueType F32VecVTs[] = {
209 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
210 static const MVT::SimpleValueType F64VecVTs[] = {
211 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
212 static const MVT::SimpleValueType VecTupleVTs[] = {
213 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
214 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
215 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
216 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
217 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
218 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
219 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
220 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
221 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
222 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
223 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
224
225 if (Subtarget.hasVInstructions()) {
226 auto addRegClassForRVV = [this](MVT VT) {
227 // Disable the smallest fractional LMUL types if ELEN is less than
228 // RVVBitsPerBlock.
229 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
230 if (VT.getVectorMinNumElements() < MinElts)
231 return;
232
233 unsigned Size = VT.getSizeInBits().getKnownMinValue();
234 const TargetRegisterClass *RC;
235 if (Size <= RISCV::RVVBitsPerBlock)
236 RC = &RISCV::VRRegClass;
237 else if (Size == 2 * RISCV::RVVBitsPerBlock)
238 RC = &RISCV::VRM2RegClass;
239 else if (Size == 4 * RISCV::RVVBitsPerBlock)
240 RC = &RISCV::VRM4RegClass;
241 else if (Size == 8 * RISCV::RVVBitsPerBlock)
242 RC = &RISCV::VRM8RegClass;
243 else
244 llvm_unreachable("Unexpected size");
245
246 addRegisterClass(VT, RC);
247 };
248
249 for (MVT VT : BoolVecVTs)
250 addRegClassForRVV(VT);
251 for (MVT VT : IntVecVTs) {
252 if (VT.getVectorElementType() == MVT::i64 &&
253 !Subtarget.hasVInstructionsI64())
254 continue;
255 addRegClassForRVV(VT);
256 }
257
258 if (Subtarget.hasVInstructionsF16Minimal() ||
259 Subtarget.hasVendorXAndesVPackFPH())
260 for (MVT VT : F16VecVTs)
261 addRegClassForRVV(VT);
262
263 if (Subtarget.hasVInstructionsBF16Minimal() ||
264 Subtarget.hasVendorXAndesVBFHCvt())
265 for (MVT VT : BF16VecVTs)
266 addRegClassForRVV(VT);
267
268 if (Subtarget.hasVInstructionsF32())
269 for (MVT VT : F32VecVTs)
270 addRegClassForRVV(VT);
271
272 if (Subtarget.hasVInstructionsF64())
273 for (MVT VT : F64VecVTs)
274 addRegClassForRVV(VT);
275
276 if (Subtarget.useRVVForFixedLengthVectors()) {
277 auto addRegClassForFixedVectors = [this](MVT VT) {
278 MVT ContainerVT = getContainerForFixedLengthVector(VT);
279 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
280 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
281 addRegisterClass(VT, TRI.getRegClass(RCID));
282 };
283 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
284 if (useRVVForFixedLengthVectorVT(VT))
285 addRegClassForFixedVectors(VT);
286
287 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
288 if (useRVVForFixedLengthVectorVT(VT))
289 addRegClassForFixedVectors(VT);
290 }
291
292 addRegisterClass(MVT::riscv_nxv1i8x2, &RISCV::VRN2M1RegClass);
293 addRegisterClass(MVT::riscv_nxv1i8x3, &RISCV::VRN3M1RegClass);
294 addRegisterClass(MVT::riscv_nxv1i8x4, &RISCV::VRN4M1RegClass);
295 addRegisterClass(MVT::riscv_nxv1i8x5, &RISCV::VRN5M1RegClass);
296 addRegisterClass(MVT::riscv_nxv1i8x6, &RISCV::VRN6M1RegClass);
297 addRegisterClass(MVT::riscv_nxv1i8x7, &RISCV::VRN7M1RegClass);
298 addRegisterClass(MVT::riscv_nxv1i8x8, &RISCV::VRN8M1RegClass);
299 addRegisterClass(MVT::riscv_nxv2i8x2, &RISCV::VRN2M1RegClass);
300 addRegisterClass(MVT::riscv_nxv2i8x3, &RISCV::VRN3M1RegClass);
301 addRegisterClass(MVT::riscv_nxv2i8x4, &RISCV::VRN4M1RegClass);
302 addRegisterClass(MVT::riscv_nxv2i8x5, &RISCV::VRN5M1RegClass);
303 addRegisterClass(MVT::riscv_nxv2i8x6, &RISCV::VRN6M1RegClass);
304 addRegisterClass(MVT::riscv_nxv2i8x7, &RISCV::VRN7M1RegClass);
305 addRegisterClass(MVT::riscv_nxv2i8x8, &RISCV::VRN8M1RegClass);
306 addRegisterClass(MVT::riscv_nxv4i8x2, &RISCV::VRN2M1RegClass);
307 addRegisterClass(MVT::riscv_nxv4i8x3, &RISCV::VRN3M1RegClass);
308 addRegisterClass(MVT::riscv_nxv4i8x4, &RISCV::VRN4M1RegClass);
309 addRegisterClass(MVT::riscv_nxv4i8x5, &RISCV::VRN5M1RegClass);
310 addRegisterClass(MVT::riscv_nxv4i8x6, &RISCV::VRN6M1RegClass);
311 addRegisterClass(MVT::riscv_nxv4i8x7, &RISCV::VRN7M1RegClass);
312 addRegisterClass(MVT::riscv_nxv4i8x8, &RISCV::VRN8M1RegClass);
313 addRegisterClass(MVT::riscv_nxv8i8x2, &RISCV::VRN2M1RegClass);
314 addRegisterClass(MVT::riscv_nxv8i8x3, &RISCV::VRN3M1RegClass);
315 addRegisterClass(MVT::riscv_nxv8i8x4, &RISCV::VRN4M1RegClass);
316 addRegisterClass(MVT::riscv_nxv8i8x5, &RISCV::VRN5M1RegClass);
317 addRegisterClass(MVT::riscv_nxv8i8x6, &RISCV::VRN6M1RegClass);
318 addRegisterClass(MVT::riscv_nxv8i8x7, &RISCV::VRN7M1RegClass);
319 addRegisterClass(MVT::riscv_nxv8i8x8, &RISCV::VRN8M1RegClass);
320 addRegisterClass(MVT::riscv_nxv16i8x2, &RISCV::VRN2M2RegClass);
321 addRegisterClass(MVT::riscv_nxv16i8x3, &RISCV::VRN3M2RegClass);
322 addRegisterClass(MVT::riscv_nxv16i8x4, &RISCV::VRN4M2RegClass);
323 addRegisterClass(MVT::riscv_nxv32i8x2, &RISCV::VRN2M4RegClass);
324 }
325
326 // fixed vector is stored in GPRs for P extension packed operations
327 if (Subtarget.hasStdExtP()) {
328 if (Subtarget.is64Bit()) {
329 addRegisterClass(MVT::v2i32, &RISCV::GPRRegClass);
330 addRegisterClass(MVT::v4i16, &RISCV::GPRRegClass);
331 addRegisterClass(MVT::v8i8, &RISCV::GPRRegClass);
332 } else {
333 addRegisterClass(MVT::v2i16, &RISCV::GPRRegClass);
334 addRegisterClass(MVT::v4i8, &RISCV::GPRRegClass);
335
336 addRegisterClass(MVT::v2i32, &RISCV::GPRPairRegClass);
337 addRegisterClass(MVT::v4i16, &RISCV::GPRPairRegClass);
338 addRegisterClass(MVT::v8i8, &RISCV::GPRPairRegClass);
339 }
340 }
341
342 // Compute derived properties from the register classes.
343 computeRegisterProperties(STI.getRegisterInfo());
344
345 setStackPointerRegisterToSaveRestore(RISCV::X2);
346
347 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
348 MVT::i1, Promote);
349 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
350 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
351 MVT::i1, Promote);
352
353 // TODO: add all necessary setOperationAction calls.
354 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Custom);
355
356 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
357 setOperationAction(ISD::BR_CC, XLenVT, Expand);
358 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
359 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
360
361 setCondCodeAction(ISD::SETGT, XLenVT, Custom);
362 setCondCodeAction(ISD::SETGE, XLenVT, Expand);
363 setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
364 setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
365 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
366 setCondCodeAction(ISD::SETULE, XLenVT, Expand);
367 setCondCodeAction(ISD::SETLE, XLenVT, Expand);
368 }
369
370 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
371
372 setOperationAction(ISD::VASTART, MVT::Other, Custom);
373 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
374
375 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
376 !Subtarget.hasVendorXAndesPerf())
377 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
378
379 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
380
381 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
382 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
383 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
384 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
385
386 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
387 setOperationAction(ISD::LOAD, MVT::i64, Custom);
388 setOperationAction(ISD::STORE, MVT::i64, Custom);
389 }
390
391 if (Subtarget.is64Bit()) {
392 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
393
394 setOperationAction(ISD::LOAD, MVT::i32, Custom);
395 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
396 MVT::i32, Custom);
397 setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
398 setOperationAction({ISD::SADDO, ISD::SSUBO}, MVT::i32, Custom);
399 } else if (Subtarget.hasStdExtP()) {
400 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
401 setOperationAction({ISD::ADD, ISD::SUB}, MVT::i64, Custom);
402 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
403 }
404 if (!Subtarget.hasStdExtZmmul()) {
405 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
406 } else if (Subtarget.is64Bit()) {
407 setOperationAction(ISD::MUL, MVT::i128, Custom);
408 setOperationAction(ISD::MUL, MVT::i32, Custom);
409 } else {
410 setOperationAction(ISD::MUL, MVT::i64, Custom);
411 }
412
413 if (!Subtarget.hasStdExtM()) {
414 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, XLenVT,
415 Expand);
416 } else if (Subtarget.is64Bit()) {
417 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
418 {MVT::i8, MVT::i16, MVT::i32}, Custom);
419 }
420
421 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, XLenVT, Expand);
422
423 // On RV32, the P extension has a WMUL(U) instruction we can use for
424 // (S/U)MUL_LOHI.
425 // FIXME: Does P imply Zmmul?
426 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
427 Subtarget.is64Bit())
428 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, Expand);
429
430 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
431 Custom);
432
433 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
434 if (Subtarget.is64Bit())
435 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
436 } else if (Subtarget.hasVendorXTHeadBb()) {
437 if (Subtarget.is64Bit())
438 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
439 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
440 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
441 setOperationAction(ISD::ROTL, XLenVT, Expand);
442 } else {
443 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
444 }
445
446 if (Subtarget.hasStdExtP())
447 setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Legal);
448
449 setOperationAction(ISD::BSWAP, XLenVT,
450 Subtarget.hasREV8Like() ? Legal : Expand);
451
452 if (Subtarget.hasREVLike()) {
453 setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
454 } else {
455 // Zbkb can use rev8+brev8 to implement bitreverse.
456 setOperationAction(ISD::BITREVERSE, XLenVT,
457 Subtarget.hasStdExtZbkb() ? Custom : Expand);
458 if (Subtarget.hasStdExtZbkb())
459 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
460 }
461
462 if (Subtarget.hasStdExtZbb() ||
463 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
464 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
465 Legal);
466 }
467
468 if (Subtarget.hasCTZLike()) {
469 if (Subtarget.is64Bit())
470 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
471 } else {
472 setOperationAction(ISD::CTTZ, XLenVT, Expand);
473 }
474
475 if (!Subtarget.hasCPOPLike()) {
476 // TODO: These should be set to LibCall, but this currently breaks
477 // the Linux kernel build. See #101786. Lacks i128 tests, too.
478 if (Subtarget.is64Bit())
479 setOperationAction(ISD::CTPOP, MVT::i128, Expand);
480 else
481 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
482 setOperationAction(ISD::CTPOP, MVT::i64, Expand);
483 }
484
485 if (Subtarget.hasCLZLike()) {
486 // We need the custom lowering to make sure that the resulting sequence
487 // for the 32bit case is efficient on 64bit targets.
488 // Use default promotion for i32 without Zbb.
489 if (Subtarget.is64Bit() &&
490 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
491 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
492 } else {
493 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
494 setOperationAction(ISD::CTLZ_ZERO_POISON, XLenVT, Legal);
495 setOperationAction(ISD::CTLZ, XLenVT, Expand);
496 }
497
498 if (Subtarget.hasStdExtP()) {
499 setOperationAction(ISD::CTLS, XLenVT, Legal);
500 if (Subtarget.is64Bit())
501 setOperationAction(ISD::CTLS, MVT::i32, Custom);
502 }
503
504 if (Subtarget.hasStdExtP() ||
505 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
506 setOperationAction(ISD::ABS, XLenVT, Legal);
507 if (Subtarget.is64Bit())
508 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
509 } else if (Subtarget.hasShortForwardBranchIALU()) {
510 // We can use PseudoCCSUB to implement ABS.
511 setOperationAction(ISD::ABS, XLenVT, Legal);
512 } else if (Subtarget.is64Bit()) {
513 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
514 }
515
516 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
517 setOperationAction(ISD::SELECT, XLenVT, Custom);
518
519 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
520 !Subtarget.is64Bit()) {
521 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
522 MVT::i32, Legal);
523 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
524 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
525 MVT::i32, Custom);
526 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
527 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
528 MVT::i32, Custom);
529 }
530
531 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
532 !Subtarget.is64Bit()) {
533 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
534 // pssha.w/psshl.w and extracting.
535 setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
536 setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
537 }
538
539 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
540 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
541 // paadd.w, paaddu.w and extracting.
542 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, MVT::i32, Legal);
543 }
544
545 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) {
546 setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Legal);
547 if (Subtarget.hasStdExtZbc())
548 setOperationAction(ISD::CLMULR, XLenVT, Legal);
549 } else if (Subtarget.hasStdExtZvbc() && Subtarget.is64Bit()) {
550 // FIXME: Support i32 on RV32 by zexting from XLEN to i64 and extracting
551 // half of the result (low for CLMUL, high for CLMULH).
552 // TODO: Zvbc32e allows us to do a lot more here.
553 setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Custom);
554 }
555
556 static const unsigned FPLegalNodeTypes[] = {
557 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
558 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
559 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
560 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
561 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
562 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
563 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
564
565 static const ISD::CondCode FPCCToExpand[] = {
566 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
567 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
568 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
569
570 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
571 ISD::FPOW};
572 static const unsigned FPOpToLibCall[] = {ISD::FREM};
573
574 static const unsigned FPRndMode[] = {
575 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
576 ISD::FROUNDEVEN};
577
578 static const unsigned ZfhminZfbfminPromoteOps[] = {
579 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUM,
580 ISD::FMAXIMUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM,
581 ISD::FADD, ISD::FSUB, ISD::FMUL,
582 ISD::FMA, ISD::FDIV, ISD::FSQRT,
583 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
584 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
585 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
586 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC,
587 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN,
588 ISD::FCANONICALIZE};
589
590 if (Subtarget.hasStdExtP()) {
591 static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
592 static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
593 ArrayRef<MVT> VTs;
594 if (Subtarget.is64Bit()) {
595 VTs = P64VecVTs;
596 // There's no instruction for vector shamt in P extension so we unroll to
597 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
598 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
599 // widen for those operations that will be unrolled.
600 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA},
601 {MVT::v2i16, MVT::v4i8}, Custom);
602 setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::v2i16, MVT::v4i8},
603 Custom);
604 // Operand legalization queries the action using the illegal subvector.
605 setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
606 Custom);
607 } else {
608 VTs = P32VecVTs;
609 }
610 // By default everything must be expanded.
611 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
612 setOperationAction(Op, VTs, Expand);
613
614 for (MVT VT : VTs) {
615 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
616 setTruncStoreAction(VT, OtherVT, Expand);
617 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
618 OtherVT, Expand);
619 }
620 }
621
622 setOperationAction({ISD::LOAD, ISD::STORE}, VTs, Legal);
623 setOperationAction({ISD::ADD, ISD::SUB}, VTs, Legal);
624 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VTs, Legal);
625 setOperationAction(ISD::UADDSAT, VTs, Legal);
626 setOperationAction(ISD::SADDSAT, VTs, Legal);
627 setOperationAction(ISD::USUBSAT, VTs, Legal);
628 setOperationAction(ISD::SSUBSAT, VTs, Legal);
629 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Legal);
630 setOperationAction(ISD::BITREVERSE, VTs, Legal);
631 setOperationAction(ISD::VECTOR_SHUFFLE, VTs, Custom);
632 setOperationAction(ISD::VECTOR_REVERSE, VTs, Legal);
633 for (MVT VT : VTs) {
634 if (VT != MVT::v2i32)
635 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Legal);
636 if (VT.getVectorElementType() != MVT::i8) {
637 setOperationAction(ISD::SSHLSAT, VT, Custom);
638 setOperationAction(ISD::BSWAP, VT, Legal);
639 }
640 }
641 setOperationAction({ISD::UNDEF, ISD::POISON}, VTs, Legal);
642 setOperationAction(ISD::SPLAT_VECTOR, VTs, Legal);
643 setOperationAction(ISD::BUILD_VECTOR, VTs, Legal);
644 setOperationAction(ISD::SCALAR_TO_VECTOR, VTs, Legal);
645 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Custom);
646 setOperationAction(ISD::BITCAST, VTs, Custom);
647 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
648 Custom);
649 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
650 Legal);
651 setOperationAction(ISD::SELECT, VTs, Custom);
652 setOperationAction(ISD::VSELECT, VTs, Legal);
653 setOperationAction(ISD::SETCC, VTs, Legal);
654 setCondCodeAction(
655 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
656 Expand);
657 setCondCodeAction({ISD::SETNE, ISD::SETGT}, VTs, Custom);
658
659 if (!Subtarget.is64Bit())
660 setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v4i8}, Custom);
661
662 // P extension vector comparisons produce all 1s for true, all 0s for false
663 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
664
665 if (!Subtarget.is64Bit()) {
666 // By default everything must be expanded.
667 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
668 setOperationAction(Op, P64VecVTs, Expand);
669
670 for (MVT VT : P64VecVTs) {
671 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
672 setTruncStoreAction(VT, OtherVT, Expand);
673 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
674 OtherVT, Expand);
675 }
676 }
677
678 setOperationAction({ISD::UNDEF, ISD::POISON}, P64VecVTs, Legal);
679 setOperationAction({ISD::LOAD, ISD::STORE}, P64VecVTs, Custom);
680 setOperationAction(ISD::BITCAST, P64VecVTs, Custom);
681 setOperationAction({ISD::ADD, ISD::SUB}, P64VecVTs, Legal);
682 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, {MVT::v4i16, MVT::v8i8},
683 Custom);
684 setOperationAction(
685 {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
686 Legal);
687 setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
688 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
689 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
690 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
691 P64VecVTs, Legal);
692 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU},
693 {MVT::v4i16, MVT::v8i8}, Legal);
694 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, P64VecVTs, Custom);
695 setOperationAction(ISD::SSHLSAT, {MVT::v2i32, MVT::v4i16}, Custom);
696 setOperationAction(ISD::BSWAP, MVT::v4i16, Legal);
697 setOperationAction(ISD::BITREVERSE, {MVT::v4i16, MVT::v8i8}, Legal);
698 setOperationAction(ISD::VECTOR_SHUFFLE, P64VecVTs, Custom);
699 setOperationAction(ISD::VECTOR_REVERSE, P64VecVTs, Custom);
700 setOperationAction(ISD::SPLAT_VECTOR, P64VecVTs, Legal);
701 setOperationAction(ISD::BUILD_VECTOR, P64VecVTs, Legal);
702 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Legal);
703 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
704 {MVT::v4i16, MVT::v8i8}, Custom);
705 setOperationAction(ISD::CONCAT_VECTORS, {MVT::v4i16, MVT::v8i8}, Legal);
706 setOperationAction(ISD::EXTRACT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
707 Legal);
708 setOperationAction({ISD::SELECT, ISD::VSELECT}, {MVT::v4i16, MVT::v8i8},
709 Custom);
710 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU},
711 {MVT::v4i16, MVT::v8i8}, Custom);
712 setOperationAction(ISD::MUL, P32VecVTs, Custom);
713 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v4i8, Custom);
714 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v2i16, Legal);
715 setOperationAction({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
716 {MVT::v4i16, MVT::v2i32}, Legal);
717 setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Legal);
718 setOperationAction(ISD::SETCC, P64VecVTs, Legal);
719 setCondCodeAction(
720 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
721 P64VecVTs, Expand);
722 setCondCodeAction({ISD::SETNE, ISD::SETGT}, P64VecVTs, Custom);
723 // Operation legalization queries the action using the result type.
724 setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v4i16, MVT::v8i8},
725 Custom);
726 } else {
727 setOperationAction(ISD::MUL, P64VecVTs, Legal);
728 setOperationAction({ISD::MULHS, ISD::MULHU}, {MVT::v2i32, MVT::v4i16},
729 Legal);
730 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v8i8, Custom);
731 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG,
732 {MVT::v4i16, MVT::v2i32}, Legal);
733 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, {MVT::v4i16, MVT::v2i32},
734 Custom);
735 setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Custom);
736 }
737 // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
738 setOperationAction(
739 ISD::SIGN_EXTEND_INREG,
740 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}, Legal);
741 }
742
743 if (Subtarget.hasStdExtZfbfmin()) {
744 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
745 setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
746 setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
747 setOperationAction(ISD::SELECT, MVT::bf16, Custom);
748 setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
749 setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
750 setOperationAction(ISD::FREM, MVT::bf16, Promote);
751 setOperationAction(ISD::FABS, MVT::bf16, Custom);
752 setOperationAction(ISD::FNEG, MVT::bf16, Custom);
753 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
754 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
755 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
756 }
757
758 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
759 if (Subtarget.hasStdExtZfhOrZhinx()) {
760 setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
761 setOperationAction(FPRndMode, MVT::f16,
762 Subtarget.hasStdExtZfa() ? Legal : Custom);
763 setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
764 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
765 Subtarget.hasStdExtZfa() ? Legal : Custom);
766 if (Subtarget.hasStdExtZfa())
767 setOperationAction(ISD::ConstantFP, MVT::f16, Custom);
768 } else {
769 setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
770 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Promote);
771 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
772 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
773 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
774 setOperationAction(Op, MVT::f16, Custom);
775 setOperationAction(ISD::FABS, MVT::f16, Custom);
776 setOperationAction(ISD::FNEG, MVT::f16, Custom);
777 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
778 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
779 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
780 }
781
782 if (!Subtarget.hasStdExtD()) {
783 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
784 // instruction runs into double rounding issues, so this is wrong.
785 // Normally we'd use an f64 fma, but without the D extension the f64 type
786 // is not legal. This should probably be a libcall.
787 AddPromotedToType(ISD::FMA, MVT::f16, MVT::f32);
788 AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f32);
789 }
790
791 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
792
793 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
794 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
795 setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
796 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
797 setOperationAction(ISD::SELECT, MVT::f16, Custom);
798 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
799
800 setOperationAction(
801 ISD::FNEARBYINT, MVT::f16,
802 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
803 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
804 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
805 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
806 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
807 MVT::f16, Promote);
808
809 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
810 // complete support for all operations in LegalizeDAG.
811 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
812 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
813 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
814 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
815 MVT::f16, Promote);
816
817 // We need to custom promote this.
818 if (Subtarget.is64Bit())
819 setOperationAction(ISD::FPOWI, MVT::i32, Custom);
820 }
821
822 if (Subtarget.hasStdExtFOrZfinx()) {
823 setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
824 setOperationAction(FPRndMode, MVT::f32,
825 Subtarget.hasStdExtZfa() ? Legal : Custom);
826 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
827 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
828 setOperationAction(ISD::SELECT, MVT::f32, Custom);
829 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
830 setOperationAction(FPOpToExpand, MVT::f32, Expand);
831 setOperationAction(FPOpToLibCall, MVT::f32, LibCall);
832 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
833 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
834 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
835 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
836 setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
837 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
838 setOperationAction(ISD::FP_TO_BF16, MVT::f32,
839 Subtarget.isSoftFPABI() ? LibCall : Custom);
840 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
841 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
842 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
843 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
844
845 if (Subtarget.hasStdExtZfa()) {
846 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
847 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
848 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
849 } else {
850 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
851 }
852 }
853
854 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
855 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
856
857 if (Subtarget.hasStdExtDOrZdinx()) {
858 setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
859
860 if (!Subtarget.is64Bit())
861 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
862
863 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
864 !Subtarget.is64Bit()) {
865 setOperationAction(ISD::LOAD, MVT::f64, Custom);
866 setOperationAction(ISD::STORE, MVT::f64, Custom);
867 }
868
869 if (Subtarget.hasStdExtZfa()) {
870 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
871 setOperationAction(FPRndMode, MVT::f64, Legal);
872 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
873 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
874 } else {
875 if (Subtarget.is64Bit())
876 setOperationAction(FPRndMode, MVT::f64, Custom);
877
878 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
879 }
880
881 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
882 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
883 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
884 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
885 setOperationAction(ISD::SELECT, MVT::f64, Custom);
886 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
887 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
888 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
889 setOperationAction(FPOpToExpand, MVT::f64, Expand);
890 setOperationAction(FPOpToLibCall, MVT::f64, LibCall);
891 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
892 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
893 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
894 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
895 setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
896 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
897 setOperationAction(ISD::FP_TO_BF16, MVT::f64,
898 Subtarget.isSoftFPABI() ? LibCall : Custom);
899 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
900 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
901 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
902 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
903 }
904
905 if (Subtarget.is64Bit()) {
906 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
907 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
908 MVT::i32, Custom);
909 setOperationAction(ISD::LROUND, MVT::i32, Custom);
910 }
911
912 if (Subtarget.hasStdExtFOrZfinx()) {
913 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
914 Custom);
915
916 // f16/bf16 require custom handling.
917 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, XLenVT,
918 Custom);
919 setOperationAction({ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, XLenVT,
920 Custom);
921
922 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
923 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
924 setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
925 setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
926 setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
927 setOperationAction(ISD::GET_FPMODE, XLenVT, Custom);
928 setOperationAction(ISD::SET_FPMODE, XLenVT, Custom);
929 setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
930 }
931
932 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
933 ISD::JumpTable},
934 XLenVT, Custom);
935
936 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
937
938 if (Subtarget.is64Bit())
939 setOperationAction(ISD::Constant, MVT::i64, Custom);
940
941 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
942 // Unfortunately this can't be determined just from the ISA naming string.
943 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
944 Subtarget.is64Bit() ? Legal : Custom);
945 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
946 Subtarget.is64Bit() ? Legal : Custom);
947
948 if (Subtarget.is64Bit()) {
949 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
950 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
951 }
952
953 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
954 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
955 if (Subtarget.is64Bit())
956 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
957
958 if (Subtarget.hasVendorXMIPSCBOP())
959 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
960 else
961 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
962
963 if (Subtarget.hasStdExtZalrsc()) {
964 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
965 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
966 setMinCmpXchgSizeInBits(8);
967 else
968 setMinCmpXchgSizeInBits(32);
969 } else if (Subtarget.hasForcedAtomics()) {
970 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
971 } else {
972 setMaxAtomicSizeInBitsSupported(0);
973 }
974
975 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
976
977 setBooleanContents(ZeroOrOneBooleanContent);
978
979 if (getTargetMachine().getTargetTriple().isOSLinux()) {
980 // Custom lowering of llvm.clear_cache.
981 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
982 }
983
984 if (Subtarget.hasVInstructions()) {
985 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
986
987 setOperationAction(ISD::VSCALE, XLenVT, Custom);
988
989 // RVV intrinsics may have illegal operands.
990 // We also need to custom legalize vmv.x.s.
991 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
992 ISD::INTRINSIC_VOID},
993 {MVT::i8, MVT::i16}, Custom);
994 if (Subtarget.is64Bit())
995 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
996 MVT::i32, Custom);
997 else
998 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
999 MVT::i64, Custom);
1000
1001 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
1002 MVT::Other, Custom);
1003
1004 static const unsigned IntegerVPOps[] = {
1005 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
1006 ISD::VP_UREM, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1007 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
1008 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
1009 ISD::VP_MERGE,
1010 ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
1011 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON};
1012
1013 static const unsigned FloatingPointVPOps[] = {
1014 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
1015 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
1016 ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
1017
1018 static const unsigned IntegerVecReduceOps[] = {
1019 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
1020 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
1021 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
1022
1023 static const unsigned FloatingPointVecReduceOps[] = {
1024 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
1025 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
1026
1027 static const unsigned FloatingPointLibCallOps[] = {
1028 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
1029 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
1030
1031 if (!Subtarget.is64Bit()) {
1032 // We must custom-lower certain vXi64 operations on RV32 due to the vector
1033 // element type being illegal.
1034 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1035 MVT::i64, Custom);
1036
1037 setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
1038
1039 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1040 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
1041 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
1042 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
1043 MVT::i64, Custom);
1044 }
1045
1046 for (MVT VT : BoolVecVTs) {
1047 if (!isTypeLegal(VT))
1048 continue;
1049
1050 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1051
1052 // Mask VTs are custom-expanded into a series of standard nodes
1053 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
1054 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1055 ISD::SCALAR_TO_VECTOR},
1056 VT, Custom);
1057
1058 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1059 Custom);
1060
1061 setOperationAction(ISD::SELECT, VT, Custom);
1062 setOperationAction({ISD::SELECT_CC, ISD::VSELECT}, VT,
1063 Expand);
1064 setOperationAction(ISD::VP_MERGE, VT, Custom);
1065
1066 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
1067 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
1068 VT, Custom);
1069
1070 setOperationAction(
1071 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1072 Custom);
1073
1074 setOperationAction(
1075 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1076 Custom);
1077
1078 // RVV has native int->float & float->int conversions where the
1079 // element type sizes are within one power-of-two of each other. Any
1080 // wider distances between type sizes have to be lowered as sequences
1081 // which progressively narrow the gap in stages.
1082 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1083 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1084 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1085 ISD::STRICT_FP_TO_UINT},
1086 VT, Custom);
1087 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1088 Custom);
1089
1090 // Expand all extending loads to types larger than this, and truncating
1091 // stores from types larger than this.
1092 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1093 setTruncStoreAction(VT, OtherVT, Expand);
1094 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1095 OtherVT, Expand);
1096 }
1097
1098 setVectorInterleaveAction(
1099 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1100 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1101
1102 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
1103
1104 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1105 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1106
1107 setOperationPromotedToType(
1108 {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
1109 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
1110 }
1111
1112 for (MVT VT : IntVecVTs) {
1113 if (!isTypeLegal(VT))
1114 continue;
1115
1116 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1117 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1118
1119 // Vectors implement MULHS/MULHU.
1120 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
1121
1122 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1123 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1124 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
1125
1126 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1127 Legal);
1128
1129 if (Subtarget.hasStdExtZvabd()) {
1130 setOperationAction(ISD::ABS, VT, Legal);
1131 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Legal);
1132 } else {
1133 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1134 }
1135
1136 // Custom-lower extensions and truncations from/to mask types.
1137 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1138 VT, Custom);
1139
1140 // RVV has native int->float & float->int conversions where the
1141 // element type sizes are within one power-of-two of each other. Any
1142 // wider distances between type sizes have to be lowered as sequences
1143 // which progressively narrow the gap in stages.
1144 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1145 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1146 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1147 ISD::STRICT_FP_TO_UINT},
1148 VT, Custom);
1149 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1150 Custom);
1151 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1152 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1153 ISD::SSUBSAT, ISD::USUBSAT},
1154 VT, Legal);
1155
1156 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1157 // nodes which truncate by one power of two at a time.
1158 setOperationAction(
1159 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1160 Custom);
1161
1162 // Custom-lower insert/extract operations to simplify patterns.
1163 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1164 Custom);
1165
1166 // Custom-lower reduction operations to set up the corresponding custom
1167 // nodes' operands.
1168 setOperationAction(IntegerVecReduceOps, VT, Custom);
1169
1170 setOperationAction(IntegerVPOps, VT, Custom);
1171
1172 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1173
1174 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1175 VT, Custom);
1176
1177 setOperationAction(
1178 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1179 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1180 VT, Custom);
1181 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1182
1183 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1184 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1185 VT, Custom);
1186
1187 setOperationAction(ISD::SELECT, VT, Custom);
1188 setOperationAction(ISD::SELECT_CC, VT, Expand);
1189
1190 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
1191
1192 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1193 setTruncStoreAction(VT, OtherVT, Expand);
1194 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1195 OtherVT, Expand);
1196 }
1197
1198 setVectorInterleaveAction(
1199 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1200 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1201
1202 setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1203 VT, Custom);
1204
1205 if (Subtarget.hasStdExtZvkb()) {
1206 setOperationAction(ISD::BSWAP, VT, Legal);
1207 } else {
1208 setOperationAction(ISD::BSWAP, VT, Expand);
1209 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
1210 }
1211
1212 if (Subtarget.hasStdExtZvbb()) {
1213 setOperationAction(ISD::BITREVERSE, VT, Legal);
1214 } else {
1215 setOperationAction(ISD::BITREVERSE, VT, Expand);
1216 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
1217
1218 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1219 // range of f32.
1220 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1221 if (isTypeLegal(FloatVT)) {
1222 setOperationAction(
1223 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1224 Custom);
1225 }
1226 }
1227
1228 if (VT.getVectorElementType() == MVT::i64) {
1229 if (Subtarget.hasStdExtZvbc())
1230 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1231 } else {
1232 if (Subtarget.hasStdExtZvbc32e()) {
1233 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1234 } else if (Subtarget.hasStdExtZvbc()) {
1235 // Promote to i64 if the lmul is small enough.
1236 // FIXME: Split if necessary to widen.
1237 // FIXME: Promote clmulh directly without legalizing to clmul first.
1238 MVT I64VecVT = MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
1239 if (isTypeLegal(I64VecVT))
1240 setOperationAction(ISD::CLMUL, VT, Custom);
1241 }
1242 }
1243
1244 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1245 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
1246 ISD::MASKED_SREM},
1247 VT, Legal);
1248 }
1249
1250 for (MVT VT : VecTupleVTs) {
1251 if (!isTypeLegal(VT))
1252 continue;
1253
1254 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1255 }
1256
1257 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1258 // but no other unordered comparisons, and supports all ordered comparisons
1259 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1260 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1261 // and we pattern-match those back to the "original", swapping operands once
1262 // more. This way we catch both operations and both "vf" and "fv" forms with
1263 // fewer patterns.
1264 static const ISD::CondCode VFPCCToExpand[] = {
1265 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1266 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1267 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1268 };
1269
1270 // TODO: support more ops.
1271 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1272 ISD::FMINNUM,
1273 ISD::FMAXNUM,
1274 ISD::FMINIMUMNUM,
1275 ISD::FMAXIMUMNUM,
1276 ISD::FADD,
1277 ISD::FSUB,
1278 ISD::FMUL,
1279 ISD::FMA,
1280 ISD::FDIV,
1281 ISD::FSQRT,
1282 ISD::FCEIL,
1283 ISD::FTRUNC,
1284 ISD::FFLOOR,
1285 ISD::FROUND,
1286 ISD::FROUNDEVEN,
1287 ISD::FRINT,
1288 ISD::FNEARBYINT,
1289 ISD::IS_FPCLASS,
1290 ISD::SETCC,
1291 ISD::FMAXIMUM,
1292 ISD::FMINIMUM,
1293 ISD::STRICT_FADD,
1294 ISD::STRICT_FSUB,
1295 ISD::STRICT_FMUL,
1296 ISD::STRICT_FDIV,
1297 ISD::STRICT_FSQRT,
1298 ISD::STRICT_FMA,
1299 ISD::VECREDUCE_FADD,
1300 ISD::VECREDUCE_FMIN,
1301 ISD::VECREDUCE_FMAX,
1302 ISD::VECREDUCE_FMINIMUM,
1303 ISD::VECREDUCE_FMAXIMUM,
1304 ISD::FCANONICALIZE};
1305
1306 // TODO: Make more of these ops legal.
1307 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1308 ISD::FSQRT,
1309 ISD::FCEIL,
1310 ISD::FTRUNC,
1311 ISD::FFLOOR,
1312 ISD::FROUND,
1313 ISD::FROUNDEVEN,
1314 ISD::FRINT,
1315 ISD::FNEARBYINT,
1316 ISD::STRICT_FDIV,
1317 ISD::STRICT_FSQRT,
1318 ISD::VECREDUCE_FADD,
1319 ISD::VECREDUCE_FMIN,
1320 ISD::VECREDUCE_FMAX,
1321 ISD::VECREDUCE_FMINIMUM,
1322 ISD::VECREDUCE_FMAXIMUM};
1323
1324 // TODO: support more vp ops.
1325 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1326 ISD::VP_REDUCE_FMIN,
1327 ISD::VP_REDUCE_FMAX,
1328 ISD::VP_REDUCE_FMINIMUM,
1329 ISD::VP_REDUCE_FMAXIMUM};
1330
1331 // Sets common operation actions on RVV floating-point vector types.
1332 const auto SetCommonVFPActions = [&](MVT VT) {
1333 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1334 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1335 // sizes are within one power-of-two of each other. Therefore conversions
1336 // between vXf16 and vXf64 must be lowered as sequences which convert via
1337 // vXf32.
1338 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1339 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1340 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1341 // Custom-lower insert/extract operations to simplify patterns.
1342 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1343 Custom);
1344 // Expand various condition codes (explained above).
1345 setCondCodeAction(VFPCCToExpand, VT, Expand);
1346
1347 setOperationAction(
1348 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1349 Legal);
1350 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1351
1352 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1353 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1354 ISD::IS_FPCLASS},
1355 VT, Custom);
1356
1357 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1358
1359 // Expand FP operations that need libcalls.
1360 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1361
1362 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1363
1364 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1365
1366 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1367
1368 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1369 VT, Custom);
1370
1371 setOperationAction(
1372 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1373 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1374 VT, Custom);
1375 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1376
1377 setOperationAction(ISD::SELECT, VT, Custom);
1378 setOperationAction(ISD::SELECT_CC, VT, Expand);
1379
1380 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1381 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1382 VT, Custom);
1383
1384 setVectorInterleaveAction(
1385 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1386 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1387
1388 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1389 ISD::VECTOR_SPLICE_RIGHT},
1390 VT, Custom);
1391 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1392 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1393
1394 setOperationAction(FloatingPointVPOps, VT, Custom);
1395
1396 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1397 Custom);
1398 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1399 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1400 VT, Legal);
1401 setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1402 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1403 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1404 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1405 VT, Custom);
1406
1407 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1408 };
1409
1410 // Sets common extload/truncstore actions on RVV floating-point vector
1411 // types.
1412 const auto SetCommonVFPExtLoadTruncStoreActions =
1413 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1414 for (auto SmallVT : SmallerVTs) {
1415 setTruncStoreAction(VT, SmallVT, Expand);
1416 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
1417 }
1418 };
1419
1420 // Sets common actions for f16 and bf16 for when there's only
1421 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1422 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1423 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1424 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1425 Custom);
1426 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1427 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1428 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1429 Custom);
1430 setOperationAction(ISD::SELECT_CC, VT, Expand);
1431 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1432 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1433 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1434 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1435 VT, Custom);
1436 setVectorInterleaveAction(
1437 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1438 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1439 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1440 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1441 MVT EltVT = VT.getVectorElementType();
1442 if (isTypeLegal(EltVT))
1443 setOperationAction({ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1444 VT, Custom);
1445 else
1446 setOperationAction(ISD::SPLAT_VECTOR, EltVT, Custom);
1447 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1448 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1449 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1450 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1451 ISD::VP_SCATTER},
1452 VT, Custom);
1453 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1454
1455 setOperationAction(ISD::FNEG, VT, Expand);
1456 setOperationAction(ISD::FABS, VT, Expand);
1457 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1458
1459 // Expand FP operations that need libcalls.
1460 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1461
1462 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1463
1464 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1465 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1466 setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
1467 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1468 } else {
1469 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1470 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1471 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1472 }
1473 };
1474
1475 // Sets common actions for zvfbfa, some of instructions are supported
1476 // natively so that we don't need to promote them.
1477 const auto SetZvfbfaActions = [&](MVT VT) {
1478 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1479 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1480 Custom);
1481 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1482 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1483 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1484 Custom);
1485 setOperationAction(ISD::SELECT_CC, VT, Expand);
1486 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1487 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1488 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1489 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1490 ISD::VECTOR_COMPRESS},
1491 VT, Custom);
1492 setVectorInterleaveAction(
1493 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1494 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1495 setOperationAction(
1496 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1497 Legal);
1498 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1499 setOperationAction(ISD::IS_FPCLASS, VT, Custom);
1500 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1501 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1502
1503 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1504 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1505 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1506 ISD::STRICT_FMA},
1507 VT, Legal);
1508 setCondCodeAction(VFPCCToExpand, VT, Expand);
1509
1510 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1511 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1512 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1513 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1514 ISD::VP_SCATTER},
1515 VT, Custom);
1516 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1517
1518 // Expand FP operations that need libcalls.
1519 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1520
1521 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1522
1523 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1524 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1525 setOperationAction(ZvfbfaPromoteOps, VT, Custom);
1526 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1527 } else {
1528 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1529 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1530 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1531 }
1532 };
1533
1534 if (Subtarget.hasVInstructionsF16()) {
1535 for (MVT VT : F16VecVTs) {
1536 if (!isTypeLegal(VT))
1537 continue;
1538 SetCommonVFPActions(VT);
1539 }
1540 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1541 for (MVT VT : F16VecVTs) {
1542 if (!isTypeLegal(VT))
1543 continue;
1544 SetCommonPromoteToF32Actions(VT);
1545 }
1546 }
1547
1548 if (Subtarget.hasVInstructionsBF16()) {
1549 for (MVT VT : BF16VecVTs) {
1550 if (!isTypeLegal(VT))
1551 continue;
1552 SetZvfbfaActions(VT);
1553 }
1554 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1555 for (MVT VT : BF16VecVTs) {
1556 if (!isTypeLegal(VT))
1557 continue;
1558 SetCommonPromoteToF32Actions(VT);
1559 }
1560 }
1561
1562 if (Subtarget.hasStdExtZvfofp8min()) {
1563 for (MVT VT : BF16VecVTs) {
1564 if (!isTypeLegal(VT))
1565 continue;
1566 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
1567 }
1568 }
1569
1570 if (Subtarget.hasVInstructionsF32()) {
1571 for (MVT VT : F32VecVTs) {
1572 if (!isTypeLegal(VT))
1573 continue;
1574 SetCommonVFPActions(VT);
1575 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1576 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1577 }
1578 }
1579
1580 if (Subtarget.hasVInstructionsF64()) {
1581 for (MVT VT : F64VecVTs) {
1582 if (!isTypeLegal(VT))
1583 continue;
1584 SetCommonVFPActions(VT);
1585 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1586 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1587 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1588 }
1589 }
1590
1591 if (Subtarget.useRVVForFixedLengthVectors()) {
1592 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1593 if (!useRVVForFixedLengthVectorVT(VT))
1594 continue;
1595
1596 // By default everything must be expanded.
1597 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1598 setOperationAction(Op, VT, Expand);
1599 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1600 setTruncStoreAction(VT, OtherVT, Expand);
1601 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1602 OtherVT, Expand);
1603 }
1604
1605 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1606 // expansion to a build_vector of 0s.
1607 setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
1608
1609 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1610 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1611 Custom);
1612
1613 setOperationAction(
1614 {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1615 Custom);
1616
1617 setVectorInterleaveAction(
1618 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1619 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1620
1621 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1622 VT, Custom);
1623
1624 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1625
1626 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1627
1628 setOperationAction(ISD::SETCC, VT, Custom);
1629
1630 setOperationAction(ISD::SELECT, VT, Custom);
1631
1632 setOperationAction(
1633 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1634 Custom);
1635
1636 setOperationAction(ISD::BITCAST, VT, Custom);
1637
1638 setOperationAction(
1639 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1640 Custom);
1641
1642 setOperationAction(
1643 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1644 Custom);
1645
1646 setOperationAction(
1647 {
1648 ISD::SINT_TO_FP,
1649 ISD::UINT_TO_FP,
1650 ISD::FP_TO_SINT,
1651 ISD::FP_TO_UINT,
1652 ISD::STRICT_SINT_TO_FP,
1653 ISD::STRICT_UINT_TO_FP,
1654 ISD::STRICT_FP_TO_SINT,
1655 ISD::STRICT_FP_TO_UINT,
1656 },
1657 VT, Custom);
1658 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1659 Custom);
1660
1661 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1662
1663 // Operations below are different for between masks and other vectors.
1664 if (VT.getVectorElementType() == MVT::i1) {
1665 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Custom);
1666
1667 setOperationAction(ISD::VP_MERGE, VT, Custom);
1668
1669 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1670 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1671
1672 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1673 Custom);
1674 continue;
1675 }
1676
1677 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1678 // it before type legalization for i64 vectors on RV32. It will then be
1679 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1680 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1681 // improvements first.
1682 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1683 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1684 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1685
1686 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1687 setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
1688 }
1689
1690 setOperationAction(
1691 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1692
1693 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1694 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1695 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1696 ISD::VP_SCATTER},
1697 VT, Custom);
1698 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1699
1700 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1701 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1702 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1703 VT, Custom);
1704
1705 setOperationAction(
1706 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
1707
1708 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1709
1710 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1711 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1712 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
1713
1714 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1715 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1716 ISD::SSUBSAT, ISD::USUBSAT},
1717 VT, Custom);
1718
1719 setOperationAction(ISD::VSELECT, VT, Custom);
1720
1721 setOperationAction(
1722 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
1723
1724 // Custom-lower reduction operations to set up the corresponding custom
1725 // nodes' operands.
1726 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1727 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1728 ISD::VECREDUCE_UMIN},
1729 VT, Custom);
1730
1731 setOperationAction(IntegerVPOps, VT, Custom);
1732
1733 if (Subtarget.hasStdExtZvkb())
1734 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
1735
1736 if (Subtarget.hasStdExtZvbb()) {
1737 setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
1738 ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
1739 VT, Custom);
1740 } else {
1741 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1742 // range of f32.
1743 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1744 if (isTypeLegal(FloatVT))
1745 setOperationAction(
1746 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1747 Custom);
1748 }
1749
1750 if (VT.getVectorElementType() == MVT::i64) {
1751 if (Subtarget.hasStdExtZvbc())
1752 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
1753 } else {
1754 if (Subtarget.hasStdExtZvbc32e()) {
1755 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
1756 } else if (Subtarget.hasStdExtZvbc()) {
1757 // Promote to i64 as is done for scalable vectors.
1758 MVT I64VecVT =
1759 MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
1760 if (I64VecVT.isValid() && useRVVForFixedLengthVectorVT(I64VecVT))
1761 setOperationAction(ISD::CLMUL, VT, Custom);
1762 }
1763 }
1764
1765 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1766 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV,
1767 ISD::MASKED_UREM, ISD::MASKED_SREM},
1768 VT, Custom);
1769 }
1770
1771 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1772 // There are no extending loads or truncating stores.
1773 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1774 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1775 setTruncStoreAction(VT, InnerVT, Expand);
1776 }
1777
1778 if (!useRVVForFixedLengthVectorVT(VT))
1779 continue;
1780
1781 // By default everything must be expanded.
1782 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1783 setOperationAction(Op, VT, Expand);
1784
1785 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1786 // expansion to a build_vector of 0s.
1787 setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
1788
1789 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1790 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1791 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1792 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1793 VT, Custom);
1794 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1795 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1796
1797 setVectorInterleaveAction(
1798 {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1799 {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
1800
1801 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1802 ISD::MGATHER, ISD::MSCATTER},
1803 VT, Custom);
1804 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1805 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1806 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1807 VT, Custom);
1808 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1809
1810 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1811 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1812 Custom);
1813
1814 setOperationAction(ISD::BITCAST, VT, Custom);
1815
1816 if (VT.getVectorElementType() == MVT::f16 &&
1817 !Subtarget.hasVInstructionsF16()) {
1818 setOperationAction(
1819 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1820 Custom);
1821 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1822 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1823 if (Subtarget.hasStdExtZfhmin()) {
1824 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1825 } else {
1826 // We need to custom legalize f16 build vectors if Zfhmin isn't
1827 // available.
1828 setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
1829 }
1830 setOperationAction(ISD::FNEG, VT, Expand);
1831 setOperationAction(ISD::FABS, VT, Expand);
1832 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1833 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1834 // Don't promote f16 vector operations to f32 if f32 vector type is
1835 // not legal.
1836 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1837 // TODO: Support more operations.
1838 if (!isTypeLegal(F32VecVT)) {
1839 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1840 ISD::VECREDUCE_FMAXIMUM,
1841 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1842 VT, Custom);
1843 setOperationAction(ISD::SETCC, VT, Custom);
1844 continue;
1845 }
1846 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1847 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1848 continue;
1849 }
1850
1851 if (VT.getVectorElementType() == MVT::bf16) {
1852 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1853 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1854 if (Subtarget.hasStdExtZvfofp8min())
1855 setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
1856 if (Subtarget.hasStdExtZfbfmin()) {
1857 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1858 } else {
1859 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1860 // available.
1861 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
1862 }
1863 if (Subtarget.hasVInstructionsBF16()) {
1864 setOperationAction(ZvfbfaOps, VT, Custom);
1865 setCondCodeAction(VFPCCToExpand, VT, Expand);
1866 }
1867 setOperationAction(
1868 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1869 Custom);
1870 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1871 // Don't promote bf16 vector operations to f32 if f32 vector type is
1872 // not legal.
1873 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1874 // TODO: Support more operations.
1875 if (!isTypeLegal(F32VecVT)) {
1876 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1877 ISD::VECREDUCE_FMAXIMUM,
1878 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1879 VT, Custom);
1880 setOperationAction(ISD::SETCC, VT, Custom);
1881 continue;
1882 }
1883
1884 if (Subtarget.hasVInstructionsBF16())
1885 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1886 else
1887 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1888 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1889 continue;
1890 }
1891
1892 setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1893 Custom);
1894
1895 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1896 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1897 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1898 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1899 ISD::FMAXIMUM, ISD::FMINIMUM},
1900 VT, Custom);
1901
1902 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1903 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1904 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1905 ISD::FNEARBYINT, ISD::FCANONICALIZE},
1906 VT, Custom);
1907
1908 setCondCodeAction(VFPCCToExpand, VT, Expand);
1909
1910 setOperationAction(ISD::SETCC, VT, Custom);
1911 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1912
1913 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1914
1915 setOperationAction(FloatingPointVPOps, VT, Custom);
1916
1917 setOperationAction(
1918 {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1919 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1920 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1921 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1922 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1923 VT, Custom);
1924 }
1925
1926 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1927 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
1928 if (Subtarget.is64Bit())
1929 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1930 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1931 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1932 if (Subtarget.hasStdExtZfbfmin())
1933 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
1934 if (Subtarget.hasStdExtFOrZfinx())
1935 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1936 if (Subtarget.hasStdExtDOrZdinx())
1937 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1938 }
1939 }
1940
1941 if (Subtarget.hasStdExtZaamo())
1942 setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
1943
1944 if (Subtarget.hasForcedAtomics()) {
1945 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1946 setOperationAction(
1947 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1948 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1949 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1950 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1951 XLenVT, LibCall);
1952 }
1953
1954 if (Subtarget.hasVendorXTHeadMemIdx()) {
1955 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1956 setIndexedLoadAction(im, MVT::i8, Legal);
1957 setIndexedStoreAction(im, MVT::i8, Legal);
1958 setIndexedLoadAction(im, MVT::i16, Legal);
1959 setIndexedStoreAction(im, MVT::i16, Legal);
1960 setIndexedLoadAction(im, MVT::i32, Legal);
1961 setIndexedStoreAction(im, MVT::i32, Legal);
1962
1963 if (Subtarget.is64Bit()) {
1964 setIndexedLoadAction(im, MVT::i64, Legal);
1965 setIndexedStoreAction(im, MVT::i64, Legal);
1966 }
1967 }
1968 }
1969
1970 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1971 setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
1972 setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
1973 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1974
1975 setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
1976 setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
1977 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1978 }
1979
1980 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1981 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1982 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1983 ISD::PARTIAL_REDUCE_UMLA,
1984 ISD::PARTIAL_REDUCE_SUMLA};
1985 setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
1986 setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
1987 setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
1988 setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
1989 setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
1990
1991 // An i64 accumulator is handled by performing an i32 vdot4a* and widening
1992 // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
1993 setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom);
1994 setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
1995 setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom);
1996 setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom);
1997
1998 if (Subtarget.useRVVForFixedLengthVectors()) {
1999 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
2000 if ((VT.getVectorElementType() != MVT::i32 &&
2001 VT.getVectorElementType() != MVT::i64) ||
2002 !useRVVForFixedLengthVectorVT(VT))
2003 continue;
2004 ElementCount EC = VT.getVectorElementCount();
2005 unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
2006 MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(Scale));
2007 setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
2008 }
2009 }
2010 }
2011
2012 // Customize load and store operation for bf16 if zfh isn't enabled.
2013 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
2014 setOperationAction(ISD::LOAD, MVT::bf16, Custom);
2015 setOperationAction(ISD::STORE, MVT::bf16, Custom);
2016 }
2017
2018 // Function alignments.
2019 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
2020 setMinFunctionAlignment(FunctionAlignment);
2021 // Set preferred alignments.
2022 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
2023 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
2024
2025 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
2026 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
2027 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
2028 ISD::SRA});
2029 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
2030
2031 if (Subtarget.hasStdExtFOrZfinx())
2032 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
2033
2034 // Allow scalar min/max to be combined with vector reductions.
2035 if (Subtarget.hasVInstructions())
2036 setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
2037 if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2038 setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
2039
2040 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
2041 Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2042 setTargetDAGCombine(ISD::TRUNCATE);
2043
2044 if (Subtarget.hasStdExtZbkb())
2045 setTargetDAGCombine(ISD::BITREVERSE);
2046
2047 if (Subtarget.hasStdExtFOrZfinx())
2048 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2049 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
2050 if (Subtarget.hasVInstructions())
2051 setTargetDAGCombine({ISD::FCOPYSIGN,
2052 ISD::VECTOR_MATCH,
2053 ISD::MGATHER,
2054 ISD::MSCATTER,
2055 ISD::MLOAD,
2056 ISD::VP_GATHER,
2057 ISD::VP_SCATTER,
2058 ISD::SRL,
2059 ISD::SHL,
2060 ISD::STORE,
2061 ISD::SPLAT_VECTOR,
2062 ISD::BUILD_VECTOR,
2063 ISD::CONCAT_VECTORS,
2064 ISD::VP_STORE,
2065 ISD::EXPERIMENTAL_VP_REVERSE,
2066 ISD::SDIV,
2067 ISD::UDIV,
2068 ISD::SREM,
2069 ISD::UREM,
2070 ISD::INSERT_VECTOR_ELT,
2071 ISD::ABS,
2072 ISD::ABS_MIN_POISON,
2073 ISD::CTPOP,
2074 ISD::VECTOR_SHUFFLE,
2075 ISD::FMA,
2076 ISD::VSELECT,
2077 ISD::VECREDUCE_ADD,
2078 ISD::VECTOR_SPLICE_RIGHT});
2079
2080 if (Subtarget.hasVendorXTHeadMemPair())
2081 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
2082 if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
2083 setTargetDAGCombine(ISD::BITCAST);
2084
2085 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2086
2087 setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2088
2089 setJumpIsExpensive(Subtarget.isJumpExpensive());
2090
2091 // Disable strict node mutation.
2092 IsStrictFPEnabled = true;
2093 EnableExtLdPromotion = true;
2094
2095 // Let the subtarget decide if a predictable select is more expensive than the
2096 // corresponding branch. This information is used in CGP/SelectOpt to decide
2097 // when to convert selects into branches.
2098 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
2099
2100 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
2101 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
2102
2103 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
2104 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
2105 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
2106
2107 MaxStoresPerMemmoveOptSize =
2108 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
2109 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
2110
2111 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
2112 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
2113}
2114
2115TargetLoweringBase::LegalizeTypeAction
2116RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
2117 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
2118 if (VT == MVT::v2i16 || VT == MVT::v4i8)
2119 return TypeWidenVector;
2120
2121 return TargetLoweringBase::getPreferredVectorAction(VT);
2122}
2123
2124EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
2125 LLVMContext &Context,
2126 EVT VT) const {
2127 if (!VT.isVector())
2128 return getPointerTy(DL);
2129 if (Subtarget.hasVInstructions() &&
2130 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
2131 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
2132 return VT.changeVectorElementTypeToInteger();
2133}
2134
2135TargetLoweringBase::CondMergingParams
2136RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
2137 const Value *LHS,
2138 const Value *RHS,
2139 const Function *F) const {
2140 if (F->hasOptSize())
2141 return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
2142
2143 // Merging conditions eliminates a branch, so the budget we are willing to
2144 // spend eagerly computing the RHS condition should scale with how expensive a
2145 // mispredicted branch is. A branch only costs the full penalty when actually
2146 // mispredicted, so scale it down by an assumed misprediction rate (~25%).
2147 int BaseCost = Subtarget.getMispredictionPenalty() / 4;
2148 if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
2149 BaseCost = BrMergingBaseCostThresh;
2150
2151 return {BaseCost, BrMergingLikelyBias, BrMergingUnlikelyBias};
2152}
2153
2154MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
2155 return Subtarget.getXLenVT();
2156}
2157
2158// Return false if we can lower get_vector_length to a vsetvli intrinsic.
2159bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
2160 unsigned VF,
2161 bool IsScalable) const {
2162 if (!Subtarget.hasVInstructions())
2163 return true;
2164
2165 if (!IsScalable)
2166 return true;
2167
2168 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2169 return true;
2170
2171 // Don't allow VF=1 if those types are't legal.
2172 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2173 return true;
2174
2175 // VLEN=32 support is incomplete.
2176 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2177 return true;
2178
2179 // The maximum VF is for the smallest element width with LMUL=8.
2180 // VF must be a power of 2.
2181 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2182 return VF > MaxVF || !isPowerOf2_32(VF);
2183}
2184
2185void RISCVTargetLowering::getTgtMemIntrinsic(
2186 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2187 MachineFunction &MF, unsigned Intrinsic) const {
2188 IntrinsicInfo Info;
2189 auto &DL = I.getDataLayout();
2190
2191 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2192 bool IsUnitStrided, bool UsePtrVal = false) {
2193 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2194 // We can't use ptrVal if the intrinsic can access memory before the
2195 // pointer. This means we can't use it for strided or indexed intrinsics.
2196 if (UsePtrVal)
2197 Info.ptrVal = I.getArgOperand(PtrOp);
2198 else
2199 Info.fallbackAddressSpace =
2200 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
2201 Type *MemTy;
2202 if (IsStore) {
2203 // Store value is the first operand.
2204 MemTy = I.getArgOperand(0)->getType();
2205 } else {
2206 // Use return type. If it's segment load, return type is a struct.
2207 MemTy = I.getType();
2208 if (MemTy->isStructTy())
2209 MemTy = MemTy->getStructElementType(0);
2210 }
2211 if (!IsUnitStrided)
2212 MemTy = MemTy->getScalarType();
2213
2214 Info.memVT = getValueType(DL, MemTy);
2215 if (MemTy->isTargetExtTy()) {
2216 // RISC-V vector tuple type's alignment type should be its element type.
2217 if (cast<TargetExtType>(MemTy)->getName() == "riscv.vector.tuple")
2218 MemTy = Type::getIntNTy(
2219 MemTy->getContext(),
2220 1 << cast<ConstantInt>(I.getArgOperand(I.arg_size() - 1))
2221 ->getZExtValue());
2222 Info.align = DL.getABITypeAlign(MemTy);
2223 } else {
2224 Info.align = Align(DL.getTypeStoreSize(MemTy->getScalarType()));
2225 }
2226 Info.size = MemoryLocation::UnknownSize;
2227 Info.flags |=
2228 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2229 Infos.push_back(Info);
2230 };
2231
2232 if (I.hasMetadata(LLVMContext::MD_nontemporal))
2233 Info.flags |= MachineMemOperand::MONonTemporal;
2234
2235 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2236 switch (Intrinsic) {
2237 default:
2238 return;
2239 case Intrinsic::riscv_masked_atomicrmw_xchg:
2240 case Intrinsic::riscv_masked_atomicrmw_add:
2241 case Intrinsic::riscv_masked_atomicrmw_sub:
2242 case Intrinsic::riscv_masked_atomicrmw_nand:
2243 case Intrinsic::riscv_masked_atomicrmw_max:
2244 case Intrinsic::riscv_masked_atomicrmw_min:
2245 case Intrinsic::riscv_masked_atomicrmw_umax:
2246 case Intrinsic::riscv_masked_atomicrmw_umin:
2247 case Intrinsic::riscv_masked_cmpxchg:
2248 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2249 // narrow atomic operation. These will be expanded to an LR/SC loop that
2250 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2251 // will be used to modify the appropriate part of the 4 byte data and
2252 // preserve the rest.
2253 Info.opc = ISD::INTRINSIC_W_CHAIN;
2254 Info.memVT = MVT::i32;
2255 Info.ptrVal = I.getArgOperand(0);
2256 Info.offset = 0;
2257 Info.align = Align(4);
2258 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2259 MachineMemOperand::MOVolatile;
2260 Infos.push_back(Info);
2261 return;
2262 case Intrinsic::riscv_seg2_load_mask:
2263 case Intrinsic::riscv_seg3_load_mask:
2264 case Intrinsic::riscv_seg4_load_mask:
2265 case Intrinsic::riscv_seg5_load_mask:
2266 case Intrinsic::riscv_seg6_load_mask:
2267 case Intrinsic::riscv_seg7_load_mask:
2268 case Intrinsic::riscv_seg8_load_mask:
2269 case Intrinsic::riscv_sseg2_load_mask:
2270 case Intrinsic::riscv_sseg3_load_mask:
2271 case Intrinsic::riscv_sseg4_load_mask:
2272 case Intrinsic::riscv_sseg5_load_mask:
2273 case Intrinsic::riscv_sseg6_load_mask:
2274 case Intrinsic::riscv_sseg7_load_mask:
2275 case Intrinsic::riscv_sseg8_load_mask:
2276 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2277 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2278 return;
2279 case Intrinsic::riscv_seg2_store_mask:
2280 case Intrinsic::riscv_seg3_store_mask:
2281 case Intrinsic::riscv_seg4_store_mask:
2282 case Intrinsic::riscv_seg5_store_mask:
2283 case Intrinsic::riscv_seg6_store_mask:
2284 case Intrinsic::riscv_seg7_store_mask:
2285 case Intrinsic::riscv_seg8_store_mask:
2286 // Operands are (vec, ..., vec, ptr, mask, vl)
2287 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2288 /*IsStore*/ true,
2289 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2290 return;
2291 case Intrinsic::riscv_sseg2_store_mask:
2292 case Intrinsic::riscv_sseg3_store_mask:
2293 case Intrinsic::riscv_sseg4_store_mask:
2294 case Intrinsic::riscv_sseg5_store_mask:
2295 case Intrinsic::riscv_sseg6_store_mask:
2296 case Intrinsic::riscv_sseg7_store_mask:
2297 case Intrinsic::riscv_sseg8_store_mask:
2298 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2299 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2300 /*IsStore*/ true,
2301 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2302 return;
2303 case Intrinsic::riscv_vlm:
2304 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2305 /*IsStore*/ false,
2306 /*IsUnitStrided*/ true,
2307 /*UsePtrVal*/ true);
2308 return;
2309 case Intrinsic::riscv_vle:
2310 case Intrinsic::riscv_vle_mask:
2311 case Intrinsic::riscv_vleff:
2312 case Intrinsic::riscv_vleff_mask:
2313 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2314 /*IsStore*/ false,
2315 /*IsUnitStrided*/ true,
2316 /*UsePtrVal*/ true);
2317 return;
2318 case Intrinsic::riscv_vsm:
2319 case Intrinsic::riscv_vse:
2320 case Intrinsic::riscv_vse_mask:
2321 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2322 /*IsStore*/ true,
2323 /*IsUnitStrided*/ true,
2324 /*UsePtrVal*/ true);
2325 return;
2326 case Intrinsic::riscv_vlse:
2327 case Intrinsic::riscv_vlse_mask:
2328 case Intrinsic::riscv_vloxei:
2329 case Intrinsic::riscv_vloxei_mask:
2330 case Intrinsic::riscv_vluxei:
2331 case Intrinsic::riscv_vluxei_mask:
2332 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2333 /*IsStore*/ false,
2334 /*IsUnitStrided*/ false);
2335 return;
2336 case Intrinsic::riscv_vsse:
2337 case Intrinsic::riscv_vsse_mask:
2338 case Intrinsic::riscv_vsoxei:
2339 case Intrinsic::riscv_vsoxei_mask:
2340 case Intrinsic::riscv_vsuxei:
2341 case Intrinsic::riscv_vsuxei_mask:
2342 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2343 /*IsStore*/ true,
2344 /*IsUnitStrided*/ false);
2345 return;
2346 case Intrinsic::riscv_vlseg2:
2347 case Intrinsic::riscv_vlseg3:
2348 case Intrinsic::riscv_vlseg4:
2349 case Intrinsic::riscv_vlseg5:
2350 case Intrinsic::riscv_vlseg6:
2351 case Intrinsic::riscv_vlseg7:
2352 case Intrinsic::riscv_vlseg8:
2353 case Intrinsic::riscv_vlseg2ff:
2354 case Intrinsic::riscv_vlseg3ff:
2355 case Intrinsic::riscv_vlseg4ff:
2356 case Intrinsic::riscv_vlseg5ff:
2357 case Intrinsic::riscv_vlseg6ff:
2358 case Intrinsic::riscv_vlseg7ff:
2359 case Intrinsic::riscv_vlseg8ff:
2360 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2361 /*IsStore*/ false,
2362 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2363 return;
2364 case Intrinsic::riscv_vlseg2_mask:
2365 case Intrinsic::riscv_vlseg3_mask:
2366 case Intrinsic::riscv_vlseg4_mask:
2367 case Intrinsic::riscv_vlseg5_mask:
2368 case Intrinsic::riscv_vlseg6_mask:
2369 case Intrinsic::riscv_vlseg7_mask:
2370 case Intrinsic::riscv_vlseg8_mask:
2371 case Intrinsic::riscv_vlseg2ff_mask:
2372 case Intrinsic::riscv_vlseg3ff_mask:
2373 case Intrinsic::riscv_vlseg4ff_mask:
2374 case Intrinsic::riscv_vlseg5ff_mask:
2375 case Intrinsic::riscv_vlseg6ff_mask:
2376 case Intrinsic::riscv_vlseg7ff_mask:
2377 case Intrinsic::riscv_vlseg8ff_mask:
2378 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2379 /*IsStore*/ false,
2380 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2381 return;
2382 case Intrinsic::riscv_vlsseg2:
2383 case Intrinsic::riscv_vlsseg3:
2384 case Intrinsic::riscv_vlsseg4:
2385 case Intrinsic::riscv_vlsseg5:
2386 case Intrinsic::riscv_vlsseg6:
2387 case Intrinsic::riscv_vlsseg7:
2388 case Intrinsic::riscv_vlsseg8:
2389 case Intrinsic::riscv_vloxseg2:
2390 case Intrinsic::riscv_vloxseg3:
2391 case Intrinsic::riscv_vloxseg4:
2392 case Intrinsic::riscv_vloxseg5:
2393 case Intrinsic::riscv_vloxseg6:
2394 case Intrinsic::riscv_vloxseg7:
2395 case Intrinsic::riscv_vloxseg8:
2396 case Intrinsic::riscv_vluxseg2:
2397 case Intrinsic::riscv_vluxseg3:
2398 case Intrinsic::riscv_vluxseg4:
2399 case Intrinsic::riscv_vluxseg5:
2400 case Intrinsic::riscv_vluxseg6:
2401 case Intrinsic::riscv_vluxseg7:
2402 case Intrinsic::riscv_vluxseg8:
2403 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2404 /*IsStore*/ false,
2405 /*IsUnitStrided*/ false);
2406 return;
2407 case Intrinsic::riscv_vlsseg2_mask:
2408 case Intrinsic::riscv_vlsseg3_mask:
2409 case Intrinsic::riscv_vlsseg4_mask:
2410 case Intrinsic::riscv_vlsseg5_mask:
2411 case Intrinsic::riscv_vlsseg6_mask:
2412 case Intrinsic::riscv_vlsseg7_mask:
2413 case Intrinsic::riscv_vlsseg8_mask:
2414 case Intrinsic::riscv_vloxseg2_mask:
2415 case Intrinsic::riscv_vloxseg3_mask:
2416 case Intrinsic::riscv_vloxseg4_mask:
2417 case Intrinsic::riscv_vloxseg5_mask:
2418 case Intrinsic::riscv_vloxseg6_mask:
2419 case Intrinsic::riscv_vloxseg7_mask:
2420 case Intrinsic::riscv_vloxseg8_mask:
2421 case Intrinsic::riscv_vluxseg2_mask:
2422 case Intrinsic::riscv_vluxseg3_mask:
2423 case Intrinsic::riscv_vluxseg4_mask:
2424 case Intrinsic::riscv_vluxseg5_mask:
2425 case Intrinsic::riscv_vluxseg6_mask:
2426 case Intrinsic::riscv_vluxseg7_mask:
2427 case Intrinsic::riscv_vluxseg8_mask:
2428 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2429 /*IsStore*/ false,
2430 /*IsUnitStrided*/ false);
2431 return;
2432 case Intrinsic::riscv_vsseg2:
2433 case Intrinsic::riscv_vsseg3:
2434 case Intrinsic::riscv_vsseg4:
2435 case Intrinsic::riscv_vsseg5:
2436 case Intrinsic::riscv_vsseg6:
2437 case Intrinsic::riscv_vsseg7:
2438 case Intrinsic::riscv_vsseg8:
2439 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2440 /*IsStore*/ true,
2441 /*IsUnitStrided*/ false);
2442 return;
2443 case Intrinsic::riscv_vsseg2_mask:
2444 case Intrinsic::riscv_vsseg3_mask:
2445 case Intrinsic::riscv_vsseg4_mask:
2446 case Intrinsic::riscv_vsseg5_mask:
2447 case Intrinsic::riscv_vsseg6_mask:
2448 case Intrinsic::riscv_vsseg7_mask:
2449 case Intrinsic::riscv_vsseg8_mask:
2450 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2451 /*IsStore*/ true,
2452 /*IsUnitStrided*/ false);
2453 return;
2454 case Intrinsic::riscv_vssseg2:
2455 case Intrinsic::riscv_vssseg3:
2456 case Intrinsic::riscv_vssseg4:
2457 case Intrinsic::riscv_vssseg5:
2458 case Intrinsic::riscv_vssseg6:
2459 case Intrinsic::riscv_vssseg7:
2460 case Intrinsic::riscv_vssseg8:
2461 case Intrinsic::riscv_vsoxseg2:
2462 case Intrinsic::riscv_vsoxseg3:
2463 case Intrinsic::riscv_vsoxseg4:
2464 case Intrinsic::riscv_vsoxseg5:
2465 case Intrinsic::riscv_vsoxseg6:
2466 case Intrinsic::riscv_vsoxseg7:
2467 case Intrinsic::riscv_vsoxseg8:
2468 case Intrinsic::riscv_vsuxseg2:
2469 case Intrinsic::riscv_vsuxseg3:
2470 case Intrinsic::riscv_vsuxseg4:
2471 case Intrinsic::riscv_vsuxseg5:
2472 case Intrinsic::riscv_vsuxseg6:
2473 case Intrinsic::riscv_vsuxseg7:
2474 case Intrinsic::riscv_vsuxseg8:
2475 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2476 /*IsStore*/ true,
2477 /*IsUnitStrided*/ false);
2478 return;
2479 case Intrinsic::riscv_vssseg2_mask:
2480 case Intrinsic::riscv_vssseg3_mask:
2481 case Intrinsic::riscv_vssseg4_mask:
2482 case Intrinsic::riscv_vssseg5_mask:
2483 case Intrinsic::riscv_vssseg6_mask:
2484 case Intrinsic::riscv_vssseg7_mask:
2485 case Intrinsic::riscv_vssseg8_mask:
2486 case Intrinsic::riscv_vsoxseg2_mask:
2487 case Intrinsic::riscv_vsoxseg3_mask:
2488 case Intrinsic::riscv_vsoxseg4_mask:
2489 case Intrinsic::riscv_vsoxseg5_mask:
2490 case Intrinsic::riscv_vsoxseg6_mask:
2491 case Intrinsic::riscv_vsoxseg7_mask:
2492 case Intrinsic::riscv_vsoxseg8_mask:
2493 case Intrinsic::riscv_vsuxseg2_mask:
2494 case Intrinsic::riscv_vsuxseg3_mask:
2495 case Intrinsic::riscv_vsuxseg4_mask:
2496 case Intrinsic::riscv_vsuxseg5_mask:
2497 case Intrinsic::riscv_vsuxseg6_mask:
2498 case Intrinsic::riscv_vsuxseg7_mask:
2499 case Intrinsic::riscv_vsuxseg8_mask:
2500 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2501 /*IsStore*/ true,
2502 /*IsUnitStrided*/ false);
2503 return;
2504 case Intrinsic::riscv_sf_vlte8:
2505 case Intrinsic::riscv_sf_vlte16:
2506 case Intrinsic::riscv_sf_vlte32:
2507 case Intrinsic::riscv_sf_vlte64:
2508 Info.opc = ISD::INTRINSIC_VOID;
2509 Info.ptrVal = I.getArgOperand(1);
2510 switch (Intrinsic) {
2511 case Intrinsic::riscv_sf_vlte8:
2512 Info.memVT = MVT::i8;
2513 Info.align = Align(1);
2514 break;
2515 case Intrinsic::riscv_sf_vlte16:
2516 Info.memVT = MVT::i16;
2517 Info.align = Align(2);
2518 break;
2519 case Intrinsic::riscv_sf_vlte32:
2520 Info.memVT = MVT::i32;
2521 Info.align = Align(4);
2522 break;
2523 case Intrinsic::riscv_sf_vlte64:
2524 Info.memVT = MVT::i64;
2525 Info.align = Align(8);
2526 break;
2527 }
2528 Info.size = MemoryLocation::UnknownSize;
2529 Info.flags |= MachineMemOperand::MOLoad;
2530 Infos.push_back(Info);
2531 return;
2532 case Intrinsic::riscv_sf_vste8:
2533 case Intrinsic::riscv_sf_vste16:
2534 case Intrinsic::riscv_sf_vste32:
2535 case Intrinsic::riscv_sf_vste64:
2536 Info.opc = ISD::INTRINSIC_VOID;
2537 Info.ptrVal = I.getArgOperand(1);
2538 switch (Intrinsic) {
2539 case Intrinsic::riscv_sf_vste8:
2540 Info.memVT = MVT::i8;
2541 Info.align = Align(1);
2542 break;
2543 case Intrinsic::riscv_sf_vste16:
2544 Info.memVT = MVT::i16;
2545 Info.align = Align(2);
2546 break;
2547 case Intrinsic::riscv_sf_vste32:
2548 Info.memVT = MVT::i32;
2549 Info.align = Align(4);
2550 break;
2551 case Intrinsic::riscv_sf_vste64:
2552 Info.memVT = MVT::i64;
2553 Info.align = Align(8);
2554 break;
2555 }
2556 Info.size = MemoryLocation::UnknownSize;
2557 Info.flags |= MachineMemOperand::MOStore;
2558 Infos.push_back(Info);
2559 return;
2560 }
2561}
2562
2563bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2564 const AddrMode &AM, Type *Ty,
2565 unsigned AS,
2566 Instruction *I) const {
2567 // No global is ever allowed as a base.
2568 if (AM.BaseGV)
2569 return false;
2570
2571 // None of our addressing modes allows a scalable offset
2572 if (AM.ScalableOffset)
2573 return false;
2574
2575 // RVV instructions only support register addressing.
2576 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
2577 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2578
2579 // The Xqcilo extension provides load/store instructions with a 26-bit signed
2580 // offset.
2581 if (Subtarget.hasVendorXqcilo()) {
2582 if (!isInt<26>(AM.BaseOffs))
2583 return false;
2584 } else if (!isInt<12>(AM.BaseOffs)) {
2585 // Otherwise require a 12-bit signed offset.
2586 return false;
2587 }
2588
2589 switch (AM.Scale) {
2590 case 0: // "r+i" or just "i", depending on HasBaseReg.
2591 break;
2592 case 1:
2593 if (!AM.HasBaseReg) // allow "r+i".
2594 break;
2595 return false; // disallow "r+r" or "r+r+i".
2596 default:
2597 return false;
2598 }
2599
2600 return true;
2601}
2602
2603bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2604 return isInt<12>(Imm);
2605}
2606
2607bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2608 // The Xqcilia extension provides add-immediate instructions with a 26-bit
2609 // signed immediate.
2610 if (Subtarget.hasVendorXqcilia())
2611 return isInt<26>(Imm);
2612 return isInt<12>(Imm);
2613}
2614
2615// On RV32, 64-bit integers are split into their high and low parts and held
2616// in two different registers, so the trunc is free since the low register can
2617// just be used.
2618// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2619// isTruncateFree?
2620bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2621 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2622 return false;
2623 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2624 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2625 return (SrcBits == 64 && DestBits == 32);
2626}
2627
2628bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2629 // We consider i64->i32 free on RV64 since we have good selection of W
2630 // instructions that make promoting operations back to i64 free in many cases.
2631 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2632 !DstVT.isInteger())
2633 return false;
2634 unsigned SrcBits = SrcVT.getSizeInBits();
2635 unsigned DestBits = DstVT.getSizeInBits();
2636 return (SrcBits == 64 && DestBits == 32);
2637}
2638
2639bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2640 EVT SrcVT = Val.getValueType();
2641 // free truncate from vnsrl and vnsra
2642 if (Subtarget.hasVInstructions() &&
2643 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2644 SrcVT.isVector() && VT2.isVector()) {
2645 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2646 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2647 if (SrcBits == DestBits * 2) {
2648 return true;
2649 }
2650 }
2651 return TargetLowering::isTruncateFree(Val, VT2);
2652}
2653
2654bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2655 // Zexts are free if they can be combined with a load.
2656 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2657 // poorly with type legalization of compares preferring sext.
2658 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2659 EVT MemVT = LD->getMemoryVT();
2660 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2661 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2662 LD->getExtensionType() == ISD::ZEXTLOAD))
2663 return true;
2664 }
2665
2666 return TargetLowering::isZExtFree(Val, VT2);
2667}
2668
2669bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2670 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2671}
2672
2673bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2674 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
2675}
2676
2677bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2678 return Subtarget.hasCTZLike();
2679}
2680
2681bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2682 return Subtarget.hasCLZLike();
2683}
2684
2685bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2686 const Instruction &AndI) const {
2687 // We expect to be able to match a bit extraction instruction if the Zbs
2688 // extension is supported and the mask is a power of two. However, we
2689 // conservatively return false if the mask would fit in an ANDI instruction,
2690 // on the basis that it's possible the sinking+duplication of the AND in
2691 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2692 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2693 if (!Subtarget.hasBEXTILike())
2694 return false;
2695 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
2696 if (!Mask)
2697 return false;
2698 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
2699}
2700
2701bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2702 EVT VT = Y.getValueType();
2703
2704 if (VT.isVector())
2705 return false;
2706
2707 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2708 (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
2709}
2710
2711bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2712 EVT VT = Y.getValueType();
2713
2714 if (!VT.isVector())
2715 return hasAndNotCompare(Y);
2716
2717 return Subtarget.hasStdExtZvkb();
2718}
2719
2720bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2721 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2722 if (Subtarget.hasStdExtZbs())
2723 return X.getValueType().isScalarInteger();
2724 auto *C = dyn_cast<ConstantSDNode>(Y);
2725 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2726 if (Subtarget.hasVendorXTHeadBs())
2727 return C != nullptr;
2728 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2729 return C && C->getAPIntValue().ule(10);
2730}
2731
2732bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2733 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2734 SDValue Y) const {
2735 if (SelectOpcode != ISD::VSELECT)
2736 return false;
2737
2738 // Only enable for rvv.
2739 if (!VT.isVector() || !Subtarget.hasVInstructions())
2740 return false;
2741
2742 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2743 return false;
2744
2745 return true;
2746}
2747
2748bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2749 Type *Ty) const {
2750 assert(Ty->isIntegerTy());
2751
2752 unsigned BitSize = Ty->getIntegerBitWidth();
2753 if (BitSize > Subtarget.getXLen())
2754 return false;
2755
2756 // Fast path, assume 32-bit immediates are cheap.
2757 int64_t Val = Imm.getSExtValue();
2758 if (isInt<32>(Val))
2759 return true;
2760
2761 // A constant pool entry may be more aligned than the load we're trying to
2762 // replace. If we don't support unaligned scalar mem, prefer the constant
2763 // pool.
2764 // TODO: Can the caller pass down the alignment?
2765 if (!Subtarget.enableUnalignedScalarMem())
2766 return true;
2767
2768 // Prefer to keep the load if it would require many instructions.
2769 // This uses the same threshold we use for constant pools but doesn't
2770 // check useConstantPoolForLargeInts.
2771 // TODO: Should we keep the load only when we're definitely going to emit a
2772 // constant pool?
2773
2774 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget);
2775 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2776}
2777
2778bool RISCVTargetLowering::
2779 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2780 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2781 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2782 SelectionDAG &DAG) const {
2783 // One interesting pattern that we'd want to form is 'bit extract':
2784 // ((1 >> Y) & 1) ==/!= 0
2785 // But we also need to be careful not to try to reverse that fold.
2786
2787 // Is this '((1 >> Y) & 1)'?
2788 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2789 return false; // Keep the 'bit extract' pattern.
2790
2791 // Will this be '((1 >> Y) & 1)' after the transform?
2792 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2793 return true; // Do form the 'bit extract' pattern.
2794
2795 // If 'X' is a constant, and we transform, then we will immediately
2796 // try to undo the fold, thus causing endless combine loop.
2797 // So only do the transform if X is not a constant. This matches the default
2798 // implementation of this function.
2799 return !XC;
2800}
2801
2802bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2803 unsigned Opc = VecOp.getOpcode();
2804
2805 // Assume target opcodes can't be scalarized.
2806 // TODO - do we have any exceptions?
2807 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
2808 return false;
2809
2810 // If the vector op is not supported, try to convert to scalar.
2811 EVT VecVT = VecOp.getValueType();
2812 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
2813 return true;
2814
2815 // If the vector op is supported, but the scalar op is not, the transform may
2816 // not be worthwhile.
2817 // Permit a vector binary operation can be converted to scalar binary
2818 // operation which is custom lowered with illegal type.
2819 EVT ScalarVT = VecVT.getScalarType();
2820 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
2821 isOperationCustom(Opc, ScalarVT);
2822}
2823
2824bool RISCVTargetLowering::isOffsetFoldingLegal(
2825 const GlobalAddressSDNode *GA) const {
2826 // In order to maximise the opportunity for common subexpression elimination,
2827 // keep a separate ADD node for the global address offset instead of folding
2828 // it in the global address node. Later peephole optimisations may choose to
2829 // fold it back in when profitable.
2830 return false;
2831}
2832
2833// Returns 0-31 if the fli instruction is available for the type and this is
2834// legal FP immediate for the type. Returns -1 otherwise.
2835int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2836 if (!Subtarget.hasStdExtZfa())
2837 return -1;
2838
2839 bool IsSupportedVT = false;
2840 if (VT == MVT::f16) {
2841 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2842 } else if (VT == MVT::f32) {
2843 IsSupportedVT = true;
2844 } else if (VT == MVT::f64) {
2845 assert(Subtarget.hasStdExtD() && "Expect D extension");
2846 IsSupportedVT = true;
2847 }
2848
2849 if (!IsSupportedVT)
2850 return -1;
2851
2852 return RISCVLoadFPImm::getLoadFPImm(Imm);
2853}
2854
2855bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2856 bool ForCodeSize) const {
2857 bool IsLegalVT = false;
2858 if (VT == MVT::f16)
2859 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2860 else if (VT == MVT::f32)
2861 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2862 else if (VT == MVT::f64)
2863 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2864 else if (VT == MVT::bf16)
2865 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2866
2867 if (!IsLegalVT)
2868 return false;
2869
2870 if (getLegalZfaFPImm(Imm, VT) >= 0)
2871 return true;
2872
2873 // Some constants can be produced by fli+fneg.
2874 if (Imm.isNegative() && getLegalZfaFPImm(-Imm, VT) >= 0)
2875 return true;
2876
2877 // Cannot create a 64 bit floating-point immediate value for rv32.
2878 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2879 // td can handle +0.0 or -0.0 already.
2880 // -0.0 can be created by fmv + fneg.
2881 return Imm.isZero();
2882 }
2883
2884 // Special case: fmv + fneg
2885 if (Imm.isNegZero())
2886 return true;
2887
2888 // Building an integer and then converting requires a fmv at the end of
2889 // the integer sequence. The fmv is not required for Zfinx.
2890 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2891 const int Cost =
2892 FmvCost + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
2893 Subtarget.getXLen(), Subtarget);
2894 return Cost <= FPImmCost;
2895}
2896
2897// TODO: This is very conservative.
2898TargetLowering::ExtractSubvectorCost
2899RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2900 unsigned Index) const {
2901 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
2902 (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) {
2903 if ((Index % ResVT.getVectorNumElements()) == 0)
2904 return ExtractSubvectorCost::Free;
2905 return ExtractSubvectorCost::Expensive;
2906 }
2907
2908 if (!Subtarget.hasVInstructions())
2909 return ExtractSubvectorCost::Expensive;
2910
2911 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
2912 return ExtractSubvectorCost::Expensive;
2913
2914 // Extracts from index 0 are just subreg extracts.
2915 if (Index == 0)
2916 return ExtractSubvectorCost::Free;
2917
2918 // Only support extracting a fixed from a fixed vector for now.
2919 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2920 return ExtractSubvectorCost::Expensive;
2921
2922 EVT EltVT = ResVT.getVectorElementType();
2923 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2924
2925 // The smallest type we can slide is i8.
2926 if (EltVT == MVT::i1)
2927 return ExtractSubvectorCost::Expensive;
2928
2929 unsigned ResElts = ResVT.getVectorNumElements();
2930 unsigned SrcElts = SrcVT.getVectorNumElements();
2931
2932 unsigned MinVLen = Subtarget.getRealMinVLen();
2933 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2934
2935 // If we're extracting only data from the first VLEN bits of the source
2936 // then we can always do this with an m1 vslidedown.vx. Restricting the
2937 // Index ensures we can use a vslidedown.vi.
2938 // TODO: We can generalize this when the exact VLEN is known.
2939 if (Index + ResElts <= MinVLMAX && Index < 31)
2940 return ExtractSubvectorCost::Free;
2941
2942 // Convervatively only handle extracting half of a vector.
2943 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2944 // the upper half of a vector until we have more test coverage.
2945 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2946 // a cheap extract. However, this case is important in practice for
2947 // shuffled extracts of longer vectors. How resolve?
2948 if ((ResElts * 2) == SrcElts && Index == ResElts)
2949 return ExtractSubvectorCost::Free;
2950 return ExtractSubvectorCost::Expensive;
2951}
2952
2953MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2954 CallingConv::ID CC,
2955 EVT VT) const {
2956 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2957 // We might still end up using a GPR but that will be decided based on ABI.
2958 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2959 !Subtarget.hasStdExtZfhminOrZhinxmin())
2960 return MVT::f32;
2961
2962 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2963}
2964
2965unsigned
2966RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2967 std::optional<MVT> RegisterVT) const {
2968 // Pair inline assembly operand
2969 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2970 *RegisterVT == MVT::Untyped)
2971 return 1;
2972
2973 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2974}
2975
2976unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2977 CallingConv::ID CC,
2978 EVT VT) const {
2979 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2980 // We might still end up using a GPR but that will be decided based on ABI.
2981 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2982 !Subtarget.hasStdExtZfhminOrZhinxmin())
2983 return 1;
2984
2985 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2986}
2987
2988// Changes the condition code and swaps operands if necessary, so the SetCC
2989// operation matches one of the comparisons supported directly by branches
2990// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2991// with 1/-1.
2992static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2993 ISD::CondCode &CC, SelectionDAG &DAG,
2994 const RISCVSubtarget &Subtarget) {
2995 // If this is a single bit test that can't be handled by ANDI, shift the
2996 // bit to be tested to the MSB and perform a signed compare with 0.
2997 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
2998 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2999 isa<ConstantSDNode>(LHS.getOperand(1)) &&
3000 // XAndesPerf supports branch on test bit.
3001 !Subtarget.hasVendorXAndesPerf()) {
3002 uint64_t Mask = LHS.getConstantOperandVal(1);
3003 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
3004 unsigned ShAmt = 0;
3005 if (isPowerOf2_64(Mask)) {
3006 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3007 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
3008 } else {
3009 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
3010 }
3011
3012 LHS = LHS.getOperand(0);
3013 if (ShAmt != 0)
3014 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
3015 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
3016 return;
3017 }
3018 }
3019
3020 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
3021 int64_t C = RHSC->getSExtValue();
3022 switch (CC) {
3023 default: break;
3024 case ISD::SETGT:
3025 // Convert X > -1 to X >= 0.
3026 if (C == -1) {
3027 RHS = DAG.getConstant(0, DL, RHS.getValueType());
3028 CC = ISD::SETGE;
3029 return;
3030 }
3031 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3032 C != INT64_MAX && isInt<5>(C + 1)) {
3033 // We have a conditional move instruction for SETGE but not SETGT.
3034 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3035 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
3036 CC = ISD::SETGE;
3037 return;
3038 }
3039 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1)) {
3040 // We have a branch immediate instruction for SETGE but not SETGT.
3041 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
3042 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
3043 CC = ISD::SETGE;
3044 return;
3045 }
3046 break;
3047 case ISD::SETLT:
3048 // Convert X < 1 to 0 >= X.
3049 if (C == 1) {
3050 RHS = LHS;
3051 LHS = DAG.getConstant(0, DL, RHS.getValueType());
3052 CC = ISD::SETGE;
3053 return;
3054 }
3055 break;
3056 case ISD::SETUGT:
3057 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3058 C != INT64_MAX && isUInt<5>(C + 1)) {
3059 // We have a conditional move instruction for SETUGE but not SETUGT.
3060 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3061 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
3062 CC = ISD::SETUGE;
3063 return;
3064 }
3065 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
3066 // We have a branch immediate instruction for SETUGE but not SETUGT.
3067 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
3068 // immediate.
3069 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
3070 CC = ISD::SETUGE;
3071 return;
3072 }
3073 break;
3074 }
3075 }
3076
3077 switch (CC) {
3078 default:
3079 break;
3080 case ISD::SETGT:
3081 case ISD::SETLE:
3082 case ISD::SETUGT:
3083 case ISD::SETULE:
3084 CC = ISD::getSetCCSwappedOperands(CC);
3085 std::swap(LHS, RHS);
3086 break;
3087 }
3088}
3089
3090RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
3091 if (VT.isRISCVVectorTuple()) {
3092 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
3093 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
3094 return RISCVVType::LMUL_F8;
3095 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
3096 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
3097 return RISCVVType::LMUL_F4;
3098 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
3099 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
3100 return RISCVVType::LMUL_F2;
3101 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
3102 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
3103 return RISCVVType::LMUL_1;
3104 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
3105 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
3106 return RISCVVType::LMUL_2;
3107 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
3108 return RISCVVType::LMUL_4;
3109 llvm_unreachable("Invalid vector tuple type LMUL.");
3110 }
3111
3112 assert(VT.isScalableVector() && "Expecting a scalable vector type");
3113 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
3114 if (VT.getVectorElementType() == MVT::i1)
3115 KnownSize *= 8;
3116
3117 switch (KnownSize) {
3118 default:
3119 llvm_unreachable("Invalid LMUL.");
3120 case 8:
3121 return RISCVVType::LMUL_F8;
3122 case 16:
3123 return RISCVVType::LMUL_F4;
3124 case 32:
3125 return RISCVVType::LMUL_F2;
3126 case 64:
3127 return RISCVVType::LMUL_1;
3128 case 128:
3129 return RISCVVType::LMUL_2;
3130 case 256:
3131 return RISCVVType::LMUL_4;
3132 case 512:
3133 return RISCVVType::LMUL_8;
3134 }
3135}
3136
3137unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
3138 switch (LMul) {
3139 default:
3140 llvm_unreachable("Invalid LMUL.");
3141 case RISCVVType::LMUL_F8:
3142 case RISCVVType::LMUL_F4:
3143 case RISCVVType::LMUL_F2:
3144 case RISCVVType::LMUL_1:
3145 return RISCV::VRRegClassID;
3146 case RISCVVType::LMUL_2:
3147 return RISCV::VRM2RegClassID;
3148 case RISCVVType::LMUL_4:
3149 return RISCV::VRM4RegClassID;
3150 case RISCVVType::LMUL_8:
3151 return RISCV::VRM8RegClassID;
3152 }
3153}
3154
3155unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
3156 RISCVVType::VLMUL LMUL = getLMUL(VT);
3157 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
3158 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
3159 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
3160 "Unexpected subreg numbering");
3161 return RISCV::sub_vrm1_0 + Index;
3162 }
3163 if (LMUL == RISCVVType::LMUL_2) {
3164 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
3165 "Unexpected subreg numbering");
3166 return RISCV::sub_vrm2_0 + Index;
3167 }
3168 if (LMUL == RISCVVType::LMUL_4) {
3169 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
3170 "Unexpected subreg numbering");
3171 return RISCV::sub_vrm4_0 + Index;
3172 }
3173 llvm_unreachable("Invalid vector type.");
3174}
3175
3176unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
3177 if (VT.isRISCVVectorTuple()) {
3178 unsigned NF = VT.getRISCVVectorTupleNumFields();
3179 unsigned RegsPerField =
3180 std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
3181 (NF * RISCV::RVVBitsPerBlock));
3182 switch (RegsPerField) {
3183 case 1:
3184 if (NF == 2)
3185 return RISCV::VRN2M1RegClassID;
3186 if (NF == 3)
3187 return RISCV::VRN3M1RegClassID;
3188 if (NF == 4)
3189 return RISCV::VRN4M1RegClassID;
3190 if (NF == 5)
3191 return RISCV::VRN5M1RegClassID;
3192 if (NF == 6)
3193 return RISCV::VRN6M1RegClassID;
3194 if (NF == 7)
3195 return RISCV::VRN7M1RegClassID;
3196 if (NF == 8)
3197 return RISCV::VRN8M1RegClassID;
3198 break;
3199 case 2:
3200 if (NF == 2)
3201 return RISCV::VRN2M2RegClassID;
3202 if (NF == 3)
3203 return RISCV::VRN3M2RegClassID;
3204 if (NF == 4)
3205 return RISCV::VRN4M2RegClassID;
3206 break;
3207 case 4:
3208 assert(NF == 2);
3209 return RISCV::VRN2M4RegClassID;
3210 default:
3211 break;
3212 }
3213 llvm_unreachable("Invalid vector tuple type RegClass.");
3214 }
3215
3216 if (VT.getVectorElementType() == MVT::i1)
3217 return RISCV::VRRegClassID;
3218 return getRegClassIDForLMUL(getLMUL(VT));
3219}
3220
3221// Attempt to decompose a subvector insert/extract between VecVT and
3222// SubVecVT via subregister indices. Returns the subregister index that
3223// can perform the subvector insert/extract with the given element index, as
3224// well as the index corresponding to any leftover subvectors that must be
3225// further inserted/extracted within the register class for SubVecVT.
3226std::pair<unsigned, unsigned>
3227RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3228 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3229 const RISCVRegisterInfo *TRI) {
3230 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3231 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3232 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3233 "Register classes not ordered");
3234 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
3235 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
3236
3237 // If VecVT is a vector tuple type, either it's the tuple type with same
3238 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3239 if (VecVT.isRISCVVectorTuple()) {
3240 if (VecRegClassID == SubRegClassID)
3241 return {RISCV::NoSubRegister, 0};
3242
3243 assert(SubVecVT.isScalableVector() &&
3244 "Only allow scalable vector subvector.");
3245 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3246 "Invalid vector tuple insert/extract for vector and subvector with "
3247 "different LMUL.");
3248 return {getSubregIndexByMVT(VecVT, InsertExtractIdx), 0};
3249 }
3250
3251 // Try to compose a subregister index that takes us from the incoming
3252 // LMUL>1 register class down to the outgoing one. At each step we half
3253 // the LMUL:
3254 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3255 // Note that this is not guaranteed to find a subregister index, such as
3256 // when we are extracting from one VR type to another.
3257 unsigned SubRegIdx = RISCV::NoSubRegister;
3258 for (const unsigned RCID :
3259 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3260 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3261 VecVT = VecVT.getHalfNumVectorElementsVT();
3262 bool IsHi =
3263 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3264 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
3265 getSubregIndexByMVT(VecVT, IsHi));
3266 if (IsHi)
3267 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3268 }
3269 return {SubRegIdx, InsertExtractIdx};
3270}
3271
3272// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3273// stores for those types.
3274bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3275 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
3276 (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
3277 return false;
3278
3279 return !Subtarget.useRVVForFixedLengthVectors() ||
3280 VT.isFixedLengthVectorOf(MVT::i1);
3281}
3282
3283bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3284 if (!ScalarTy.isSimple())
3285 return false;
3286 switch (ScalarTy.getSimpleVT().SimpleTy) {
3287 case MVT::iPTR:
3288 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3289 case MVT::i8:
3290 case MVT::i16:
3291 case MVT::i32:
3292 return Subtarget.hasVInstructions();
3293 case MVT::i64:
3294 return Subtarget.hasVInstructionsI64();
3295 case MVT::f16:
3296 return Subtarget.hasVInstructionsF16Minimal();
3297 case MVT::bf16:
3298 return Subtarget.hasVInstructionsBF16Minimal();
3299 case MVT::f32:
3300 return Subtarget.hasVInstructionsF32();
3301 case MVT::f64:
3302 return Subtarget.hasVInstructionsF64();
3303 default:
3304 return false;
3305 }
3306}
3307
3308
3309unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3310 return NumRepeatedDivisors;
3311}
3312
3313static SDValue getVLOperand(SDValue Op) {
3314 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3315 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3316 "Unexpected opcode");
3317 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3318 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
3319 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3320 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
3321 if (!II)
3322 return SDValue();
3323 return Op.getOperand(II->VLOperand + 1 + HasChain);
3324}
3325
3326static bool useRVVForFixedLengthVectorVT(MVT VT,
3327 const RISCVSubtarget &Subtarget) {
3328 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3329 if (!Subtarget.useRVVForFixedLengthVectors())
3330 return false;
3331
3332 // We only support a set of vector types with a consistent maximum fixed size
3333 // across all supported vector element types to avoid legalization issues.
3334 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3335 // fixed-length vector type we support is 1024 bytes.
3336 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3337 return false;
3338
3339 unsigned MinVLen = Subtarget.getRealMinVLen();
3340
3341 MVT EltVT = VT.getVectorElementType();
3342
3343 // Don't use RVV for vectors we cannot scalarize if required.
3344 switch (EltVT.SimpleTy) {
3345 // i1 is supported but has different rules.
3346 default:
3347 return false;
3348 case MVT::i1:
3349 // Masks can only use a single register.
3350 if (VT.getVectorNumElements() > MinVLen)
3351 return false;
3352 MinVLen /= 8;
3353 break;
3354 case MVT::i8:
3355 case MVT::i16:
3356 case MVT::i32:
3357 break;
3358 case MVT::i64:
3359 if (!Subtarget.hasVInstructionsI64())
3360 return false;
3361 break;
3362 case MVT::f16:
3363 if (!Subtarget.hasVInstructionsF16Minimal())
3364 return false;
3365 break;
3366 case MVT::bf16:
3367 if (!Subtarget.hasVInstructionsBF16Minimal())
3368 return false;
3369 break;
3370 case MVT::f32:
3371 if (!Subtarget.hasVInstructionsF32())
3372 return false;
3373 break;
3374 case MVT::f64:
3375 if (!Subtarget.hasVInstructionsF64())
3376 return false;
3377 break;
3378 }
3379
3380 // Reject elements larger than ELEN.
3381 if (EltVT.getSizeInBits() > Subtarget.getELen())
3382 return false;
3383
3384 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
3385 // Don't use RVV for types that don't fit.
3386 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3387 return false;
3388
3389 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3390 // the base fixed length RVV support in place.
3391 if (!VT.isPow2VectorType())
3392 return false;
3393
3394 return true;
3395}
3396
3397bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3398 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3399}
3400
3401// Return the largest legal scalable vector type that matches VT's element type.
3402static MVT getContainerForFixedLengthVector(MVT VT,
3403 const RISCVSubtarget &Subtarget) {
3404 // This may be called before legal types are setup.
3405 assert(((VT.isFixedLengthVector() &&
3406 Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
3407 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3408 "Expected legal fixed length vector!");
3409
3410 unsigned MinVLen = Subtarget.getRealMinVLen();
3411 unsigned MaxELen = Subtarget.getELen();
3412
3413 MVT EltVT = VT.getVectorElementType();
3414 switch (EltVT.SimpleTy) {
3415 default:
3416 llvm_unreachable("unexpected element type for RVV container");
3417 case MVT::i1:
3418 case MVT::i8:
3419 case MVT::i16:
3420 case MVT::i32:
3421 case MVT::i64:
3422 case MVT::bf16:
3423 case MVT::f16:
3424 case MVT::f32:
3425 case MVT::f64: {
3426 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3427 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3428 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3429 unsigned NumElts =
3430 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3431 NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
3432 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3433 return MVT::getScalableVectorVT(EltVT, NumElts);
3434 }
3435 }
3436}
3437
3438MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3439 return ::getContainerForFixedLengthVector(VT, getSubtarget());
3440}
3441
3442// Grow V to consume an entire RVV register.
3443static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3444 const RISCVSubtarget &Subtarget) {
3445 assert(VT.isScalableVector() &&
3446 "Expected to convert into a scalable vector!");
3447 assert(V.getValueType().isFixedLengthVector() &&
3448 "Expected a fixed length vector operand!");
3449 SDLoc DL(V);
3450 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), V, 0);
3451}
3452
3453// Shrink V so it's just big enough to maintain a VT's worth of data.
3454static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3455 const RISCVSubtarget &Subtarget) {
3456 assert(VT.isFixedLengthVector() &&
3457 "Expected to convert into a fixed length vector!");
3458 assert(V.getValueType().isScalableVector() &&
3459 "Expected a scalable vector operand!");
3460 SDLoc DL(V);
3461 return DAG.getExtractSubvector(DL, VT, V, 0);
3462}
3463
3464/// Return the type of the mask type suitable for masking the provided
3465/// vector type. This is simply an i1 element type vector of the same
3466/// (possibly scalable) length.
3467static MVT getMaskTypeFor(MVT VecVT) {
3468 assert(VecVT.isVector());
3469 ElementCount EC = VecVT.getVectorElementCount();
3470 return MVT::getVectorVT(MVT::i1, EC);
3471}
3472
3473/// Creates an all ones mask suitable for masking a vector of type VecTy with
3474/// vector length VL. .
3475static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3476 SelectionDAG &DAG) {
3477 MVT MaskVT = getMaskTypeFor(VecVT);
3478 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
3479}
3480
3481static std::pair<SDValue, SDValue>
3482getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3483 const RISCVSubtarget &Subtarget) {
3484 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3485 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
3486 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3487 return {Mask, VL};
3488}
3489
3490static std::pair<SDValue, SDValue>
3491getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3492 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3493 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3494 SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
3495 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
3496 return {Mask, VL};
3497}
3498
3499// Gets the two common "VL" operands: an all-ones mask and the vector length.
3500// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3501// the vector type that the fixed-length vector is contained in. Otherwise if
3502// VecVT is scalable, then ContainerVT should be the same as VecVT.
3503static std::pair<SDValue, SDValue>
3504getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3505 const RISCVSubtarget &Subtarget) {
3506 if (VecVT.isFixedLengthVector())
3507 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3508 Subtarget);
3509 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3510 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
3511}
3512
3513SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3514 SelectionDAG &DAG) const {
3515 assert(VecVT.isScalableVector() && "Expected scalable vector");
3516 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
3517 VecVT.getVectorElementCount());
3518}
3519
3520std::pair<unsigned, unsigned>
3521RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3522 const RISCVSubtarget &Subtarget) {
3523 assert(VecVT.isScalableVector() && "Expected scalable vector");
3524
3525 unsigned EltSize = VecVT.getScalarSizeInBits();
3526 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3527
3528 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3529 unsigned MaxVLMAX =
3530 RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
3531
3532 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3533 unsigned MinVLMAX =
3534 RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
3535
3536 return std::make_pair(MinVLMAX, MaxVLMAX);
3537}
3538
3539// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3540// of either is (currently) supported. This can get us into an infinite loop
3541// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3542// as a ..., etc.
3543// Until either (or both) of these can reliably lower any node, reporting that
3544// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3545// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3546// which is not desirable.
3547bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3548 EVT VT, unsigned DefinedValues) const {
3549 return false;
3550}
3551
3552InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3553 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3554 // implementation-defined.
3555 if (!VT.isVector())
3556 return InstructionCost::getInvalid();
3557 unsigned DLenFactor = Subtarget.getDLenFactor();
3558 unsigned Cost;
3559 if (VT.isScalableVector()) {
3560 unsigned LMul;
3561 bool Fractional;
3562 std::tie(LMul, Fractional) =
3563 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
3564 if (Fractional)
3565 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3566 else
3567 Cost = (LMul * DLenFactor);
3568 } else {
3569 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
3570 }
3571 return Cost;
3572}
3573
3574
3575/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3576/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3577/// be by default. VRGatherCostModel reflects available options. Note that
3578/// operand (index and possibly mask) are handled separately.
3579InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3580 auto LMULCost = getLMULCost(VT);
3581 bool Log2CostModel =
3582 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3583 if (Log2CostModel && LMULCost.isValid()) {
3584 unsigned Log = Log2_64(LMULCost.getValue());
3585 if (Log > 0)
3586 return LMULCost * Log;
3587 }
3588 return LMULCost * LMULCost;
3589}
3590
3591/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3592/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3593/// or may track the vrgather.vv cost. It is implementation-dependent.
3597
3598/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3599/// for the type VT. (This does not cover the vslide1up or vslide1down
3600/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3601/// or may track the vrgather.vv cost. It is implementation-dependent.
3605
3606/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3607/// for the type VT. (This does not cover the vslide1up or vslide1down
3608/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3609/// or may track the vrgather.vv cost. It is implementation-dependent.
3613
3614static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3615 const RISCVSubtarget &Subtarget) {
3616 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3617 // bf16 conversions are always promoted to f32.
3618 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3619 Op.getValueType() == MVT::bf16) {
3620 bool IsStrict = Op->isStrictFPOpcode();
3621
3622 SDLoc DL(Op);
3623 if (IsStrict) {
3624 SDValue Val = DAG.getNode(Op.getOpcode(), DL, {MVT::f32, MVT::Other},
3625 {Op.getOperand(0), Op.getOperand(1)});
3626 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
3627 {Op.getValueType(), MVT::Other},
3628 {Val.getValue(1), Val.getValue(0),
3629 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
3630 }
3631 return DAG.getNode(
3632 ISD::FP_ROUND, DL, Op.getValueType(),
3633 DAG.getNode(Op.getOpcode(), DL, MVT::f32, Op.getOperand(0)),
3634 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
3635 }
3636
3637 // Other operations are legal.
3638 return Op;
3639}
3640
3641static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3642 const RISCVSubtarget &Subtarget) {
3643 // RISC-V FP-to-int conversions saturate to the destination register size, but
3644 // don't produce 0 for nan. We can use a conversion instruction and fix the
3645 // nan case with a compare and a select.
3646 SDValue Src = Op.getOperand(0);
3647
3648 MVT DstVT = Op.getSimpleValueType();
3649 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3650
3651 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3652
3653 if (!DstVT.isVector()) {
3654 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3655 // the result.
3656 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3657 Src.getValueType() == MVT::bf16) {
3658 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
3659 }
3660
3661 unsigned Opc;
3662 if (SatVT == DstVT)
3663 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3664 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3665 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3666 else
3667 return SDValue();
3668 // FIXME: Support other SatVTs by clamping before or after the conversion.
3669
3670 SDLoc DL(Op);
3671 SDValue FpToInt = DAG.getNode(
3672 Opc, DL, DstVT, Src,
3673 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
3674
3675 if (Opc == RISCVISD::FCVT_WU_RV64)
3676 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
3677
3678 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
3679 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
3680 ISD::CondCode::SETUO);
3681 }
3682
3683 // Vectors.
3684
3685 MVT DstEltVT = DstVT.getVectorElementType();
3686 MVT SrcVT = Src.getSimpleValueType();
3687 MVT SrcEltVT = SrcVT.getVectorElementType();
3688 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3689 unsigned DstEltSize = DstEltVT.getSizeInBits();
3690
3691 // Only handle saturating to the destination type.
3692 if (SatVT != DstEltVT)
3693 return SDValue();
3694
3695 MVT DstContainerVT = DstVT;
3696 MVT SrcContainerVT = SrcVT;
3697 if (DstVT.isFixedLengthVector()) {
3698 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
3699 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
3700 assert(DstContainerVT.getVectorElementCount() ==
3701 SrcContainerVT.getVectorElementCount() &&
3702 "Expected same element count");
3703 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3704 }
3705
3706 SDLoc DL(Op);
3707
3708 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
3709
3710 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
3711 {Src, Src, DAG.getCondCode(ISD::SETNE),
3712 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
3713
3714 // Need to widen by more than 1 step, promote the FP type, then do a widening
3715 // convert.
3716 if (DstEltSize > (2 * SrcEltSize)) {
3717 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3718 MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
3719 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
3720 }
3721
3722 MVT CvtContainerVT = DstContainerVT;
3723 MVT CvtEltVT = DstEltVT;
3724 if (SrcEltSize > (2 * DstEltSize)) {
3725 CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3726 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3727 }
3728
3729 unsigned RVVOpc =
3730 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3731 SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
3732
3733 while (CvtContainerVT != DstContainerVT) {
3734 CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
3735 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3736 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3737 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3738 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3739 Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
3740 }
3741
3742 SDValue SplatZero = DAG.getNode(
3743 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
3744 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
3745 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
3746 Res, DAG.getUNDEF(DstContainerVT), VL);
3747
3748 if (DstVT.isFixedLengthVector())
3749 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
3750
3751 return Res;
3752}
3753
3754static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3755 const RISCVSubtarget &Subtarget) {
3756 bool IsStrict = Op->isStrictFPOpcode();
3757 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3758
3759 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3760 // bf16 conversions are always promoted to f32.
3761 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3762 SrcVal.getValueType() == MVT::bf16) {
3763 SDLoc DL(Op);
3764 if (IsStrict) {
3765 SDValue Ext =
3766 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
3767 {Op.getOperand(0), SrcVal});
3768 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
3769 {Ext.getValue(1), Ext.getValue(0)});
3770 }
3771 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
3772 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
3773 }
3774
3775 // Other operations are legal.
3776 return Op;
3777}
3778
3779static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3780 switch (Opc) {
3781 case ISD::FROUNDEVEN:
3782 case ISD::STRICT_FROUNDEVEN:
3783 return RISCVFPRndMode::RNE;
3784 case ISD::FTRUNC:
3785 case ISD::STRICT_FTRUNC:
3786 return RISCVFPRndMode::RTZ;
3787 case ISD::FFLOOR:
3788 case ISD::STRICT_FFLOOR:
3789 return RISCVFPRndMode::RDN;
3790 case ISD::FCEIL:
3791 case ISD::STRICT_FCEIL:
3792 return RISCVFPRndMode::RUP;
3793 case ISD::FROUND:
3794 case ISD::LROUND:
3795 case ISD::LLROUND:
3796 case ISD::STRICT_FROUND:
3797 case ISD::STRICT_LROUND:
3798 case ISD::STRICT_LLROUND:
3799 return RISCVFPRndMode::RMM;
3800 case ISD::FRINT:
3801 case ISD::LRINT:
3802 case ISD::LLRINT:
3803 case ISD::STRICT_FRINT:
3804 case ISD::STRICT_LRINT:
3805 case ISD::STRICT_LLRINT:
3806 return RISCVFPRndMode::DYN;
3807 }
3808
3809 return RISCVFPRndMode::Invalid;
3810}
3811
3812// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
3813// the integer domain and back. Taking care to avoid converting values that are
3814// nan or already correct.
3815static SDValue
3816lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3817 const RISCVSubtarget &Subtarget) {
3818 MVT VT = Op.getSimpleValueType();
3819 assert(VT.isVector() && "Unexpected type");
3820
3821 SDLoc DL(Op);
3822
3823 SDValue Src = Op.getOperand(0);
3824
3825 // Freeze the source since we are increasing the number of uses.
3826 Src = DAG.getFreeze(Src);
3827
3828 MVT ContainerVT = VT;
3829 if (VT.isFixedLengthVector()) {
3830 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3831 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3832 }
3833
3834 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3835
3836 // We do the conversion on the absolute value and fix the sign at the end.
3837 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3838
3839 // Determine the largest integer that can be represented exactly. This and
3840 // values larger than it don't have any fractional bits so don't need to
3841 // be converted.
3842 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3843 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3844 APFloat MaxVal = APFloat(FltSem);
3845 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3846 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3847 SDValue MaxValNode =
3848 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3849 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3850 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3851
3852 // If abs(Src) was larger than MaxVal or nan, keep it.
3853 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3854 Mask =
3855 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
3856 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3857 Mask, Mask, VL});
3858
3859 // Truncate to integer and convert back to FP.
3860 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3861 MVT XLenVT = Subtarget.getXLenVT();
3862 SDValue Truncated;
3863
3864 switch (Op.getOpcode()) {
3865 default:
3866 llvm_unreachable("Unexpected opcode");
3867 case ISD::FRINT:
3868 case ISD::FCEIL:
3869 case ISD::FFLOOR:
3870 case ISD::FROUND:
3871 case ISD::FROUNDEVEN: {
3872 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3873 assert(FRM != RISCVFPRndMode::Invalid);
3874 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3875 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3876 break;
3877 }
3878 case ISD::FTRUNC:
3879 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3880 Mask, VL);
3881 break;
3882 case ISD::FNEARBYINT:
3883 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3884 Mask, VL);
3885 break;
3886 }
3887
3888 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3889 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3890 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3891 Mask, VL);
3892
3893 // Restore the original sign so that -0.0 is preserved.
3894 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3895 Src, Src, Mask, VL);
3896
3897 if (!VT.isFixedLengthVector())
3898 return Truncated;
3899
3900 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3901}
3902
3903// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3904// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3905// qNan and converting the new source to integer and back to FP.
3906static SDValue
3907lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3908 const RISCVSubtarget &Subtarget) {
3909 SDLoc DL(Op);
3910 MVT VT = Op.getSimpleValueType();
3911 SDValue Chain = Op.getOperand(0);
3912 SDValue Src = Op.getOperand(1);
3913
3914 MVT ContainerVT = VT;
3915 if (VT.isFixedLengthVector()) {
3916 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3917 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3918 }
3919
3920 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3921
3922 // Freeze the source since we are increasing the number of uses.
3923 Src = DAG.getFreeze(Src);
3924
3925 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3926 MVT MaskVT = Mask.getSimpleValueType();
3927 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3928 DAG.getVTList(MaskVT, MVT::Other),
3929 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3930 DAG.getUNDEF(MaskVT), Mask, VL});
3931 Chain = Unorder.getValue(1);
3932 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3933 DAG.getVTList(ContainerVT, MVT::Other),
3934 {Chain, Src, Src, Src, Unorder, VL});
3935 Chain = Src.getValue(1);
3936
3937 // We do the conversion on the absolute value and fix the sign at the end.
3938 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3939
3940 // Determine the largest integer that can be represented exactly. This and
3941 // values larger than it don't have any fractional bits so don't need to
3942 // be converted.
3943 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3944 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3945 APFloat MaxVal = APFloat(FltSem);
3946 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3947 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3948 SDValue MaxValNode =
3949 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3950 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3951 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3952
3953 // If abs(Src) was larger than MaxVal or nan, keep it.
3954 Mask = DAG.getNode(
3955 RISCVISD::SETCC_VL, DL, MaskVT,
3956 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3957
3958 // Truncate to integer and convert back to FP.
3959 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3960 MVT XLenVT = Subtarget.getXLenVT();
3961 SDValue Truncated;
3962
3963 switch (Op.getOpcode()) {
3964 default:
3965 llvm_unreachable("Unexpected opcode");
3966 case ISD::STRICT_FCEIL:
3967 case ISD::STRICT_FFLOOR:
3968 case ISD::STRICT_FROUND:
3969 case ISD::STRICT_FROUNDEVEN: {
3970 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3971 assert(FRM != RISCVFPRndMode::Invalid);
3972 Truncated = DAG.getNode(
3973 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3974 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3975 break;
3976 }
3977 case ISD::STRICT_FTRUNC:
3978 Truncated =
3979 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3980 DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3981 break;
3982 case ISD::STRICT_FNEARBYINT:
3983 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3984 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3985 Mask, VL);
3986 break;
3987 }
3988 Chain = Truncated.getValue(1);
3989
3990 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3991 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3992 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3993 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3994 Truncated, Mask, VL);
3995 Chain = Truncated.getValue(1);
3996 }
3997
3998 // Restore the original sign so that -0.0 is preserved.
3999 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
4000 Src, Src, Mask, VL);
4001
4002 if (VT.isFixedLengthVector())
4003 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
4004 return DAG.getMergeValues({Truncated, Chain}, DL);
4005}
4006
4007static SDValue
4008lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
4009 const RISCVSubtarget &Subtarget) {
4010 MVT VT = Op.getSimpleValueType();
4011 if (VT.isVector())
4012 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
4013
4014 if (DAG.shouldOptForSize())
4015 return SDValue();
4016
4017 SDLoc DL(Op);
4018 SDValue Src = Op.getOperand(0);
4019
4020 // Create an integer the size of the mantissa with the MSB set. This and all
4021 // values larger than it don't have any fractional bits so don't need to be
4022 // converted.
4023 const fltSemantics &FltSem = VT.getFltSemantics();
4024 unsigned Precision = APFloat::semanticsPrecision(FltSem);
4025 APFloat MaxVal = APFloat(FltSem);
4026 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
4027 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
4028 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
4029
4030 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
4031 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
4032 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
4033}
4034
4035// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
4036static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
4037 const RISCVSubtarget &Subtarget) {
4038 SDLoc DL(Op);
4039 MVT DstVT = Op.getSimpleValueType();
4040 SDValue Src = Op.getOperand(0);
4041 MVT SrcVT = Src.getSimpleValueType();
4042 assert(SrcVT.isVector() && DstVT.isVector() &&
4043 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
4044 "Unexpected type");
4045
4046 MVT DstContainerVT = DstVT;
4047 MVT SrcContainerVT = SrcVT;
4048
4049 if (DstVT.isFixedLengthVector()) {
4050 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
4051 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
4052 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4053 }
4054
4055 auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
4056
4057 // [b]f16 -> f32
4058 MVT SrcElemType = SrcVT.getVectorElementType();
4059 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
4060 MVT F32VT = SrcContainerVT.changeVectorElementType(MVT::f32);
4061 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, F32VT, Src, Mask, VL);
4062 }
4063
4064 SDValue Res =
4065 DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
4066 DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
4067 Subtarget.getXLenVT()),
4068 VL);
4069
4070 if (!DstVT.isFixedLengthVector())
4071 return Res;
4072
4073 return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
4074}
4075
4076static SDValue
4077getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
4078 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
4079 SDValue Offset, SDValue Mask, SDValue VL,
4080 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4081 if (Passthru.isUndef())
4082 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4083 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
4084 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4085 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
4086}
4087
4088static SDValue
4089getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
4090 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
4091 SDValue VL,
4092 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4093 if (Passthru.isUndef())
4094 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4095 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
4096 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4097 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
4098}
4099
4100struct VIDSequence {
4101 int64_t StepNumerator;
4102 unsigned StepDenominator;
4103 int64_t Addend;
4104};
4105
4106static std::optional<APInt> getExactInteger(const APFloat &APF,
4107 uint32_t BitWidth) {
4108 // We will use a SINT_TO_FP to materialize this constant so we should use a
4109 // signed APSInt here.
4110 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
4111 // We use an arbitrary rounding mode here. If a floating-point is an exact
4112 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
4113 // the rounding mode changes the output value, then it is not an exact
4114 // integer.
4115 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
4116 bool IsExact;
4117 // If it is out of signed integer range, it will return an invalid operation.
4118 // If it is not an exact integer, IsExact is false.
4119 if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
4120 APFloatBase::opInvalidOp) ||
4121 !IsExact)
4122 return std::nullopt;
4123 return ValInt.extractBits(BitWidth, 0);
4124}
4125
4126// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
4127// to the (non-zero) step S and start value X. This can be then lowered as the
4128// RVV sequence (VID * S) + X, for example.
4129// The step S is represented as an integer numerator divided by a positive
4130// denominator. Note that the implementation currently only identifies
4131// sequences in which either the numerator is +/- 1 or the denominator is 1. It
4132// cannot detect 2/3, for example.
4133// Note that this method will also match potentially unappealing index
4134// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
4135// determine whether this is worth generating code for.
4136//
4137// EltSizeInBits is the size of the type that the sequence will be calculated
4138// in, i.e. SEW for build_vectors or XLEN for address calculations.
4139static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
4140 unsigned EltSizeInBits) {
4141 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
4142 if (!cast<BuildVectorSDNode>(Op)->isConstant())
4143 return std::nullopt;
4144 bool IsInteger = Op.getValueType().isInteger();
4145
4146 std::optional<unsigned> SeqStepDenom;
4147 std::optional<APInt> SeqStepNum;
4148 std::optional<APInt> SeqAddend;
4149 std::optional<std::pair<APInt, unsigned>> PrevElt;
4150 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
4151
4152 // First extract the ops into a list of constant integer values. This may not
4153 // be possible for floats if they're not all representable as integers.
4154 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
4155 const unsigned OpSize = Op.getScalarValueSizeInBits();
4156 for (auto [Idx, Elt] : enumerate(Op->op_values())) {
4157 if (Elt.isUndef()) {
4158 Elts[Idx] = std::nullopt;
4159 continue;
4160 }
4161 if (IsInteger) {
4162 Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
4163 } else {
4164 auto ExactInteger =
4165 getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
4166 if (!ExactInteger)
4167 return std::nullopt;
4168 Elts[Idx] = *ExactInteger;
4169 }
4170 }
4171
4172 for (auto [Idx, Elt] : enumerate(Elts)) {
4173 // Assume undef elements match the sequence; we just have to be careful
4174 // when interpolating across them.
4175 if (!Elt)
4176 continue;
4177
4178 if (PrevElt) {
4179 // Calculate the step since the last non-undef element, and ensure
4180 // it's consistent across the entire sequence.
4181 unsigned IdxDiff = Idx - PrevElt->second;
4182 APInt ValDiff = *Elt - PrevElt->first;
4183
4184 // A zero-value value difference means that we're somewhere in the middle
4185 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4186 // step change before evaluating the sequence.
4187 if (ValDiff == 0)
4188 continue;
4189
4190 int64_t Remainder = ValDiff.srem(IdxDiff);
4191 // Normalize the step if it's greater than 1.
4192 if (Remainder != ValDiff.getSExtValue()) {
4193 // The difference must cleanly divide the element span.
4194 if (Remainder != 0)
4195 return std::nullopt;
4196 ValDiff = ValDiff.sdiv(IdxDiff);
4197 IdxDiff = 1;
4198 }
4199
4200 if (!SeqStepNum)
4201 SeqStepNum = ValDiff;
4202 else if (ValDiff != SeqStepNum)
4203 return std::nullopt;
4204
4205 if (!SeqStepDenom)
4206 SeqStepDenom = IdxDiff;
4207 else if (IdxDiff != *SeqStepDenom)
4208 return std::nullopt;
4209 }
4210
4211 // Record this non-undef element for later.
4212 if (!PrevElt || PrevElt->first != *Elt)
4213 PrevElt = std::make_pair(*Elt, Idx);
4214 }
4215
4216 // We need to have logged a step for this to count as a legal index sequence.
4217 if (!SeqStepNum || !SeqStepDenom)
4218 return std::nullopt;
4219
4220 // Loop back through the sequence and validate elements we might have skipped
4221 // while waiting for a valid step. While doing this, log any sequence addend.
4222 for (auto [Idx, Elt] : enumerate(Elts)) {
4223 if (!Elt)
4224 continue;
4225 APInt ExpectedVal =
4226 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4227 *SeqStepNum)
4228 .sdiv(*SeqStepDenom);
4229
4230 APInt Addend = *Elt - ExpectedVal;
4231 if (!SeqAddend)
4232 SeqAddend = Addend;
4233 else if (Addend != SeqAddend)
4234 return std::nullopt;
4235 }
4236
4237 assert(SeqAddend && "Must have an addend if we have a step");
4238
4239 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
4240 SeqAddend->getSExtValue()};
4241}
4242
4243// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4244// and lower it as a VRGATHER_VX_VL from the source vector.
4245static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4246 SelectionDAG &DAG,
4247 const RISCVSubtarget &Subtarget) {
4248 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4249 return SDValue();
4250 SDValue Src = SplatVal.getOperand(0);
4251 // Don't perform this optimization for i1 vectors, or if the element types are
4252 // different
4253 // FIXME: Support i1 vectors, maybe by promoting to i8?
4254 MVT EltTy = VT.getVectorElementType();
4255 if (EltTy == MVT::i1 ||
4256 !DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
4257 return SDValue();
4258 MVT SrcVT = Src.getSimpleValueType();
4259 if (EltTy != SrcVT.getVectorElementType())
4260 return SDValue();
4261 SDValue Idx = SplatVal.getOperand(1);
4262 // The index must be a legal type.
4263 if (Idx.getValueType() != Subtarget.getXLenVT())
4264 return SDValue();
4265
4266 // Check that we know Idx lies within VT
4267 if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
4268 auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
4269 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4270 return SDValue();
4271 }
4272
4273 // Convert fixed length vectors to scalable
4274 MVT ContainerVT = VT;
4275 if (VT.isFixedLengthVector())
4276 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4277
4278 MVT SrcContainerVT = SrcVT;
4279 if (SrcVT.isFixedLengthVector()) {
4280 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
4281 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4282 }
4283
4284 // Put Vec in a VT sized vector
4285 if (SrcContainerVT.getVectorMinNumElements() <
4286 ContainerVT.getVectorMinNumElements())
4287 Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Src, 0);
4288 else
4289 Src = DAG.getExtractSubvector(DL, ContainerVT, Src, 0);
4290
4291 // We checked that Idx fits inside VT earlier
4292 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4293 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Src,
4294 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
4295 if (VT.isFixedLengthVector())
4296 Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
4297 return Gather;
4298}
4299
4300static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4301 const RISCVSubtarget &Subtarget) {
4302 MVT VT = Op.getSimpleValueType();
4303 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4304
4305 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4306
4307 SDLoc DL(Op);
4308 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4309
4310 if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
4311 int64_t StepNumerator = SimpleVID->StepNumerator;
4312 unsigned StepDenominator = SimpleVID->StepDenominator;
4313 int64_t Addend = SimpleVID->Addend;
4314
4315 assert(StepNumerator != 0 && "Invalid step");
4316 bool Negate = false;
4317 int64_t SplatStepVal = StepNumerator;
4318 unsigned StepOpcode = ISD::MUL;
4319 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4320 // anyway as the shift of 63 won't fit in uimm5.
4321 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4322 isPowerOf2_64(std::abs(StepNumerator))) {
4323 Negate = StepNumerator < 0;
4324 StepOpcode = ISD::SHL;
4325 SplatStepVal = Log2_64(std::abs(StepNumerator));
4326 }
4327
4328 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4329 // since it's the immediate value many RVV instructions accept. There is
4330 // no vmul.vi instruction so ensure multiply constant can fit in a
4331 // single addi instruction. For the addend, we allow up to 32 bits..
4332 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
4333 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
4334 isPowerOf2_32(StepDenominator) &&
4335 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
4336 MVT VIDVT =
4337 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4338 MVT VIDContainerVT = getContainerForFixedLengthVector(VIDVT, Subtarget);
4339 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
4340 // Convert right out of the scalable type so we can use standard ISD
4341 // nodes for the rest of the computation. If we used scalable types with
4342 // these, we'd lose the fixed-length vector info and generate worse
4343 // vsetvli code.
4344 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
4345 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4346 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4347 SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
4348 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
4349 }
4350 if (StepDenominator != 1) {
4351 SDValue SplatStep =
4352 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
4353 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
4354 }
4355 if (Addend != 0 || Negate) {
4356 SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
4357 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
4358 VID);
4359 }
4360 if (VT.isFloatingPoint()) {
4361 // TODO: Use vfwcvt to reduce register pressure.
4362 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
4363 }
4364 return VID;
4365 }
4366 }
4367
4368 return SDValue();
4369}
4370
4371/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4372/// which constitute a large proportion of the elements. In such cases we can
4373/// splat a vector with the dominant element and make up the shortfall with
4374/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4375/// Note that this includes vectors of 2 elements by association. The
4376/// upper-most element is the "dominant" one, allowing us to use a splat to
4377/// "insert" the upper element, and an insert of the lower element at position
4378/// 0, which improves codegen.
4379static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4380 const RISCVSubtarget &Subtarget) {
4381 MVT VT = Op.getSimpleValueType();
4382 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4383
4384 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4385
4386 SDLoc DL(Op);
4387 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4388
4389 MVT XLenVT = Subtarget.getXLenVT();
4390 unsigned NumElts = Op.getNumOperands();
4391
4392 SDValue DominantValue;
4393 unsigned MostCommonCount = 0;
4394 DenseMap<SDValue, unsigned> ValueCounts;
4395 unsigned NumUndefElts =
4396 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4397
4398 // Track the number of scalar loads we know we'd be inserting, estimated as
4399 // any non-zero floating-point constant. Other kinds of element are either
4400 // already in registers or are materialized on demand. The threshold at which
4401 // a vector load is more desirable than several scalar materializion and
4402 // vector-insertion instructions is not known.
4403 unsigned NumScalarLoads = 0;
4404
4405 for (SDValue V : Op->op_values()) {
4406 if (V.isUndef())
4407 continue;
4408
4409 unsigned &Count = ValueCounts[V];
4410 if (0 == Count)
4411 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
4412 NumScalarLoads += !CFP->isPosZero();
4413
4414 // Is this value dominant? In case of a tie, prefer the highest element as
4415 // it's cheaper to insert near the beginning of a vector than it is at the
4416 // end.
4417 if (++Count >= MostCommonCount) {
4418 DominantValue = V;
4419 MostCommonCount = Count;
4420 }
4421 }
4422
4423 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4424 unsigned NumDefElts = NumElts - NumUndefElts;
4425 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4426
4427 // Don't perform this optimization when optimizing for size, since
4428 // materializing elements and inserting them tends to cause code bloat.
4429 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4430 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
4431 ((MostCommonCount > DominantValueCountThreshold) ||
4432 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
4433 // Start by splatting the most common element.
4434 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
4435
4436 DenseSet<SDValue> Processed{DominantValue};
4437
4438 // We can handle an insert into the last element (of a splat) via
4439 // v(f)slide1down. This is slightly better than the vslideup insert
4440 // lowering as it avoids the need for a vector group temporary. It
4441 // is also better than using vmerge.vx as it avoids the need to
4442 // materialize the mask in a vector register.
4443 if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
4444 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4445 LastOp != DominantValue) {
4446 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4447 auto OpCode =
4448 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4449 if (!VT.isFloatingPoint())
4450 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
4451 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4452 LastOp, Mask, VL);
4453 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
4454 Processed.insert(LastOp);
4455 }
4456
4457 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
4458 for (const auto &OpIdx : enumerate(Op->ops())) {
4459 const SDValue &V = OpIdx.value();
4460 if (V.isUndef() || !Processed.insert(V).second)
4461 continue;
4462 if (ValueCounts[V] == 1) {
4463 Vec = DAG.getInsertVectorElt(DL, Vec, V, OpIdx.index());
4464 } else {
4465 // Blend in all instances of this value using a VSELECT, using a
4466 // mask where each bit signals whether that element is the one
4467 // we're after.
4468 SmallVector<SDValue> Ops;
4469 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
4470 return DAG.getConstant(V == V1, DL, XLenVT);
4471 });
4472 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
4473 DAG.getBuildVector(SelMaskTy, DL, Ops),
4474 DAG.getSplatBuildVector(VT, DL, V), Vec);
4475 }
4476 }
4477
4478 return Vec;
4479 }
4480
4481 return SDValue();
4482}
4483
4484static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4485 const RISCVSubtarget &Subtarget) {
4486 MVT VT = Op.getSimpleValueType();
4487 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4488
4489 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4490
4491 SDLoc DL(Op);
4492 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4493
4494 MVT XLenVT = Subtarget.getXLenVT();
4495 unsigned NumElts = Op.getNumOperands();
4496
4497 if (VT.getVectorElementType() == MVT::i1) {
4498 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
4499 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
4500 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
4501 }
4502
4503 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
4504 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4505 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
4506 }
4507
4508 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4509 // scalar integer chunks whose bit-width depends on the number of mask
4510 // bits and XLEN.
4511 // First, determine the most appropriate scalar integer type to use. This
4512 // is at most XLenVT, but may be shrunk to a smaller vector element type
4513 // according to the size of the final vector - use i8 chunks rather than
4514 // XLenVT if we're producing a v8i1. This results in more consistent
4515 // codegen across RV32 and RV64.
4516 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
4517 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
4518 // If we have to use more than one INSERT_VECTOR_ELT then this
4519 // optimization is likely to increase code size; avoid performing it in
4520 // such a case. We can use a load from a constant pool in this case.
4521 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4522 return SDValue();
4523 // Now we can create our integer vector type. Note that it may be larger
4524 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4525 unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
4526 MVT IntegerViaVecVT =
4527 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
4528 IntegerViaVecElts);
4529
4530 uint64_t Bits = 0;
4531 unsigned BitPos = 0, IntegerEltIdx = 0;
4532 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4533
4534 for (unsigned I = 0; I < NumElts;) {
4535 SDValue V = Op.getOperand(I);
4536 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4537 Bits |= ((uint64_t)BitValue << BitPos);
4538 ++BitPos;
4539 ++I;
4540
4541 // Once we accumulate enough bits to fill our scalar type or process the
4542 // last element, insert into our vector and clear our accumulated data.
4543 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4544 if (NumViaIntegerBits <= 32)
4545 Bits = SignExtend64<32>(Bits);
4546 SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
4547 Elts[IntegerEltIdx] = Elt;
4548 Bits = 0;
4549 BitPos = 0;
4550 IntegerEltIdx++;
4551 }
4552 }
4553
4554 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
4555
4556 if (NumElts < NumViaIntegerBits) {
4557 // If we're producing a smaller vector than our minimum legal integer
4558 // type, bitcast to the equivalent (known-legal) mask type, and extract
4559 // our final mask.
4560 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4561 Vec = DAG.getBitcast(MVT::v8i1, Vec);
4562 Vec = DAG.getExtractSubvector(DL, VT, Vec, 0);
4563 } else {
4564 // Else we must have produced an integer type with the same size as the
4565 // mask type; bitcast for the final result.
4566 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4567 Vec = DAG.getBitcast(VT, Vec);
4568 }
4569
4570 return Vec;
4571 }
4572
4573 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4574 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4575 : RISCVISD::VMV_V_X_VL;
4576 if (!VT.isFloatingPoint())
4577 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4578 Splat =
4579 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4580 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4581 }
4582
4583 // Try and match index sequences, which we can lower to the vid instruction
4584 // with optional modifications. An all-undef vector is matched by
4585 // getSplatValue, above.
4586 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4587 return Res;
4588
4589 // For very small build_vectors, use a single scalar insert of a constant.
4590 // TODO: Base this on constant rematerialization cost, not size.
4591 const unsigned EltBitSize = VT.getScalarSizeInBits();
4592 if (VT.getSizeInBits() <= 32 &&
4593 ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
4594 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
4595 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4596 "Unexpected sequence type");
4597 // If we can use the original VL with the modified element type, this
4598 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4599 // be moved into InsertVSETVLI?
4600 unsigned ViaVecLen =
4601 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4602 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4603
4604 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4605 uint64_t SplatValue = 0;
4606 // Construct the amalgamated value at this larger vector type.
4607 for (const auto &OpIdx : enumerate(Op->op_values())) {
4608 const auto &SeqV = OpIdx.value();
4609 if (!SeqV.isUndef())
4610 SplatValue |=
4611 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4612 }
4613
4614 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4615 // achieve better constant materializion.
4616 // On RV32, we need to sign-extend to use getSignedConstant.
4617 if (ViaIntVT == MVT::i32)
4618 SplatValue = SignExtend64<32>(SplatValue);
4619
4620 SDValue Vec = DAG.getInsertVectorElt(
4621 DL, DAG.getUNDEF(ViaVecVT),
4622 DAG.getSignedConstant(SplatValue, DL, XLenVT), 0);
4623 if (ViaVecLen != 1)
4624 Vec = DAG.getExtractSubvector(DL, MVT::getVectorVT(ViaIntVT, 1), Vec, 0);
4625 return DAG.getBitcast(VT, Vec);
4626 }
4627
4628
4629 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4630 // when re-interpreted as a vector with a larger element type. For example,
4631 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4632 // could be instead splat as
4633 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4634 // TODO: This optimization could also work on non-constant splats, but it
4635 // would require bit-manipulation instructions to construct the splat value.
4636 SmallVector<SDValue> Sequence;
4637 const auto *BV = cast<BuildVectorSDNode>(Op);
4638 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4639 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
4640 BV->getRepeatedSequence(Sequence) &&
4641 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4642 unsigned SeqLen = Sequence.size();
4643 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
4644 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4645 ViaIntVT == MVT::i64) &&
4646 "Unexpected sequence type");
4647
4648 // If we can use the original VL with the modified element type, this
4649 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4650 // be moved into InsertVSETVLI?
4651 const unsigned RequiredVL = NumElts / SeqLen;
4652 const unsigned ViaVecLen =
4653 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4654 NumElts : RequiredVL;
4655 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4656
4657 unsigned EltIdx = 0;
4658 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4659 uint64_t SplatValue = 0;
4660 // Construct the amalgamated value which can be splatted as this larger
4661 // vector type.
4662 for (const auto &SeqV : Sequence) {
4663 if (!SeqV.isUndef())
4664 SplatValue |=
4665 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4666 EltIdx++;
4667 }
4668
4669 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4670 // achieve better constant materializion.
4671 // On RV32, we need to sign-extend to use getSignedConstant.
4672 if (ViaIntVT == MVT::i32)
4673 SplatValue = SignExtend64<32>(SplatValue);
4674
4675 // Since we can't introduce illegal i64 types at this stage, we can only
4676 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4677 // way we can use RVV instructions to splat.
4678 assert((ViaIntVT.bitsLE(XLenVT) ||
4679 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4680 "Unexpected bitcast sequence");
4681 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
4682 SDValue ViaVL =
4683 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
4684 MVT ViaContainerVT =
4685 getContainerForFixedLengthVector(ViaVecVT, Subtarget);
4686 SDValue Splat =
4687 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
4688 DAG.getUNDEF(ViaContainerVT),
4689 DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
4690 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
4691 if (ViaVecLen != RequiredVL)
4692 Splat = DAG.getExtractSubvector(
4693 DL, MVT::getVectorVT(ViaIntVT, RequiredVL), Splat, 0);
4694 return DAG.getBitcast(VT, Splat);
4695 }
4696 }
4697
4698 // If the number of signbits allows, see if we can lower as a <N x i8>.
4699 // Our main goal here is to reduce LMUL (and thus work) required to
4700 // build the constant, but we will also narrow if the resulting
4701 // narrow vector is known to materialize cheaply.
4702 // TODO: We really should be costing the smaller vector. There are
4703 // profitable cases this misses.
4704 if (EltBitSize > 8 && VT.isInteger() &&
4705 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4706 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4707 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
4708 DL, Op->ops());
4709 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
4710 Source, DAG, Subtarget);
4711 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
4712 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4713 }
4714
4715 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4716 return Res;
4717
4718 // For constant vectors, use generic constant pool lowering. Otherwise,
4719 // we'd have to materialize constants in GPRs just to move them into the
4720 // vector.
4721 return SDValue();
4722}
4723
4724static unsigned getPACKOpcode(unsigned DestBW,
4725 const RISCVSubtarget &Subtarget) {
4726 switch (DestBW) {
4727 default:
4728 llvm_unreachable("Unsupported pack size");
4729 case 16:
4730 return RISCV::PACKH;
4731 case 32:
4732 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4733 case 64:
4734 assert(Subtarget.is64Bit());
4735 return RISCV::PACK;
4736 }
4737}
4738
4739/// Double the element size of the build vector to reduce the number
4740/// of vslide1down in the build vector chain. In the worst case, this
4741/// trades three scalar operations for 1 vector operation. Scalar
4742/// operations are generally lower latency, and for out-of-order cores
4743/// we also benefit from additional parallelism.
4744static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4745 const RISCVSubtarget &Subtarget) {
4746 SDLoc DL(Op);
4747 MVT VT = Op.getSimpleValueType();
4748 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4749 MVT ElemVT = VT.getVectorElementType();
4750 if (!ElemVT.isInteger())
4751 return SDValue();
4752
4753 // TODO: Relax these architectural restrictions, possibly with costing
4754 // of the actual instructions required.
4755 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4756 return SDValue();
4757
4758 unsigned NumElts = VT.getVectorNumElements();
4759 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4760 if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
4761 NumElts % 2 != 0)
4762 return SDValue();
4763
4764 // Produce [B,A] packed into a type twice as wide. Note that all
4765 // scalars are XLenVT, possibly masked (see below).
4766 MVT XLenVT = Subtarget.getXLenVT();
4767 SDValue Mask = DAG.getConstant(
4768 APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
4769 auto pack = [&](SDValue A, SDValue B) {
4770 // Bias the scheduling of the inserted operations to near the
4771 // definition of the element - this tends to reduce register
4772 // pressure overall.
4773 SDLoc ElemDL(B);
4774 if (Subtarget.hasStdExtZbkb())
4775 // Note that we're relying on the high bits of the result being
4776 // don't care. For PACKW, the result is *sign* extended.
4777 return SDValue(
4778 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
4779 ElemDL, XLenVT, A, B),
4780 0);
4781
4782 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
4783 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
4784 SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
4785 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
4786 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
4787 SDNodeFlags::Disjoint);
4788 };
4789
4790 SmallVector<SDValue> NewOperands;
4791 NewOperands.reserve(NumElts / 2);
4792 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4793 NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
4794 assert(NumElts == NewOperands.size() * 2);
4795 MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
4796 MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
4797 return DAG.getNode(ISD::BITCAST, DL, VT,
4798 DAG.getBuildVector(WideVecVT, DL, NewOperands));
4799}
4800
4801static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4802 const RISCVSubtarget &Subtarget) {
4803 MVT VT = Op.getSimpleValueType();
4804 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4805
4806 MVT EltVT = VT.getVectorElementType();
4807 MVT XLenVT = Subtarget.getXLenVT();
4808
4809 SDLoc DL(Op);
4810
4811 if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
4812 if (VT == MVT::v2i16) {
4813 SDValue Lo = DAG.getBitcast(
4814 MVT::v2i16,
4815 DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32));
4816 SDValue Hi = DAG.getBitcast(
4817 MVT::v2i16,
4818 DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32));
4819 return DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16, Lo, Hi);
4820 }
4821
4822 if (VT == MVT::v4i8) {
4823 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
4824 // d))
4825 SDValue Val0 =
4826 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(0));
4827 SDValue Val1 =
4828 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(1));
4829 SDValue Val2 =
4830 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(2));
4831 SDValue Val3 =
4832 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(3));
4833 SDValue Concat1 =
4834 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val0, Val2);
4835 SDValue Concat2 =
4836 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val1, Val3);
4837 SDValue PPairE =
4838 DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v8i8, Concat1, Concat2);
4839
4840 SDValue Lo = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 0);
4841 SDValue Hi = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 4);
4842
4843 return DAG.getBitcast(MVT::v4i8,
4844 DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16,
4845 DAG.getBitcast(MVT::v2i16, Lo),
4846 DAG.getBitcast(MVT::v2i16, Hi)));
4847 }
4848
4849 llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
4850 }
4851
4852 // Proper support for f16 requires Zvfh. bf16 always requires special
4853 // handling. We need to cast the scalar to integer and create an integer
4854 // build_vector.
4855 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
4856 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4857 MVT IVT = VT.changeVectorElementType(MVT::i16);
4858 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4859 for (const auto &[I, U] : enumerate(Op->ops())) {
4860 SDValue Elem = U.get();
4861 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4862 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4863 // Called by LegalizeDAG, we need to use XLenVT operations since we
4864 // can't create illegal types.
4865 if (auto *C = dyn_cast<ConstantFPSDNode>(Elem)) {
4866 // Manually constant fold so the integer build_vector can be lowered
4867 // better. Waiting for DAGCombine will be too late.
4868 APInt V =
4869 C->getValueAPF().bitcastToAPInt().sext(XLenVT.getSizeInBits());
4870 NewOps[I] = DAG.getConstant(V, DL, XLenVT);
4871 } else {
4872 NewOps[I] = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Elem);
4873 }
4874 } else {
4875 // Called by scalar type legalizer, we can use i16.
4876 NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
4877 }
4878 }
4879 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, IVT, NewOps);
4880 return DAG.getBitcast(VT, Res);
4881 }
4882
4883 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
4884 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
4885 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4886
4887 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4888
4889 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4890
4891 if (VT.getVectorElementType() == MVT::i1) {
4892 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4893 // vector type, we have a legal equivalently-sized i8 type, so we can use
4894 // that.
4895 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
4896 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
4897
4898 SDValue WideVec;
4899 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4900 // For a splat, perform a scalar truncate before creating the wider
4901 // vector.
4902 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4903 DAG.getConstant(1, DL, Splat.getValueType()));
4904 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
4905 } else {
4906 SmallVector<SDValue, 8> Ops(Op->op_values());
4907 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
4908 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
4909 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4910 }
4911
4912 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4913 }
4914
4915 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4916 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4917 return Gather;
4918
4919 if (!VT.isFloatingPoint())
4920 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4921
4922 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4923 // pressure at high LMUL.
4924 bool IsScalar = all_of(Op->ops().drop_front(),
4925 [](const SDUse &U) { return U.get().isUndef(); });
4926 unsigned Opc =
4927 VT.isFloatingPoint()
4928 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4929 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4930 Splat =
4931 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4932 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4933 }
4934
4935 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4936 return Res;
4937
4938 // If we're compiling for an exact VLEN value, we can split our work per
4939 // register in the register group.
4940 if (const auto VLen = Subtarget.getRealVLen();
4941 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4942 MVT ElemVT = VT.getVectorElementType();
4943 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4944 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4945 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
4946 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
4947 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4948
4949 // The following semantically builds up a fixed length concat_vector
4950 // of the component build_vectors. We eagerly lower to scalable and
4951 // insert_subvector here to avoid DAG combining it back to a large
4952 // build_vector.
4953 SmallVector<SDValue> BuildVectorOps(Op->ops());
4954 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4955 SDValue Vec = DAG.getUNDEF(ContainerVT);
4956 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4957 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
4958 SDValue SubBV =
4959 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4960 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
4961 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4962 Vec = DAG.getInsertSubvector(DL, Vec, SubBV, InsertIdx);
4963 }
4964 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4965 }
4966
4967 // If we're about to resort to vslide1down (or stack usage), pack our
4968 // elements into the widest scalar type we can. This will force a VL/VTYPE
4969 // toggle, but reduces the critical path, the number of vslide1down ops
4970 // required, and possibly enables scalar folds of the values.
4971 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4972 return Res;
4973
4974 // For m1 vectors, if we have non-undef values in both halves of our vector,
4975 // split the vector into low and high halves, build them separately, then
4976 // use a vselect to combine them. For long vectors, this cuts the critical
4977 // path of the vslide1down sequence in half, and gives us an opportunity
4978 // to special case each half independently. Note that we don't change the
4979 // length of the sub-vectors here, so if both fallback to the generic
4980 // vslide1down path, we should be able to fold the vselect into the final
4981 // vslidedown (for the undef tail) for the first half w/ masking.
4982 unsigned NumElts = VT.getVectorNumElements();
4983 unsigned NumUndefElts =
4984 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4985 unsigned NumDefElts = NumElts - NumUndefElts;
4986 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4987 ContainerVT.bitsLE(RISCVTargetLowering::getM1VT(ContainerVT))) {
4988 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4989 SmallVector<SDValue> MaskVals;
4990 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4991 SubVecAOps.reserve(NumElts);
4992 SubVecBOps.reserve(NumElts);
4993 for (const auto &[Idx, U] : enumerate(Op->ops())) {
4994 SDValue Elem = U.get();
4995 if (Idx < NumElts / 2) {
4996 SubVecAOps.push_back(Elem);
4997 SubVecBOps.push_back(UndefElem);
4998 } else {
4999 SubVecAOps.push_back(UndefElem);
5000 SubVecBOps.push_back(Elem);
5001 }
5002 bool SelectMaskVal = (Idx < NumElts / 2);
5003 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5004 }
5005 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
5006 MaskVals.size() == NumElts);
5007
5008 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
5009 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
5010 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
5011 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5012 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
5013 }
5014
5015 // Cap the cost at a value linear to the number of elements in the vector.
5016 // The default lowering is to use the stack. The vector store + scalar loads
5017 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
5018 // being (at least) linear in LMUL. As a result, using the vslidedown
5019 // lowering for every element ends up being VL*LMUL..
5020 // TODO: Should we be directly costing the stack alternative? Doing so might
5021 // give us a more accurate upper bound.
5022 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
5023
5024 // TODO: unify with TTI getSlideCost.
5025 InstructionCost PerSlideCost = 1;
5026 switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
5027 default: break;
5028 case RISCVVType::LMUL_2:
5029 PerSlideCost = 2;
5030 break;
5031 case RISCVVType::LMUL_4:
5032 PerSlideCost = 4;
5033 break;
5034 case RISCVVType::LMUL_8:
5035 PerSlideCost = 8;
5036 break;
5037 }
5038
5039 // TODO: Should we be using the build instseq then cost + evaluate scheme
5040 // we use for integer constants here?
5041 unsigned UndefCount = 0;
5042 for (const SDValue &V : Op->ops()) {
5043 if (V.isUndef()) {
5044 UndefCount++;
5045 continue;
5046 }
5047 if (UndefCount) {
5048 LinearBudget -= PerSlideCost;
5049 UndefCount = 0;
5050 }
5051 LinearBudget -= PerSlideCost;
5052 }
5053 if (UndefCount) {
5054 LinearBudget -= PerSlideCost;
5055 }
5056
5057 if (LinearBudget < 0)
5058 return SDValue();
5059
5060 assert((!VT.isFloatingPoint() ||
5061 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
5062 "Illegal type which will result in reserved encoding");
5063
5064 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
5065
5066 // General case: splat the first operand and slide other operands down one
5067 // by one to form a vector. Alternatively, if every operand is an
5068 // extraction from element 0 of a vector, we use that vector from the last
5069 // extraction as the start value and slide up instead of slide down. Such that
5070 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
5071 // vslideup of 1 later and eliminate the vector to scalar movement, which is
5072 // something we cannot do with vslide1down/vslidedown.
5073 // Of course, using vslide1up/vslideup might increase the register pressure,
5074 // and that's why we conservatively limit to cases where every operand is an
5075 // extraction from the first element.
5076 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
5077 SDValue EVec;
5078 bool SlideUp = false;
5079 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
5080 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
5081 if (SlideUp)
5082 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
5083 Mask, VL, Policy);
5084 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
5085 Mask, VL, Policy);
5086 };
5087
5088 // The reason we don't use all_of here is because we're also capturing EVec
5089 // from the last non-undef operand. If the std::execution_policy of the
5090 // underlying std::all_of is anything but std::sequenced_policy we might
5091 // capture the wrong EVec.
5092 for (SDValue V : Operands) {
5093 using namespace SDPatternMatch;
5094 SlideUp = V.isUndef() || sd_match(V, m_ExtractElt(m_Value(EVec), m_Zero()));
5095 if (!SlideUp)
5096 break;
5097 }
5098
5099 // Do not slideup if the element type of EVec is different.
5100 if (SlideUp) {
5101 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
5102 MVT ContainerEltVT = ContainerVT.getVectorElementType();
5103 if (EVecEltVT != ContainerEltVT)
5104 SlideUp = false;
5105 }
5106
5107 if (SlideUp) {
5108 MVT EVecContainerVT = EVec.getSimpleValueType();
5109 // Make sure the original vector has scalable vector type.
5110 if (EVecContainerVT.isFixedLengthVector()) {
5111 EVecContainerVT =
5112 getContainerForFixedLengthVector(EVecContainerVT, Subtarget);
5113 EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget);
5114 }
5115
5116 // Adapt EVec's type into ContainerVT.
5117 if (EVecContainerVT.getVectorMinNumElements() <
5118 ContainerVT.getVectorMinNumElements())
5119 EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0);
5120 else
5121 EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0);
5122
5123 // Reverse the elements as we're going to slide up from the last element.
5124 std::reverse(Operands.begin(), Operands.end());
5125 }
5126
5127 SDValue Vec;
5128 UndefCount = 0;
5129 for (SDValue V : Operands) {
5130 if (V.isUndef()) {
5131 UndefCount++;
5132 continue;
5133 }
5134
5135 // Start our sequence with either a TA splat or extract source in the
5136 // hopes that hardware is able to recognize there's no dependency on the
5137 // prior value of our temporary register.
5138 if (!Vec) {
5139 if (SlideUp) {
5140 Vec = EVec;
5141 } else {
5142 Vec = DAG.getSplatVector(VT, DL, V);
5143 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5144 }
5145
5146 UndefCount = 0;
5147 continue;
5148 }
5149
5150 if (UndefCount) {
5151 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5152 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5153 VL);
5154 UndefCount = 0;
5155 }
5156
5157 unsigned Opcode;
5158 if (VT.isFloatingPoint())
5159 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
5160 else
5161 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
5162
5163 if (!VT.isFloatingPoint())
5164 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
5165 Vec = DAG.getNode(Opcode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
5166 V, Mask, VL);
5167 }
5168 if (UndefCount) {
5169 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5170 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5171 VL);
5172 }
5173 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5174}
5175
5176static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5177 SDValue Lo, SDValue Hi, SDValue VL,
5178 SelectionDAG &DAG) {
5179 if (!Passthru)
5180 Passthru = DAG.getUNDEF(VT);
5181 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
5182 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
5183 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
5184 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5185 // node in order to try and match RVV vector/scalar instructions.
5186 if ((LoC >> 31) == HiC)
5187 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5188
5189 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5190 // VL. This can temporarily increase VL if VL less than VLMAX.
5191 if (LoC == HiC) {
5192 SDValue NewVL;
5193 if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
5194 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
5195 else
5196 NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
5197 MVT InterVT =
5198 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
5199 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
5200 DAG.getUNDEF(InterVT), Lo, NewVL);
5201 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
5202 }
5203 }
5204
5205 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5206 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
5207 isa<ConstantSDNode>(Hi.getOperand(1)) &&
5208 Hi.getConstantOperandVal(1) == 31)
5209 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5210
5211 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5212 // even if it might be sign extended.
5213 if (Hi.isUndef())
5214 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5215
5216 // Fall back to a stack store and stride x0 vector load.
5217 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
5218 Hi, VL);
5219}
5220
5221// Called by type legalization to handle splat of i64 on RV32.
5222// FIXME: We can optimize this when the type has sign or zero bits in one
5223// of the halves.
5224static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5225 SDValue Scalar, SDValue VL,
5226 SelectionDAG &DAG) {
5227 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5228 SDValue Lo, Hi;
5229 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
5230 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5231}
5232
5233// This function lowers a splat of a scalar operand Splat with the vector
5234// length VL. It ensures the final sequence is type legal, which is useful when
5235// lowering a splat after type legalization.
5236static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5237 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5238 const RISCVSubtarget &Subtarget) {
5239 bool HasPassthru = Passthru && !Passthru.isUndef();
5240 if (!HasPassthru && !Passthru)
5241 Passthru = DAG.getUNDEF(VT);
5242
5243 MVT EltVT = VT.getVectorElementType();
5244 MVT XLenVT = Subtarget.getXLenVT();
5245
5246 if (VT.isFloatingPoint()) {
5247 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
5248 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5249 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5250 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5251 Scalar = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Scalar);
5252 else
5253 Scalar = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Scalar);
5254 MVT IVT = VT.changeVectorElementType(MVT::i16);
5255 Passthru = DAG.getNode(ISD::BITCAST, DL, IVT, Passthru);
5256 SDValue Splat =
5257 lowerScalarSplat(Passthru, Scalar, VL, IVT, DL, DAG, Subtarget);
5258 return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
5259 }
5260 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
5261 }
5262
5263 // Simplest case is that the operand needs to be promoted to XLenVT.
5264 if (Scalar.getValueType().bitsLE(XLenVT)) {
5265 // If the operand is a constant, sign extend to increase our chances
5266 // of being able to use a .vi instruction. ANY_EXTEND would become a
5267 // a zero extend and the simm5 check in isel would fail.
5268 // FIXME: Should we ignore the upper bits in isel instead?
5269 unsigned ExtOpc =
5270 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5271 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5272 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
5273 }
5274
5275 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5276 "Unexpected scalar for splat lowering!");
5277
5278 if (isOneConstant(VL) && isNullConstant(Scalar))
5279 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
5280 DAG.getConstant(0, DL, XLenVT), VL);
5281
5282 // Otherwise use the more complicated splatting algorithm.
5283 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5284}
5285
5286// This function lowers an insert of a scalar operand Scalar into lane
5287// 0 of the vector regardless of the value of VL. The contents of the
5288// remaining lanes of the result vector are unspecified. VL is assumed
5289// to be non-zero.
5290static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5291 const SDLoc &DL, SelectionDAG &DAG,
5292 const RISCVSubtarget &Subtarget) {
5293 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5294
5295 const MVT XLenVT = Subtarget.getXLenVT();
5296 SDValue Passthru = DAG.getUNDEF(VT);
5297
5298 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5299 isNullConstant(Scalar.getOperand(1))) {
5300 SDValue ExtractedVal = Scalar.getOperand(0);
5301 // The element types must be the same.
5302 if (ExtractedVal.getValueType().getVectorElementType() ==
5303 VT.getVectorElementType()) {
5304 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5305 MVT ExtractedContainerVT = ExtractedVT;
5306 if (ExtractedContainerVT.isFixedLengthVector()) {
5307 ExtractedContainerVT =
5308 getContainerForFixedLengthVector(ExtractedContainerVT, Subtarget);
5309 ExtractedVal = convertToScalableVector(ExtractedContainerVT,
5310 ExtractedVal, DAG, Subtarget);
5311 }
5312 if (ExtractedContainerVT.bitsLE(VT))
5313 return DAG.getInsertSubvector(DL, Passthru, ExtractedVal, 0);
5314 return DAG.getExtractSubvector(DL, VT, ExtractedVal, 0);
5315 }
5316 }
5317
5318 if (VT.isFloatingPoint())
5319 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5320 VL);
5321
5322 // Avoid the tricky legalization cases by falling back to using the
5323 // splat code which already handles it gracefully.
5324 if (!Scalar.getValueType().bitsLE(XLenVT))
5325 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
5326 DAG.getConstant(1, DL, XLenVT),
5327 VT, DL, DAG, Subtarget);
5328
5329 // If the operand is a constant, sign extend to increase our chances
5330 // of being able to use a .vi instruction. ANY_EXTEND would become a
5331 // a zero extend and the simm5 check in isel would fail.
5332 // FIXME: Should we ignore the upper bits in isel instead?
5333 unsigned ExtOpc =
5334 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5335 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5336 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5337 VL);
5338}
5339
5340/// If concat_vector(V1,V2) could be folded away to some existing
5341/// vector source, return it. Note that the source may be larger
5342/// than the requested concat_vector (i.e. a extract_subvector
5343/// might be required.)
5344static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5345 EVT VT = V1.getValueType();
5346 assert(VT == V2.getValueType() && "argument types must match");
5347 // Both input must be extracts.
5348 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5349 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5350 return SDValue();
5351
5352 // Extracting from the same source.
5353 SDValue Src = V1.getOperand(0);
5354 if (Src != V2.getOperand(0) ||
5355 VT.isScalableVector() != Src.getValueType().isScalableVector())
5356 return SDValue();
5357
5358 // The extracts must extract the two halves of the source.
5359 if (V1.getConstantOperandVal(1) != 0 ||
5360 V2.getConstantOperandVal(1) != VT.getVectorMinNumElements())
5361 return SDValue();
5362
5363 return Src;
5364}
5365
5366// Can this shuffle be performed on exactly one (possibly larger) input?
5367static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5368
5369 if (V2.isUndef())
5370 return V1;
5371
5372 unsigned NumElts = VT.getVectorNumElements();
5373 // Src needs to have twice the number of elements.
5374 // TODO: Update shuffle lowering to add the extract subvector
5375 if (SDValue Src = foldConcatVector(V1, V2);
5376 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5377 return Src;
5378
5379 return SDValue();
5380}
5381
5382static bool isLegalVTForZvzipOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5383 MVT ContainerVT = VT;
5384 if (VT.isFixedLengthVector())
5385 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5386 // Determine LMUL of the container vector.
5387 return RISCVTargetLowering::getLMUL(ContainerVT) != RISCVVType::LMUL_8;
5388}
5389
5390/// Is this shuffle interleaving contiguous elements from one vector into the
5391/// even elements and contiguous elements from another vector into the odd
5392/// elements. \p EvenSrc will contain the element that should be in the first
5393/// even element. \p OddSrc will contain the element that should be in the first
5394/// odd element. These can be the first element in a source or the element half
5395/// way through the source.
5396static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5397 int &OddSrc, const RISCVSubtarget &Subtarget) {
5398 // We need to be able to widen elements to the next larger integer type or
5399 // use the vzip instruction at e64.
5400 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
5401 if (!Subtarget.hasStdExtZvzip())
5402 return false;
5403 if (!isLegalVTForZvzipOperand(VT, Subtarget))
5404 return false;
5405 }
5406
5407 int Size = Mask.size();
5408 int NumElts = VT.getVectorNumElements();
5409 assert(Size == (int)NumElts && "Unexpected mask size");
5410
5411 SmallVector<unsigned, 2> StartIndexes;
5412 if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
5413 return false;
5414
5415 EvenSrc = StartIndexes[0];
5416 OddSrc = StartIndexes[1];
5417
5418 // One source should be low half of first vector.
5419 if (EvenSrc != 0 && OddSrc != 0)
5420 return false;
5421
5422 // Subvectors will be subtracted from either at the start of the two input
5423 // vectors, or at the start and middle of the first vector if it's an unary
5424 // interleave.
5425 // In both cases, HalfNumElts will be extracted.
5426 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5427 // we'll create an illegal extract_subvector.
5428 // FIXME: We could support other values using a slidedown first.
5429 int HalfNumElts = NumElts / 2;
5430 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5431}
5432
5433/// Is this mask representing a masked combination of two slides?
5434static bool isMaskedSlidePair(ArrayRef<int> Mask,
5435 std::array<std::pair<int, int>, 2> &SrcInfo) {
5436 if (!llvm::isMaskedSlidePair(Mask, Mask.size(), SrcInfo))
5437 return false;
5438
5439 // Avoid matching vselect idioms
5440 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5441 return false;
5442 // Prefer vslideup as the second instruction, and identity
5443 // only as the initial instruction.
5444 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5445 SrcInfo[1].second == 0)
5446 std::swap(SrcInfo[0], SrcInfo[1]);
5447 assert(SrcInfo[0].first != -1 && "Must find one slide");
5448 return true;
5449}
5450
5451// Exactly matches the semantics of a previously existing custom matcher
5452// to allow migration to new matcher without changing output.
5453static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5454 unsigned NumElts) {
5455 if (SrcInfo[1].first == -1)
5456 return true;
5457 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5458 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5459}
5460
5461static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5462 ArrayRef<int> Mask, unsigned Factor,
5463 bool RequiredPolarity) {
5464 int NumElts = Mask.size();
5465 for (const auto &[Idx, M] : enumerate(Mask)) {
5466 if (M < 0)
5467 continue;
5468 int Src = M >= NumElts;
5469 int Diff = (int)Idx - (M % NumElts);
5470 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5471 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5472 "Must match exactly one of the two slides");
5473 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5474 return false;
5475 }
5476 return true;
5477}
5478
5479/// Given a shuffle which can be represented as a pair of two slides,
5480/// see if it is a pair-even idiom.
5481/// Pair-even is:
5482/// vs2: a0 a1 a2 a3
5483/// vs1: b0 b1 b2 b3
5484/// vd: a0 b0 a2 b2
5485static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5486 ArrayRef<int> Mask, unsigned &Factor) {
5487 Factor = SrcInfo[1].second;
5488 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5489 Mask.size() % Factor == 0 &&
5490 isAlternating(SrcInfo, Mask, Factor, true);
5491}
5492
5493/// Given a shuffle which can be represented as a pair of two slides,
5494/// see if it is a pair-odd idiom.
5495/// Pair-odd is:
5496/// vs2: a0 a1 a2 a3
5497/// vs1: b0 b1 b2 b3
5498/// vd: a1 b1 a3 b3
5499/// Note that the operand order is swapped due to the way we canonicalize
5500/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5501static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5502 ArrayRef<int> Mask, unsigned &Factor) {
5503 Factor = -SrcInfo[1].second;
5504 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5505 Mask.size() % Factor == 0 &&
5506 isAlternating(SrcInfo, Mask, Factor, false);
5507}
5508
5509// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5510// 2, 4, 8 and the integer type Factor-times larger than VT's
5511// element type must be a legal element type.
5512// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5513// -> [p, q, r, s] (Factor=2, Index=1)
5514static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5515 SDValue Src, unsigned Factor,
5516 unsigned Index, SelectionDAG &DAG) {
5517 unsigned EltBits = VT.getScalarSizeInBits();
5518 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5519 MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
5520 SrcEC.divideCoefficientBy(Factor));
5521 MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
5522 SrcEC.divideCoefficientBy(Factor));
5523 Src = DAG.getBitcast(WideSrcVT, Src);
5524
5525 unsigned Shift = Index * EltBits;
5526 SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
5527 DAG.getConstant(Shift, DL, WideSrcVT));
5528 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
5529 MVT CastVT = ResVT.changeVectorElementType(VT.getVectorElementType());
5530 Res = DAG.getBitcast(CastVT, Res);
5531 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
5532}
5533
5534/// Match a single source shuffle which is an identity except that some
5535/// particular element is repeated. This can be lowered as a masked
5536/// vrgather.vi/vx. Note that the two source form of this is handled
5537/// by the recursive splitting logic and doesn't need special handling.
5538static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5539 const RISCVSubtarget &Subtarget,
5540 SelectionDAG &DAG) {
5541
5542 SDLoc DL(SVN);
5543 MVT VT = SVN->getSimpleValueType(0);
5544 SDValue V1 = SVN->getOperand(0);
5545 assert(SVN->getOperand(1).isUndef());
5546 ArrayRef<int> Mask = SVN->getMask();
5547 const unsigned NumElts = VT.getVectorNumElements();
5548 MVT XLenVT = Subtarget.getXLenVT();
5549
5550 std::optional<int> SplatIdx;
5551 for (auto [I, M] : enumerate(Mask)) {
5552 if (M == -1 || I == (unsigned)M)
5553 continue;
5554 if (SplatIdx && *SplatIdx != M)
5555 return SDValue();
5556 SplatIdx = M;
5557 }
5558
5559 if (!SplatIdx)
5560 return SDValue();
5561
5562 SmallVector<SDValue> MaskVals;
5563 for (int MaskIndex : Mask) {
5564 bool SelectMaskVal = MaskIndex == *SplatIdx;
5565 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5566 }
5567 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5568 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
5569 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5570 SDValue Splat = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT),
5571 SmallVector<int>(NumElts, *SplatIdx));
5572 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, Splat, V1);
5573}
5574
5575// Lower the following shuffle to vslidedown.
5576// a)
5577// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5578// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5579// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5580// b)
5581// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5582// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5583// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5584// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5585// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5586// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5587static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5588 SDValue V1, SDValue V2,
5589 ArrayRef<int> Mask,
5590 const RISCVSubtarget &Subtarget,
5591 SelectionDAG &DAG) {
5592 auto findNonEXTRACT_SUBVECTORParent =
5593 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5594 uint64_t Offset = 0;
5595 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5596 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5597 // a scalable vector. But we don't want to match the case.
5598 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
5599 Offset += Parent.getConstantOperandVal(1);
5600 Parent = Parent.getOperand(0);
5601 }
5602 return std::make_pair(Parent, Offset);
5603 };
5604
5605 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5606 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5607
5608 // Extracting from the same source.
5609 SDValue Src = V1Src;
5610 if (Src != V2Src)
5611 return SDValue();
5612
5613 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5614 SmallVector<int, 16> NewMask(Mask);
5615 for (size_t i = 0; i != NewMask.size(); ++i) {
5616 if (NewMask[i] == -1)
5617 continue;
5618
5619 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5620 NewMask[i] = NewMask[i] + V1IndexOffset;
5621 } else {
5622 // Minus NewMask.size() is needed. Otherwise, the b case would be
5623 // <5,6,7,12> instead of <5,6,7,8>.
5624 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5625 }
5626 }
5627
5628 // First index must be known and non-zero. It will be used as the slidedown
5629 // amount.
5630 if (NewMask[0] <= 0)
5631 return SDValue();
5632
5633 // NewMask is also continuous.
5634 for (unsigned i = 1; i != NewMask.size(); ++i)
5635 if (NewMask[i - 1] + 1 != NewMask[i])
5636 return SDValue();
5637
5638 MVT XLenVT = Subtarget.getXLenVT();
5639 MVT SrcVT = Src.getSimpleValueType();
5640 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
5641 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
5642 SDValue Slidedown =
5643 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
5644 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
5645 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
5646 return DAG.getExtractSubvector(
5647 DL, VT, convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), 0);
5648}
5649
5650// Because vslideup leaves the destination elements at the start intact, we can
5651// use it to perform shuffles that insert subvectors:
5652//
5653// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5654// ->
5655// vsetvli zero, 8, e8, mf2, ta, ma
5656// vslideup.vi v8, v9, 4
5657//
5658// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5659// ->
5660// vsetvli zero, 5, e8, mf2, tu, ma
5661// vslideup.v1 v8, v9, 2
5662static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5663 SDValue V1, SDValue V2,
5664 ArrayRef<int> Mask,
5665 const RISCVSubtarget &Subtarget,
5666 SelectionDAG &DAG) {
5667 unsigned NumElts = VT.getVectorNumElements();
5668 int NumSubElts, Index;
5669 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumElts, NumSubElts,
5670 Index))
5671 return SDValue();
5672
5673 bool OpsSwapped = Mask[Index] < (int)NumElts;
5674 SDValue InPlace = OpsSwapped ? V2 : V1;
5675 SDValue ToInsert = OpsSwapped ? V1 : V2;
5676
5677 MVT XLenVT = Subtarget.getXLenVT();
5678 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5679 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
5680 // We slide up by the index that the subvector is being inserted at, and set
5681 // VL to the index + the number of elements being inserted.
5682 unsigned Policy =
5683 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5684 // If the we're adding a suffix to the in place vector, i.e. inserting right
5685 // up to the very end of it, then we don't actually care about the tail.
5686 if (NumSubElts + Index >= (int)NumElts)
5687 Policy |= RISCVVType::TAIL_AGNOSTIC;
5688
5689 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
5690 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
5691 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
5692
5693 SDValue Res;
5694 // If we're inserting into the lowest elements, use a tail undisturbed
5695 // vmv.v.v.
5696 if (Index == 0)
5697 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
5698 VL);
5699 else
5700 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
5701 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
5702 return convertFromScalableVector(VT, Res, DAG, Subtarget);
5703}
5704
5705// A shuffle of shuffles where the final data only is drawn from 2 input ops
5706// can be compressed into a single shuffle
5707static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5708 const RISCVSubtarget &Subtarget,
5709 SelectionDAG &DAG) {
5710 SDValue V1 = SVN->getOperand(0);
5711 SDValue V2 = SVN->getOperand(1);
5712
5713 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5714 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5715 return SDValue();
5716
5717 if (!V1.hasOneUse() || !V2.hasOneUse())
5718 return SDValue();
5719
5720 ArrayRef<int> Mask = SVN->getMask();
5721 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(V1.getNode())->getMask();
5722 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(V2.getNode())->getMask();
5723 unsigned NumElts = Mask.size();
5724 SmallVector<int> NewMask(NumElts, -1);
5725 for (unsigned Idx : seq<unsigned>(NumElts)) {
5726 int Lane = Mask[Idx];
5727 // Don't assign if poison
5728 if (Lane == -1)
5729 continue;
5730 int OrigLane;
5731 bool SecondOp = false;
5732 if ((unsigned)Lane < NumElts) {
5733 OrigLane = V1Mask[Lane];
5734 } else {
5735 OrigLane = V2Mask[Lane - NumElts];
5736 SecondOp = true;
5737 }
5738 if (OrigLane == -1)
5739 continue;
5740 // Don't handle if shuffling from a second operand
5741 if ((unsigned)OrigLane >= NumElts)
5742 return SDValue();
5743 if (SecondOp)
5744 OrigLane += NumElts;
5745 NewMask[Idx] = OrigLane;
5746 }
5747
5748 EVT VT = SVN->getValueType(0);
5749 SDLoc DL(SVN);
5750
5751 return DAG.getVectorShuffle(VT, DL, V1->getOperand(0), V2->getOperand(0),
5752 NewMask);
5753}
5754
5755/// Match v(f)slide1up/down idioms. These operations involve sliding
5756/// N-1 elements to make room for an inserted scalar at one end.
5757static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5758 SDValue V1, SDValue V2,
5759 ArrayRef<int> Mask,
5760 const RISCVSubtarget &Subtarget,
5761 SelectionDAG &DAG) {
5762 bool OpsSwapped = false;
5763 if (!isa<BuildVectorSDNode>(V1)) {
5764 if (!isa<BuildVectorSDNode>(V2))
5765 return SDValue();
5766 std::swap(V1, V2);
5767 OpsSwapped = true;
5768 }
5769 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
5770 if (!Splat)
5771 return SDValue();
5772
5773 // Return true if the mask could describe a slide of Mask.size() - 1
5774 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5775 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5776 const unsigned S = (Offset > 0) ? 0 : -Offset;
5777 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5778 for (unsigned i = S; i != E; ++i)
5779 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5780 return false;
5781 return true;
5782 };
5783
5784 const unsigned NumElts = VT.getVectorNumElements();
5785 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5786 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5787 return SDValue();
5788
5789 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5790 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5791 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5792 return SDValue();
5793
5794 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5795 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5796
5797 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5798 // vslide1{down,up}.vx instead.
5799 if ((VT.getVectorElementType() == MVT::bf16 &&
5800 !Subtarget.hasVInstructionsBF16()) ||
5801 (VT.getVectorElementType() == MVT::f16 &&
5802 !Subtarget.hasVInstructionsF16())) {
5803 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5804 Splat =
5805 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Splat);
5806 V2 = DAG.getBitcast(
5807 IntVT, convertToScalableVector(ContainerVT, V2, DAG, Subtarget));
5808 SDValue Vec = DAG.getNode(
5809 IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5810 IntVT, DAG.getUNDEF(IntVT), V2, Splat, TrueMask, VL);
5811 Vec = DAG.getBitcast(ContainerVT, Vec);
5812 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5813 }
5814
5815 auto OpCode = IsVSlidedown ?
5816 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5817 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5818 if (!VT.isFloatingPoint())
5819 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
5820 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
5821 DAG.getUNDEF(ContainerVT),
5822 convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
5823 Splat, TrueMask, VL);
5824 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5825}
5826
5827/// Match a mask which "spreads" the leading elements of a vector evenly
5828/// across the result. Factor is the spread amount, and Index is the
5829/// offset applied. (on success, Index < Factor) This is the inverse
5830/// of a deinterleave with the same Factor and Index. This is analogous
5831/// to an interleave, except that all but one lane is undef.
5832bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5833 unsigned &Index) {
5834 SmallVector<bool> LaneIsUndef(Factor, true);
5835 for (unsigned i = 0; i < Mask.size(); i++)
5836 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5837
5838 bool Found = false;
5839 for (unsigned i = 0; i < Factor; i++) {
5840 if (LaneIsUndef[i])
5841 continue;
5842 if (Found)
5843 return false;
5844 Index = i;
5845 Found = true;
5846 }
5847 if (!Found)
5848 return false;
5849
5850 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5851 unsigned j = i * Factor + Index;
5852 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5853 return false;
5854 }
5855 return true;
5856}
5857
5858static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
5859 const SDLoc &DL, SelectionDAG &DAG,
5860 const RISCVSubtarget &Subtarget) {
5861 assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
5862 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5863
5864 MVT VT = Op0.getSimpleValueType();
5865 MVT IntVT = VT.changeVectorElementTypeToInteger();
5866 Op0 = DAG.getBitcast(IntVT, Op0);
5867 Op1 = DAG.getBitcast(IntVT, Op1);
5868
5869 MVT ContainerVT = IntVT;
5870 if (VT.isFixedLengthVector()) {
5871 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5872 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5873 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5874 }
5875
5876 MVT InnerVT = ContainerVT;
5877 auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
5878
5879 SDValue Passthru = DAG.getUNDEF(InnerVT);
5880 SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
5881 if (IntVT.isFixedLengthVector())
5882 Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
5883 Res = DAG.getBitcast(VT, Res);
5884 return Res;
5885}
5886
5887static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
5888 SelectionDAG &DAG,
5889 const RISCVSubtarget &Subtarget) {
5890 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5891 MVT VT = Op0.getSimpleValueType();
5892 MVT IntVT = VT.changeVectorElementTypeToInteger();
5893 Op0 = DAG.getBitcast(IntVT, Op0);
5894 Op1 = DAG.getBitcast(IntVT, Op1);
5895 MVT ContainerVT = IntVT;
5896 if (VT.isFixedLengthVector()) {
5897 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5898 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5899 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5900 }
5901 MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
5902 auto [Mask, VL] = getDefaultVLOps(IntVT, ContainerVT, DL, DAG, Subtarget);
5903 SDValue Passthru = DAG.getUNDEF(ResVT);
5904 SDValue Res =
5905 DAG.getNode(RISCVISD::VZIP_VL, DL, ResVT, Op0, Op1, Passthru, Mask, VL);
5906 if (IntVT.isFixedLengthVector())
5907 Res = convertFromScalableVector(IntVT.getDoubleNumVectorElementsVT(), Res,
5908 DAG, Subtarget);
5909 Res = DAG.getBitcast(VT.getDoubleNumVectorElementsVT(), Res);
5910 return Res;
5911}
5912
5913static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
5914 SelectionDAG &DAG,
5915 const RISCVSubtarget &Subtarget) {
5916 assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
5917 MVT VT = Op.getSimpleValueType();
5918 assert(VT.getVectorMinNumElements() >= 2);
5919
5920 MVT IntVT = VT.changeVectorElementTypeToInteger();
5921 Op = DAG.getBitcast(IntVT, Op);
5922 MVT ContainerVT = IntVT;
5923 if (VT.isFixedLengthVector()) {
5924 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5925 // For E64 with LMUL <= 1, we can't represent a smaller fractional LMUL for
5926 // the result (LMUL <= 1/2 is not valid for E64). We must widen the input
5927 // container to at least LMUL=2 so the result can be LMUL=1.
5928 if (ContainerVT.getVectorElementType() == MVT::i64 &&
5929 RISCVTargetLowering::getLMUL(ContainerVT) == RISCVVType::LMUL_1) {
5930 ContainerVT = MVT::getScalableVectorVT(MVT::i64, 2);
5931 }
5932 Op = convertToScalableVector(ContainerVT, Op, DAG, Subtarget);
5933 }
5934
5935 MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
5936 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5937 MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
5938 auto [Mask, VL] = getDefaultVLOps(ResVT, ResVT, DL, DAG, Subtarget);
5939 if (VT.isFixedLengthVector())
5940 VL = DAG.getConstant(VT.getVectorNumElements() / 2, DL,
5941 Subtarget.getXLenVT());
5942 SDValue Passthru = DAG.getUNDEF(ResVT);
5943 SDValue Res = DAG.getNode(Opc, DL, ResVT, Op, Passthru, Mask, VL);
5944 if (HalfIntVT.isFixedLengthVector())
5945 Res = convertFromScalableVector(HalfIntVT, Res, DAG, Subtarget);
5946 Res = DAG.getBitcast(HalfVT, Res);
5947 return Res;
5948}
5949
5950// Given a vector a, b, c, d return a vector Factor times longer
5951// with Factor-1 undef's between elements. Ex:
5952// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5953// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5954static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5955 const SDLoc &DL, SelectionDAG &DAG) {
5956
5957 MVT VT = V.getSimpleValueType();
5958 unsigned EltBits = VT.getScalarSizeInBits();
5959 ElementCount EC = VT.getVectorElementCount();
5960 V = DAG.getBitcast(VT.changeTypeToInteger(), V);
5961
5962 MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
5963
5964 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
5965 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5966 // allow the SHL to fold away if Index is 0.
5967 if (Index != 0)
5968 Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
5969 DAG.getConstant(EltBits * Index, DL, WideVT));
5970 // Make sure to use original element type
5971 MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
5972 EC.multiplyCoefficientBy(Factor));
5973 return DAG.getBitcast(ResultVT, Result);
5974}
5975
5976// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
5977// to create an interleaved vector of <[vscale x] n*2 x ty>.
5978// This requires that the size of ty is less than the subtarget's maximum ELEN.
5979static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
5980 const SDLoc &DL, SelectionDAG &DAG,
5981 const RISCVSubtarget &Subtarget) {
5982
5983 // FIXME: Not only does this optimize the code, it fixes some correctness
5984 // issues because MIR does not have freeze.
5985 if (EvenV.isUndef())
5986 return getWideningSpread(OddV, 2, 1, DL, DAG);
5987 if (OddV.isUndef())
5988 return getWideningSpread(EvenV, 2, 0, DL, DAG);
5989
5990 MVT VecVT = EvenV.getSimpleValueType();
5991 MVT VecContainerVT = VecVT; // <vscale x n x ty>
5992 // Convert fixed vectors to scalable if needed
5993 if (VecContainerVT.isFixedLengthVector()) {
5994 VecContainerVT = getContainerForFixedLengthVector(VecVT, Subtarget);
5995 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
5996 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
5997 }
5998
5999 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
6000
6001 // We're working with a vector of the same size as the resulting
6002 // interleaved vector, but with half the number of elements and
6003 // twice the SEW (Hence the restriction on not using the maximum
6004 // ELEN)
6005 MVT WideVT =
6006 MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
6007 VecVT.getVectorElementCount());
6008 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
6009 if (WideContainerVT.isFixedLengthVector())
6010 WideContainerVT = getContainerForFixedLengthVector(WideVT, Subtarget);
6011
6012 // Bitcast the input vectors to integers in case they are FP
6013 VecContainerVT = VecContainerVT.changeTypeToInteger();
6014 EvenV = DAG.getBitcast(VecContainerVT, EvenV);
6015 OddV = DAG.getBitcast(VecContainerVT, OddV);
6016
6017 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
6018 SDValue Passthru = DAG.getUNDEF(WideContainerVT);
6019
6020 SDValue Interleaved;
6021 if (Subtarget.hasStdExtZvbb()) {
6022 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
6023 SDValue OffsetVec =
6024 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
6025 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
6026 OffsetVec, Passthru, Mask, VL);
6027 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
6028 Interleaved, EvenV, Passthru, Mask, VL);
6029 } else {
6030 // FIXME: We should freeze the odd vector here. We already handled the case
6031 // of provably undef/poison above.
6032
6033 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
6034 // vwaddu.vv
6035 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
6036 OddV, Passthru, Mask, VL);
6037
6038 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
6039 SDValue AllOnesVec = DAG.getSplatVector(
6040 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
6041 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
6042 OddV, AllOnesVec, Passthru, Mask, VL);
6043
6044 // Add the two together so we get
6045 // (OddV * 0xff...ff) + (OddV + EvenV)
6046 // = (OddV * 0x100...00) + EvenV
6047 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
6048 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
6049 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
6050 Interleaved, OddsMul, Passthru, Mask, VL);
6051 }
6052
6053 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
6054 MVT ResultContainerVT = MVT::getVectorVT(
6055 VecVT.getVectorElementType(), // Make sure to use original type
6056 VecContainerVT.getVectorElementCount().multiplyCoefficientBy(2));
6057 Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
6058
6059 // Convert back to a fixed vector if needed
6060 MVT ResultVT =
6061 MVT::getVectorVT(VecVT.getVectorElementType(),
6062 VecVT.getVectorElementCount().multiplyCoefficientBy(2));
6063 if (ResultVT.isFixedLengthVector())
6064 Interleaved =
6065 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
6066
6067 return Interleaved;
6068}
6069
6070// If we have a vector of bits that we want to reverse, we can use a vbrev on a
6071// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
6072static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
6073 SelectionDAG &DAG,
6074 const RISCVSubtarget &Subtarget) {
6075 SDLoc DL(SVN);
6076 MVT VT = SVN->getSimpleValueType(0);
6077 SDValue V = SVN->getOperand(0);
6078 unsigned NumElts = VT.getVectorNumElements();
6079
6080 assert(VT.getVectorElementType() == MVT::i1);
6081
6082 if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
6083 SVN->getMask().size()) ||
6084 !SVN->getOperand(1).isUndef())
6085 return SDValue();
6086
6087 unsigned ViaEltSize = std::max((uint64_t)8, PowerOf2Ceil(NumElts));
6088 EVT ViaVT = EVT::getVectorVT(
6089 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
6090 EVT ViaBitVT =
6091 EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
6092
6093 // If we don't have zvbb or the larger element type > ELEN, the operation will
6094 // be illegal.
6095 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
6096 ViaVT) ||
6097 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
6098 return SDValue();
6099
6100 // If the bit vector doesn't fit exactly into the larger element type, we need
6101 // to insert it into the larger vector and then shift up the reversed bits
6102 // afterwards to get rid of the gap introduced.
6103 if (ViaEltSize > NumElts)
6104 V = DAG.getInsertSubvector(DL, DAG.getUNDEF(ViaBitVT), V, 0);
6105
6106 SDValue Res =
6107 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
6108
6109 // Shift up the reversed bits if the vector didn't exactly fit into the larger
6110 // element type.
6111 if (ViaEltSize > NumElts)
6112 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
6113 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
6114
6115 Res = DAG.getBitcast(ViaBitVT, Res);
6116
6117 if (ViaEltSize > NumElts)
6118 Res = DAG.getExtractSubvector(DL, VT, Res, 0);
6119 return Res;
6120}
6121
6122static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
6123 const RISCVSubtarget &Subtarget,
6124 MVT &RotateVT, unsigned &RotateAmt) {
6125 unsigned NumElts = VT.getVectorNumElements();
6126 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6127 unsigned NumSubElts;
6128 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, 2,
6129 NumElts, NumSubElts, RotateAmt))
6130 return false;
6131 RotateVT = MVT::getVectorVT(MVT::getIntegerVT(EltSizeInBits * NumSubElts),
6132 NumElts / NumSubElts);
6133
6134 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
6135 return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
6136}
6137
6138// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
6139// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
6140// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
6141static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
6142 SelectionDAG &DAG,
6143 const RISCVSubtarget &Subtarget) {
6144 SDLoc DL(SVN);
6145
6146 EVT VT = SVN->getValueType(0);
6147 unsigned RotateAmt;
6148 MVT RotateVT;
6149 if (!isLegalBitRotate(SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
6150 return SDValue();
6151
6152 SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
6153
6154 SDValue Rotate;
6155 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
6156 // so canonicalize to vrev8.
6157 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
6158 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
6159 else
6160 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
6161 DAG.getConstant(RotateAmt, DL, RotateVT));
6162
6163 return DAG.getBitcast(VT, Rotate);
6164}
6165
6166// If compiling with an exactly known VLEN, see if we can split a
6167// shuffle on m2 or larger into a small number of m1 sized shuffles
6168// which write each destination registers exactly once.
6169static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
6170 SelectionDAG &DAG,
6171 const RISCVSubtarget &Subtarget) {
6172 SDLoc DL(SVN);
6173 MVT VT = SVN->getSimpleValueType(0);
6174 SDValue V1 = SVN->getOperand(0);
6175 SDValue V2 = SVN->getOperand(1);
6176 ArrayRef<int> Mask = SVN->getMask();
6177
6178 // If we don't know exact data layout, not much we can do. If this
6179 // is already m1 or smaller, no point in splitting further.
6180 const auto VLen = Subtarget.getRealVLen();
6181 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
6182 return SDValue();
6183
6184 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
6185 // expansion for.
6186 unsigned RotateAmt;
6187 MVT RotateVT;
6188 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
6189 return SDValue();
6190
6191 MVT ElemVT = VT.getVectorElementType();
6192 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
6193
6194 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
6195 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
6196 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
6197 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
6198 unsigned NumOpElts = M1VT.getVectorMinNumElements();
6199 unsigned NumElts = ContainerVT.getVectorMinNumElements();
6200 unsigned NumOfSrcRegs = NumElts / NumOpElts;
6201 unsigned NumOfDestRegs = NumElts / NumOpElts;
6202 // The following semantically builds up a fixed length concat_vector
6203 // of the component shuffle_vectors. We eagerly lower to scalable here
6204 // to avoid DAG combining it back to a large shuffle_vector again.
6205 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6206 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
6207 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
6208 Operands;
6209 processShuffleMasks(
6210 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
6211 [&]() { Operands.emplace_back(); },
6212 [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
6213 Operands.emplace_back().emplace_back(SrcVecIdx, UINT_MAX,
6214 SmallVector<int>(SrcSubMask));
6215 },
6216 [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
6217 if (NewReg)
6218 Operands.emplace_back();
6219 Operands.back().emplace_back(Idx1, Idx2, SmallVector<int>(SrcSubMask));
6220 });
6221 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
6222 // Note: check that we do not emit too many shuffles here to prevent code
6223 // size explosion.
6224 // TODO: investigate, if it can be improved by extra analysis of the masks to
6225 // check if the code is more profitable.
6226 unsigned NumShuffles = std::accumulate(
6227 Operands.begin(), Operands.end(), 0u,
6228 [&](unsigned N,
6229 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
6230 if (Data.empty())
6231 return N;
6232 N += Data.size();
6233 for (const auto &P : Data) {
6234 unsigned Idx2 = std::get<1>(P);
6235 ArrayRef<int> Mask = std::get<2>(P);
6236 if (Idx2 != UINT_MAX)
6237 ++N;
6238 else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
6239 --N;
6240 }
6241 return N;
6242 });
6243 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6244 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6245 return SDValue();
6246 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6247 SDValue SubVec = DAG.getExtractSubvector(DL, M1VT, SrcVec, ExtractIdx);
6248 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
6249 return SubVec;
6250 };
6251 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6252 ArrayRef<int> Mask) {
6253 SDValue SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec1, SubVec2, Mask);
6254 return SubVec;
6255 };
6256 SDValue Vec = DAG.getUNDEF(ContainerVT);
6257 for (auto [I, Data] : enumerate(Operands)) {
6258 if (Data.empty())
6259 continue;
6260 SmallDenseMap<unsigned, SDValue, 4> Values;
6261 for (unsigned I : seq<unsigned>(Data.size())) {
6262 const auto &[Idx1, Idx2, _] = Data[I];
6263 // If the shuffle contains permutation of odd number of elements,
6264 // Idx1 might be used already in the first iteration.
6265 //
6266 // Idx1 = shuffle Idx1, Idx2
6267 // Idx1 = shuffle Idx1, Idx3
6268 SDValue &V = Values.try_emplace(Idx1).first->getSecond();
6269 if (!V)
6270 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6271 (Idx1 % NumOfSrcRegs) * NumOpElts);
6272 if (Idx2 != UINT_MAX) {
6273 SDValue &V = Values.try_emplace(Idx2).first->getSecond();
6274 if (!V)
6275 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6276 (Idx2 % NumOfSrcRegs) * NumOpElts);
6277 }
6278 }
6279 SDValue V;
6280 for (const auto &[Idx1, Idx2, Mask] : Data) {
6281 SDValue V1 = Values.at(Idx1);
6282 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Idx2);
6283 V = PerformShuffle(V1, V2, Mask);
6284 Values[Idx1] = V;
6285 }
6286
6287 unsigned InsertIdx = I * NumOpElts;
6288 V = convertToScalableVector(M1VT, V, DAG, Subtarget);
6289 Vec = DAG.getInsertSubvector(DL, Vec, V, InsertIdx);
6290 }
6291 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
6292}
6293
6294// Matches a subset of compress masks with a contiguous prefix of output
6295// elements. This could be extended to allow gaps by deciding which
6296// source elements to spuriously demand.
6297static bool isCompressMask(ArrayRef<int> Mask) {
6298 int Last = -1;
6299 bool SawUndef = false;
6300 for (const auto &[Idx, M] : enumerate(Mask)) {
6301 if (M == -1) {
6302 SawUndef = true;
6303 continue;
6304 }
6305 if (SawUndef)
6306 return false;
6307 if (Idx > (unsigned)M)
6308 return false;
6309 if (M <= Last)
6310 return false;
6311 Last = M;
6312 }
6313 return true;
6314}
6315
6316/// Given a shuffle where the indices are disjoint between the two sources,
6317/// e.g.:
6318///
6319/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6320///
6321/// Merge the two sources into one and do a single source shuffle:
6322///
6323/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6324/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6325///
6326/// A vselect will either be merged into a masked instruction or be lowered as a
6327/// vmerge.vvm, which is cheaper than a vrgather.vv.
6328static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6329 SelectionDAG &DAG,
6330 const RISCVSubtarget &Subtarget) {
6331 MVT VT = SVN->getSimpleValueType(0);
6332 MVT XLenVT = Subtarget.getXLenVT();
6333 SDLoc DL(SVN);
6334
6335 const ArrayRef<int> Mask = SVN->getMask();
6336
6337 // Work out which source each lane will come from.
6338 SmallVector<int, 16> Srcs(Mask.size(), -1);
6339
6340 for (int Idx : Mask) {
6341 if (Idx == -1)
6342 continue;
6343 unsigned SrcIdx = Idx % Mask.size();
6344 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6345 if (Srcs[SrcIdx] == -1)
6346 // Mark this source as using this lane.
6347 Srcs[SrcIdx] = Src;
6348 else if (Srcs[SrcIdx] != Src)
6349 // The other source is using this lane: not disjoint.
6350 return SDValue();
6351 }
6352
6353 SmallVector<SDValue> SelectMaskVals;
6354 for (int Lane : Srcs) {
6355 if (Lane == -1)
6356 SelectMaskVals.push_back(DAG.getUNDEF(XLenVT));
6357 else
6358 SelectMaskVals.push_back(DAG.getConstant(Lane ? 0 : 1, DL, XLenVT));
6359 }
6360 MVT MaskVT = VT.changeVectorElementType(MVT::i1);
6361 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, SelectMaskVals);
6362 SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
6363 SVN->getOperand(0), SVN->getOperand(1));
6364
6365 // Move all indices relative to the first source.
6366 SmallVector<int> NewMask(Mask.size());
6367 for (unsigned I = 0; I < Mask.size(); I++) {
6368 if (Mask[I] == -1)
6369 NewMask[I] = -1;
6370 else
6371 NewMask[I] = Mask[I] % Mask.size();
6372 }
6373
6374 return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
6375}
6376
6377/// Is this mask local (i.e. elements only move within their local span), and
6378/// repeating (that is, the same rearrangement is being done within each span)?
6379static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6380 // Require a prefix from the original mask until the consumer code
6381 // is adjusted to rewrite the mask instead of just taking a prefix.
6382 for (auto [I, M] : enumerate(Mask)) {
6383 if (M == -1)
6384 continue;
6385 if ((M / Span) != (int)(I / Span))
6386 return false;
6387 int SpanIdx = I % Span;
6388 int Expected = M % Span;
6389 if (Mask[SpanIdx] != Expected)
6390 return false;
6391 }
6392 return true;
6393}
6394
6395/// Is this mask only using elements from the first span of the input?
6396static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6397 return all_of(Mask, [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6398}
6399
6400/// Return true for a mask which performs an arbitrary shuffle within the first
6401/// span, and then repeats that same result across all remaining spans. Note
6402/// that this doesn't check if all the inputs come from a single span!
6403static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6404 // Require a prefix from the original mask until the consumer code
6405 // is adjusted to rewrite the mask instead of just taking a prefix.
6406 for (auto [I, M] : enumerate(Mask)) {
6407 if (M == -1)
6408 continue;
6409 int SpanIdx = I % Span;
6410 if (Mask[SpanIdx] != M)
6411 return false;
6412 }
6413 return true;
6414}
6415
6416/// Try to widen element type to get a new mask value for a better permutation
6417/// sequence. This doesn't try to inspect the widened mask for profitability;
6418/// we speculate the widened form is equal or better. This has the effect of
6419/// reducing mask constant sizes - allowing cheaper materialization sequences
6420/// - and index sequence sizes - reducing register pressure and materialization
6421/// cost, at the cost of (possibly) an extra VTYPE toggle.
6422static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6423 SDLoc DL(Op);
6424 MVT VT = Op.getSimpleValueType();
6425 MVT ScalarVT = VT.getVectorElementType();
6426 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6427 SDValue V0 = Op.getOperand(0);
6428 SDValue V1 = Op.getOperand(1);
6429 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
6430
6431 // Avoid wasted work leading to isTypeLegal check failing below
6432 if (ElementSize > 32)
6433 return SDValue();
6434
6435 SmallVector<int, 8> NewMask;
6436 if (!widenShuffleMaskElts(Mask, NewMask))
6437 return SDValue();
6438
6439 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
6440 : MVT::getIntegerVT(ElementSize * 2);
6441 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
6442 if (!DAG.getTargetLoweringInfo().isTypeLegal(NewVT))
6443 return SDValue();
6444 V0 = DAG.getBitcast(NewVT, V0);
6445 V1 = DAG.getBitcast(NewVT, V1);
6446 return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
6447}
6448
6449// Match an interleave shuffle that forms a P-extension packed zip:
6450// <a0, b0, a1, b1, ...> -> zip*p/wzip*p
6451static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
6452 const RISCVSubtarget &Subtarget,
6453 SelectionDAG &DAG) {
6454 SDValue V1 = SVN->getOperand(0);
6455 SDValue V2 = SVN->getOperand(1);
6456 SDLoc DL(SVN);
6457 MVT VT = SVN->getSimpleValueType(0);
6458 unsigned NumElts = VT.getVectorNumElements();
6459 ArrayRef<int> Mask = SVN->getMask();
6460
6461 if (VT != MVT::v8i8 && VT != MVT::v4i16)
6462 return SDValue();
6463
6464 SmallVector<unsigned, 2> StartIndexes;
6465 if (!V2.isUndef() &&
6466 ShuffleVectorInst::isInterleaveMask(Mask, 2, NumElts * 2, StartIndexes)) {
6467 unsigned EvenSrc = StartIndexes[0];
6468 unsigned OddSrc = StartIndexes[1];
6469 if (EvenSrc == 0 && OddSrc == NumElts) {
6470 if (Subtarget.is64Bit())
6471 return DAG.getNode(RISCVISD::PZIP, DL, VT, V1, V2);
6472 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6473 V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
6474 V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
6475 return DAG.getNode(RISCVISD::PWZIP, DL, VT, V1, V2);
6476 }
6477 if (EvenSrc == NumElts && OddSrc == 0) {
6478 if (Subtarget.is64Bit())
6479 return DAG.getNode(RISCVISD::PZIP, DL, VT, V2, V1);
6480 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6481 V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
6482 V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
6483 return DAG.getNode(RISCVISD::PWZIP, DL, VT, V2, V1);
6484 }
6485 }
6486
6487 return SDValue();
6488}
6489
6490// Match a deinterleave shuffle that forms a P-extension packed unzip:
6491// <a0, a2, ..., b0, b2, ...> -> unzip*p
6492// <a1, a3, ..., b1, b3, ...> -> unzip*hp
6493static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
6494 SelectionDAG &DAG, bool IsRV64) {
6495 MVT VT = SVN->getSimpleValueType(0);
6496 if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
6497 return SDValue();
6498
6499 SDValue V1 = SVN->getOperand(0);
6500 SDValue V2 = SVN->getOperand(1);
6501 SDLoc DL(SVN);
6502 ArrayRef<int> Mask = SVN->getMask();
6503
6504 unsigned Index = 0;
6505 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index))
6506 return SDValue();
6507
6508 unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
6509 return DAG.getNode(Opc, DL, VT, V1, V2);
6510}
6511
6512// Match the packed zero-extend shuffle mask <0, N, 2, N+2, ...>: even result
6513// lanes keep operand 0's even lanes and odd result lanes come from operand 1.
6514// The odd lanes may select any element of operand 1, which is looser than a
6515// strict pair-even mask; DAGCombiner::XformToShuffleWithZero forms exactly this
6516// from a packed zero-extend `and`, leaving each zeroed lane at its own
6517// position, and lowerVECTOR_SHUFFLEAsPPair then forms the PPAIRE. Undef lanes
6518// always match.
6519static bool isPackedZExtShuffleMask(ArrayRef<int> Mask) {
6520 unsigned NumElts = Mask.size();
6521 if (NumElts % 2 != 0)
6522 return false;
6523 for (unsigned I = 0; I != NumElts / 2; ++I)
6524 if ((Mask[2 * I] >= 0 && Mask[2 * I] != (int)(2 * I)) ||
6525 (Mask[2 * I + 1] >= 0 && Mask[2 * I + 1] < (int)NumElts))
6526 return false;
6527 return true;
6528}
6529
6530// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
6531// it to an RV32 P narrowing shift on the concatenated source.
6532static SDValue
6533lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
6534 const RISCVSubtarget &Subtarget,
6535 SelectionDAG &DAG) {
6536 MVT VT = SVN->getSimpleValueType(0);
6537 if (Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
6538 return SDValue();
6539
6540 SDValue V1 = SVN->getOperand(0);
6541 SDValue V2 = SVN->getOperand(1);
6542 SDLoc DL(SVN);
6543 unsigned NumElts = VT.getVectorNumElements();
6544
6545 SDValue Src = foldConcatVector(V1, V2);
6546 if (!Src) {
6547 MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
6548 Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, V1, V2);
6549 }
6550
6551 // The source vector should be twice the size.
6552 if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
6553 return SDValue();
6554
6555 unsigned Index = 0;
6556 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(SVN->getMask(), 2, Index))
6557 return SDValue();
6558
6559 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
6560 return DAG.getNode(RISCVISD::PNSRL, DL, VT, Src,
6561 DAG.getConstant(Index * EltBits, DL, MVT::i32));
6562}
6563
6564// Match a strided-interleave shuffle that forms a P-extension packed pair:
6565// <a0, b0, a2, b2, ...> -> ppaire.*
6566// <a0, b1, a2, b3, ...> -> ppaireo.*
6567// <a1, b0, a3, b2, ...> -> ppairoe.*
6568// <a1, b1, a3, b3, ...> -> ppairo.*
6569static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
6570 SelectionDAG &DAG) {
6571 MVT VT = SVN->getSimpleValueType(0);
6572 if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v4i16)
6573 return SDValue();
6574
6575 SDValue V1 = SVN->getOperand(0);
6576 SDValue V2 = SVN->getOperand(1);
6577 SDLoc DL(SVN);
6578 unsigned NumElts = VT.getVectorNumElements();
6579 ArrayRef<int> Mask = SVN->getMask();
6580
6581 // A splat operand's lanes are all equal, so a lane selecting from it matches
6582 // any of its positions. This covers the zero operand XformToShuffleWithZero
6583 // forms for a packed zero-extend, which keeps each zeroed lane at its own
6584 // position rather than the strided one.
6585 bool V1IsSplat = DAG.isSplatValue(V1);
6586 bool V2IsSplat = DAG.isSplatValue(V2);
6587
6588 // Walk the mask once, tracking the operand feeding the destination's even
6589 // lanes (index 0) and the operand feeding its odd lanes (index 1) — either
6590 // may turn out to be V1 or V2 — along with whether each pulls the even or
6591 // odd element out of its pair. All even (resp. odd) lanes must agree on
6592 // both the operand and the parity used; a splat operand's lanes are all
6593 // equal so it never constrains the parity.
6594 SDValue Src[2];
6595 std::optional<bool> Parity[2];
6596 for (unsigned I = 0; I != NumElts; ++I) {
6597 int M = Mask[I];
6598 if (M < 0)
6599 continue;
6600 unsigned Lane = I % 2;
6601 bool FromV1 = (unsigned)M < NumElts;
6602 SDValue Cand = FromV1 ? V1 : V2;
6603 unsigned Local = (unsigned)M % NumElts;
6604 if (!Src[Lane])
6605 Src[Lane] = Cand;
6606 else if (Src[Lane] != Cand)
6607 return SDValue();
6608
6609 // Splats don't constrain parity.
6610 if (FromV1 ? V1IsSplat : V2IsSplat)
6611 continue;
6612
6613 // The index must be from the even/odd element of its pair.
6614 if (Local / 2 != I / 2)
6615 return SDValue();
6616
6617 bool P = Local % 2;
6618 if (!Parity[Lane])
6619 Parity[Lane] = P;
6620 else if (*Parity[Lane] != P)
6621 return SDValue();
6622 }
6623
6624 // Make sure we have a source for both lanes.
6625 if (!Src[0] || !Src[1])
6626 return SDValue();
6627
6628 bool EvenIsOdd = Parity[0].value_or(false);
6629 bool OddIsOdd = Parity[1].value_or(false);
6630 unsigned Opc;
6631 if (!EvenIsOdd && !OddIsOdd)
6632 Opc = RISCVISD::PPAIRE;
6633 else if (EvenIsOdd && OddIsOdd)
6634 Opc = RISCVISD::PPAIRO;
6635 else if (!EvenIsOdd && OddIsOdd)
6636 Opc = RISCVISD::PPAIREO;
6637 else
6638 Opc = RISCVISD::PPAIROE;
6639
6640 return DAG.getNode(Opc, DL, VT, Src[0], Src[1]);
6641}
6642
6643SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6644 SelectionDAG &DAG) const {
6645 SDValue V1 = Op.getOperand(0);
6646 SDValue V2 = Op.getOperand(1);
6647 SDLoc DL(Op);
6648 MVT XLenVT = Subtarget.getXLenVT();
6649 MVT VT = Op.getSimpleValueType();
6650 unsigned NumElts = VT.getVectorNumElements();
6651 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
6652
6653 // Select RVP-specific packed shuffles before falling back to the generic
6654 // fixed/scalable-vector lowering below.
6655 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
6656 ArrayRef<int> Mask = SVN->getMask();
6657
6658 // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
6659 // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
6660 // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
6661 // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
6662 auto IsLowReverse = [&](unsigned L) {
6663 return V2.isUndef() &&
6664 ShuffleVectorInst::isReverseMask(Mask.take_front(L), L) &&
6665 all_of(Mask.drop_front(L), [](int M) { return M < 0; });
6666 };
6667 if (IsLowReverse(NumElts))
6668 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6669 if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
6670 return DAG.getNode(RISCVISD::PPAIROE_H, DL, VT, V1, V1);
6671 // Widened: reversing sends the low-half lanes to the top half, so shift
6672 // them back down by half the register. Only the 64-bit packed types are
6673 // legal here, so the register is XLen (i64).
6674 if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
6675 IsLowReverse(NumElts / 2)) {
6676 SDValue Rev = DAG.getBitcast(
6677 MVT::i64, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1));
6678 SDValue Srl =
6679 DAG.getNode(ISD::SRL, DL, MVT::i64, Rev,
6680 DAG.getConstant(VT.getSizeInBits() / 2, DL, MVT::i64));
6681 return DAG.getBitcast(VT, Srl);
6682 }
6683
6684 if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, Subtarget.is64Bit()))
6685 return V;
6686 if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
6687 return V;
6688 if (SDValue V =
6689 lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
6690 return V;
6691 if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
6692 return V;
6693 return SDValue();
6694 }
6695
6696 if (VT.getVectorElementType() == MVT::i1) {
6697 // Lower to a vror.vi of a larger element type if possible before we promote
6698 // i1s to i8s.
6699 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6700 return V;
6701 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6702 return V;
6703
6704 // Promote i1 shuffle to i8 shuffle.
6705 MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
6706 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
6707 V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
6708 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
6709 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
6710 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
6711 ISD::SETNE);
6712 }
6713
6714 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6715
6716 // Store the return value in a single variable instead of structured bindings
6717 // so that we can pass it to GetSlide below, which cannot capture structured
6718 // bindings until C++20.
6719 auto TrueMaskVL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6720 auto [TrueMask, VL] = TrueMaskVL;
6721
6722 if (SVN->isSplat()) {
6723 const int Lane = SVN->getSplatIndex();
6724 if (Lane >= 0) {
6725 MVT SVT = VT.getVectorElementType();
6726
6727 // Turn splatted vector load into a strided load with an X0 stride.
6728 SDValue V = V1;
6729 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6730 // with undef.
6731 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6732 int Offset = Lane;
6733 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6734 int OpElements =
6735 V.getOperand(0).getSimpleValueType().getVectorNumElements();
6736 V = V.getOperand(Offset / OpElements);
6737 Offset %= OpElements;
6738 }
6739
6740 // We need to ensure the load isn't atomic or volatile.
6741 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
6742 auto *Ld = cast<LoadSDNode>(V);
6743 Offset *= SVT.getStoreSize();
6744 SDValue NewAddr = DAG.getMemBasePlusOffset(
6745 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
6746
6747 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6748 if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
6749 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
6750 SDValue IntID =
6751 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
6752 SDValue Ops[] = {Ld->getChain(),
6753 IntID,
6754 DAG.getUNDEF(ContainerVT),
6755 NewAddr,
6756 DAG.getRegister(RISCV::X0, XLenVT),
6757 VL};
6758 SDValue NewLoad = DAG.getMemIntrinsicNode(
6759 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
6760 DAG.getMachineFunction().getMachineMemOperand(
6761 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
6762 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
6763 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
6764 }
6765
6766 MVT SplatVT = ContainerVT;
6767
6768 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6769 if (SVT == MVT::bf16 ||
6770 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6771 SVT = MVT::i16;
6772 SplatVT = ContainerVT.changeVectorElementType(SVT);
6773 }
6774
6775 // Otherwise use a scalar load and splat. This will give the best
6776 // opportunity to fold a splat into the operation. ISel can turn it into
6777 // the x0 strided load if we aren't able to fold away the select.
6778 if (SVT.isFloatingPoint())
6779 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
6780 Ld->getPointerInfo().getWithOffset(Offset),
6781 Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
6782 else
6783 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
6784 Ld->getPointerInfo().getWithOffset(Offset), SVT,
6785 Ld->getBaseAlign(),
6786 Ld->getMemOperand()->getFlags());
6787 DAG.makeEquivalentMemoryOrdering(Ld, V);
6788
6789 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6790 : RISCVISD::VMV_V_X_VL;
6791 SDValue Splat =
6792 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
6793 Splat = DAG.getBitcast(ContainerVT, Splat);
6794 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
6795 }
6796
6797 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6798 assert(Lane < (int)NumElts && "Unexpected lane!");
6799 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
6800 V1, DAG.getConstant(Lane, DL, XLenVT),
6801 DAG.getUNDEF(ContainerVT), TrueMask, VL);
6802 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6803 }
6804 }
6805
6806 // For exact VLEN m2 or greater, try to split to m1 operations if we
6807 // can split cleanly.
6808 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6809 return V;
6810
6811 ArrayRef<int> Mask = SVN->getMask();
6812
6813 if (SDValue V =
6814 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6815 return V;
6816
6817 if (SDValue V =
6818 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6819 return V;
6820
6821 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6822 // available.
6823 if (Subtarget.hasStdExtZvkb())
6824 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6825 return V;
6826
6827 if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef() &&
6828 NumElts != 2)
6829 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6830
6831 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6832 // use shift and truncate to perform the shuffle.
6833 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6834 // shift-and-trunc reducing total cost for everything except an mf8 result.
6835 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6836 // to do the entire operation.
6837 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6838 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6839 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6840 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6841 unsigned Index = 0;
6842 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6843 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6844 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6845 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6846 if (1 < count_if(Mask,
6847 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6848 1 < count_if(Mask, [&Mask](int Idx) {
6849 return Idx >= (int)Mask.size();
6850 })) {
6851 // Narrow each source and concatenate them.
6852 // FIXME: For small LMUL it is better to concatenate first.
6853 MVT EltVT = VT.getVectorElementType();
6854 auto EltCnt = VT.getVectorElementCount();
6855 MVT SubVT =
6856 MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor));
6857
6858 SDValue Lo =
6859 getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG);
6860 SDValue Hi =
6861 getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG);
6862
6863 SDValue Concat =
6864 DAG.getNode(ISD::CONCAT_VECTORS, DL,
6865 SubVT.getDoubleNumVectorElementsVT(), Lo, Hi);
6866 if (Factor == 2)
6867 return Concat;
6868
6869 SDValue Vec = DAG.getUNDEF(VT);
6870 return DAG.getInsertSubvector(DL, Vec, Concat, 0);
6871 }
6872 }
6873 }
6874 }
6875
6876 // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
6877 // e64 which can't match above.
6878 unsigned Index = 0;
6879 if (Subtarget.hasStdExtZvzip() &&
6880 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
6881 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6882 bool UsesBothSources =
6883 1 < count_if(Mask,
6884 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6885 1 < count_if(Mask,
6886 [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
6887
6888 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
6889 unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
6890 MVT NewVT = VT.getDoubleNumVectorElementsVT();
6891 if (isTypeLegal(NewVT)) {
6892 SDValue Op;
6893 if (V2.isUndef()) {
6894 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6895 } else if (auto VLEN = Subtarget.getRealVLen();
6896 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
6897 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6898 } else if (SDValue Src = foldConcatVector(V1, V2)) {
6899 Op = DAG.getExtractSubvector(DL, NewVT, Src, 0);
6900 }
6901 if (Op)
6902 return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
6903 }
6904
6905 if (UsesBothSources &&
6906 V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
6907 V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
6908 SDValue Lo = lowerZvzipVUNZIP(Opc, V1, DL, DAG, Subtarget);
6909 SDValue Hi = lowerZvzipVUNZIP(Opc, V2, DL, DAG, Subtarget);
6910 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6911 }
6912 }
6913 }
6914
6915 if (SDValue V =
6916 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6917 return V;
6918
6919 // Detect an interleave shuffle and lower to
6920 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6921 int EvenSrc, OddSrc;
6922 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6923 !(NumElts == 2 &&
6924 ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))) {
6925 // Extract the halves of the vectors.
6926 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6927
6928 // Recognize if one half is actually undef; the matching above will
6929 // otherwise reuse the even stream for the undef one. This improves
6930 // spread(2) shuffles.
6931 bool LaneIsUndef[2] = { true, true};
6932 for (const auto &[Idx, M] : enumerate(Mask))
6933 LaneIsUndef[Idx % 2] &= (M == -1);
6934
6935 int Size = Mask.size();
6936 SDValue EvenV, OddV;
6937 if (LaneIsUndef[0]) {
6938 EvenV = DAG.getUNDEF(HalfVT);
6939 } else {
6940 assert(EvenSrc >= 0 && "Undef source?");
6941 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6942 EvenV = DAG.getExtractSubvector(DL, HalfVT, EvenV, EvenSrc % Size);
6943 }
6944
6945 if (LaneIsUndef[1]) {
6946 OddV = DAG.getUNDEF(HalfVT);
6947 } else {
6948 assert(OddSrc >= 0 && "Undef source?");
6949 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6950 OddV = DAG.getExtractSubvector(DL, HalfVT, OddV, OddSrc % Size);
6951 }
6952
6953 // Prefer vzip if available.
6954 // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
6955 if (Subtarget.hasStdExtZvzip() && isLegalVTForZvzipOperand(VT, Subtarget))
6956 return lowerZvzipVZIP(EvenV, OddV, DL, DAG, Subtarget);
6957 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6958 }
6959
6960 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6961 // instructions (in any combination) with masking on the second instruction.
6962 // Also handles masked slides into an identity source, and single slides
6963 // without masking. Avoid matching bit rotates (which are not also element
6964 // rotates) as slide pairs. This is a performance heuristic, not a
6965 // functional check.
6966 std::array<std::pair<int, int>, 2> SrcInfo;
6967 unsigned RotateAmt;
6968 MVT RotateVT;
6969 if (::isMaskedSlidePair(Mask, SrcInfo) &&
6970 (isElementRotate(SrcInfo, NumElts) ||
6971 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
6972 SDValue Sources[2];
6973 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
6974 int SrcIdx = Info.first;
6975 assert(SrcIdx == 0 || SrcIdx == 1);
6976 SDValue &Src = Sources[SrcIdx];
6977 if (!Src) {
6978 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
6979 Src = convertToScalableVector(ContainerVT, SrcV, DAG, Subtarget);
6980 }
6981 return Src;
6982 };
6983 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
6984 SDValue Passthru) {
6985 auto [TrueMask, VL] = TrueMaskVL;
6986 SDValue SrcV = GetSourceFor(Src);
6987 int SlideAmt = Src.second;
6988 if (SlideAmt == 0) {
6989 // Should never be second operation
6990 assert(Mask == TrueMask);
6991 return SrcV;
6992 }
6993 if (SlideAmt < 0)
6994 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6995 DAG.getConstant(-SlideAmt, DL, XLenVT), Mask, VL,
6996 RISCVVType::TAIL_AGNOSTIC);
6997 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6998 DAG.getConstant(SlideAmt, DL, XLenVT), Mask, VL,
6999 RISCVVType::TAIL_AGNOSTIC);
7000 };
7001
7002 if (SrcInfo[1].first == -1) {
7003 SDValue Res = DAG.getUNDEF(ContainerVT);
7004 Res = GetSlide(SrcInfo[0], TrueMask, Res);
7005 return convertFromScalableVector(VT, Res, DAG, Subtarget);
7006 }
7007
7008 if (Subtarget.hasStdExtZvzip()) {
7009 bool TryWiden = false;
7010 unsigned Factor;
7011 if (isPairEven(SrcInfo, Mask, Factor)) {
7012 if (Factor == 1) {
7013 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
7014 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
7015 return lowerZvzipVPAIR(RISCVISD::VPAIRE_VL, Src1, Src2, DL, DAG,
7016 Subtarget);
7017 }
7018 TryWiden = true;
7019 }
7020 if (isPairOdd(SrcInfo, Mask, Factor)) {
7021 if (Factor == 1) {
7022 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
7023 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
7024 return lowerZvzipVPAIR(RISCVISD::VPAIRO_VL, Src1, Src2, DL, DAG,
7025 Subtarget);
7026 }
7027 TryWiden = true;
7028 }
7029 // If we found a widening oppurtunity which would let us form a
7030 // pair-even or pair-odd, use the generic code to widen the shuffle
7031 // and recurse through this logic.
7032 if (TryWiden)
7033 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7034 return V;
7035 }
7036
7037 // Build the mask. Note that vslideup unconditionally preserves elements
7038 // below the slide amount in the destination, and thus those elements are
7039 // undefined in the mask. If the mask ends up all true (or undef), it
7040 // will be folded away by general logic.
7041 SmallVector<SDValue> MaskVals;
7042 for (const auto &[Idx, M] : enumerate(Mask)) {
7043 if (M < 0 ||
7044 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
7045 MaskVals.push_back(DAG.getUNDEF(XLenVT));
7046 continue;
7047 }
7048 int Src = M >= (int)NumElts;
7049 int Diff = (int)Idx - (M % NumElts);
7050 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
7051 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
7052 "Must match exactly one of the two slides");
7053 MaskVals.push_back(DAG.getConstant(C, DL, XLenVT));
7054 }
7055 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7056 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7057 SDValue SelectMask = convertToScalableVector(
7058 ContainerVT.changeVectorElementType(MVT::i1),
7059 DAG.getBuildVector(MaskVT, DL, MaskVals), DAG, Subtarget);
7060
7061 SDValue Res = DAG.getUNDEF(ContainerVT);
7062 Res = GetSlide(SrcInfo[0], TrueMask, Res);
7063 Res = GetSlide(SrcInfo[1], SelectMask, Res);
7064 return convertFromScalableVector(VT, Res, DAG, Subtarget);
7065 }
7066
7067 // Handle any remaining single source shuffles
7068 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
7069 if (V2.isUndef()) {
7070 // We might be able to express the shuffle as a bitrotate. But even if we
7071 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
7072 // shifts and a vor will have a higher throughput than a vrgather.
7073 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
7074 return V;
7075
7076 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
7077 return V;
7078
7079 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
7080 // is fully covered in interleave(2) above, so it is ignored here.
7081 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
7082 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
7083 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
7084 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
7085 unsigned Index;
7086 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
7087 MVT NarrowVT =
7088 MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
7089 SDValue Src = DAG.getExtractSubvector(DL, NarrowVT, V1, 0);
7090 return getWideningSpread(Src, Factor, Index, DL, DAG);
7091 }
7092 }
7093 }
7094
7095 // If only a prefix of the source elements influence a prefix of the
7096 // destination elements, try to see if we can reduce the required LMUL
7097 unsigned MinVLen = Subtarget.getRealMinVLen();
7098 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
7099 if (NumElts > MinVLMAX) {
7100 unsigned MaxIdx = 0;
7101 for (auto [I, M] : enumerate(Mask)) {
7102 if (M == -1)
7103 continue;
7104 MaxIdx = std::max({(unsigned)I, (unsigned)M, MaxIdx});
7105 }
7106 unsigned NewNumElts =
7107 std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
7108 if (NewNumElts != NumElts) {
7109 MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
7110 V1 = DAG.getExtractSubvector(DL, NewVT, V1, 0);
7111 SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
7112 Mask.take_front(NewNumElts));
7113 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
7114 }
7115 }
7116
7117 // Before hitting generic lowering fallbacks, try to widen the mask
7118 // to a wider SEW.
7119 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7120 return V;
7121
7122 // Can we generate a vcompress instead of a vrgather? These scale better
7123 // at high LMUL, at the cost of not being able to fold a following select
7124 // into them. The mask constants are also smaller than the index vector
7125 // constants, and thus easier to materialize.
7126 if (isCompressMask(Mask)) {
7127 SmallVector<SDValue> MaskVals(NumElts,
7128 DAG.getConstant(false, DL, XLenVT));
7129 for (auto Idx : Mask) {
7130 if (Idx == -1)
7131 break;
7132 assert(Idx >= 0 && (unsigned)Idx < NumElts);
7133 MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
7134 }
7135 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7136 SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
7137 return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
7138 DAG.getUNDEF(VT));
7139 }
7140
7141 if (VT.getScalarSizeInBits() == 8 &&
7142 any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
7143 // On such a vector we're unable to use i8 as the index type.
7144 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
7145 // may involve vector splitting if we're already at LMUL=8, or our
7146 // user-supplied maximum fixed-length LMUL.
7147 return SDValue();
7148 }
7149
7150 // Base case for the two operand recursion below - handle the worst case
7151 // single source shuffle.
7152 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
7153 MVT IndexVT = VT.changeTypeToInteger();
7154 // Since we can't introduce illegal index types at this stage, use i16 and
7155 // vrgatherei16 if the corresponding index type for plain vrgather is greater
7156 // than XLenVT.
7157 if (IndexVT.getScalarType().bitsGT(XLenVT)) {
7158 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7159 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
7160 }
7161
7162 // If the mask allows, we can do all the index computation in 16 bits. This
7163 // requires less work and less register pressure at high LMUL, and creates
7164 // smaller constants which may be cheaper to materialize.
7165 if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
7166 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
7167 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7168 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
7169 }
7170
7171 MVT IndexContainerVT =
7172 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
7173
7174 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
7175 SmallVector<SDValue> GatherIndicesLHS;
7176 for (int MaskIndex : Mask) {
7177 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
7178 GatherIndicesLHS.push_back(IsLHSIndex
7179 ? DAG.getConstant(MaskIndex, DL, XLenVT)
7180 : DAG.getUNDEF(XLenVT));
7181 }
7182 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
7183 LHSIndices =
7184 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
7185 // At m1 and less, there's no point trying any of the high LMUL splitting
7186 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
7187 if (NumElts <= MinVLMAX) {
7188 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
7189 DAG.getUNDEF(ContainerVT), TrueMask, VL);
7190 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7191 }
7192
7193 const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
7194 EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
7195 auto [InnerTrueMask, InnerVL] =
7196 getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
7197 int N =
7198 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
7199 assert(isPowerOf2_32(N) && N <= 8);
7200
7201 // If we have a locally repeating mask, then we can reuse the first
7202 // register in the index register group for all registers within the
7203 // source register group. TODO: This generalizes to m2, and m4.
7204 if (isLocalRepeatingShuffle(Mask, MinVLMAX)) {
7205 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7206 SDValue Gather = DAG.getUNDEF(ContainerVT);
7207 for (int i = 0; i < N; i++) {
7208 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
7209 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, SubIdx);
7210 SDValue SubVec =
7211 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7212 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7213 Gather = DAG.getInsertSubvector(DL, Gather, SubVec, SubIdx);
7214 }
7215 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7216 }
7217
7218 // If we have a shuffle which only uses the first register in our source
7219 // register group, and repeats the same index across all spans, we can
7220 // use a single vrgather (and possibly some register moves).
7221 // TODO: This can be generalized for m2 or m4, or for any shuffle for
7222 // which we can do a linear number of shuffles to form an m1 which
7223 // contains all the output elements.
7224 if (isLowSourceShuffle(Mask, MinVLMAX) &&
7225 isSpanSplatShuffle(Mask, MinVLMAX)) {
7226 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
7227 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7228 SDValue SubVec = DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7229 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7230 SDValue Gather = DAG.getUNDEF(ContainerVT);
7231 for (int i = 0; i < N; i++)
7232 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
7233 M1VT.getVectorMinNumElements() * i);
7234 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7235 }
7236
7237 // If we have a shuffle which only uses the first register in our
7238 // source register group, we can do a linear number of m1 vrgathers
7239 // reusing the same source register (but with different indices)
7240 // TODO: This can be generalized for m2 or m4, or for any shuffle
7241 // for which we can do a vslidedown followed by this expansion.
7242 if (isLowSourceShuffle(Mask, MinVLMAX)) {
7243 SDValue SlideAmt =
7244 DAG.getElementCount(DL, XLenVT, M1VT.getVectorElementCount());
7245 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
7246 SDValue Gather = DAG.getUNDEF(ContainerVT);
7247 for (int i = 0; i < N; i++) {
7248 if (i != 0)
7249 LHSIndices = getVSlidedown(DAG, Subtarget, DL, IndexContainerVT,
7250 DAG.getUNDEF(IndexContainerVT), LHSIndices,
7251 SlideAmt, TrueMask, VL);
7252 SDValue SubIndex =
7253 DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
7254 SDValue SubVec =
7255 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
7256 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
7257 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
7258 M1VT.getVectorMinNumElements() * i);
7259 }
7260 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7261 }
7262
7263 // Fallback to generic vrgather if we can't find anything better.
7264 // On many machines, this will be O(LMUL^2)
7265 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
7266 DAG.getUNDEF(ContainerVT), TrueMask, VL);
7267 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
7268 }
7269
7270 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
7271 // merged with a second vrgather.
7272 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
7273
7274 // Now construct the mask that will be used by the blended vrgather operation.
7275 // Construct the appropriate indices into each vector.
7276 for (int MaskIndex : Mask) {
7277 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
7278 ShuffleMaskLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
7279 ? MaskIndex : -1);
7280 ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
7281 }
7282
7283 // If the mask indices are disjoint between the two sources, we can lower it
7284 // as a vselect + a single source vrgather.vv. Don't do this if we think the
7285 // operands may end up being lowered to something cheaper than a vrgather.vv.
7286 if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
7287 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
7288 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
7289 !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
7290 !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
7291 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
7292 return V;
7293
7294 // Before hitting generic lowering fallbacks, try to widen the mask
7295 // to a wider SEW.
7296 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7297 return V;
7298
7299 // Try to pick a profitable operand order.
7300 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
7301 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts);
7302
7303 // Recursively invoke lowering for each operand if we had two
7304 // independent single source shuffles, and then combine the result via a
7305 // vselect. Note that the vselect will likely be folded back into the
7306 // second permute (vrgather, or other) by the post-isel combine.
7307 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
7308 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
7309
7310 SmallVector<SDValue> MaskVals;
7311 for (int MaskIndex : Mask) {
7312 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
7313 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
7314 }
7315
7316 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7317 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7318 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
7319
7320 if (SwapOps)
7321 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
7322 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
7323}
7324
7325bool RISCVTargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
7326 EVT VT) const {
7327 // Enable DAGCombiner::XformToShuffleWithZero to rewrite a packed zero-extend
7328 // `and` into shuffle(src, zero, ...), which lowerVECTOR_SHUFFLE turns into
7329 // RISCVISD::PPAIRE. Accept the packed byte/halfword views that lowering
7330 // handles; the 32-bit views are illegal on RV64 but reachable before type
7331 // legalization widens them to the legal 64-bit view.
7332 if (!Subtarget.hasStdExtP() || !VT.isSimple())
7333 return false;
7334 MVT SVT = VT.getSimpleVT();
7335 return (SVT == MVT::v4i8 || SVT == MVT::v8i8 || SVT == MVT::v4i16) &&
7336 isPackedZExtShuffleMask(M);
7337}
7338
7339bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7340 // Only support legal VTs for other shuffles for now.
7341 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
7342 return false;
7343
7344 // Support splats for any type. These should type legalize well.
7345 if (ShuffleVectorSDNode::isSplatMask(M))
7346 return true;
7347
7348 const unsigned NumElts = M.size();
7349 MVT SVT = VT.getSimpleVT();
7350
7351 // Not for i1 vectors.
7352 if (SVT.getScalarType() == MVT::i1)
7353 return false;
7354
7355 std::array<std::pair<int, int>, 2> SrcInfo;
7356 int Dummy1, Dummy2;
7357 return ShuffleVectorInst::isReverseMask(M, NumElts) ||
7358 (::isMaskedSlidePair(M, SrcInfo) &&
7359 isElementRotate(SrcInfo, NumElts)) ||
7360 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
7361}
7362
7363// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
7364// the exponent.
7365SDValue
7366RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
7367 SelectionDAG &DAG) const {
7368 MVT VT = Op.getSimpleValueType();
7369 unsigned EltSize = VT.getScalarSizeInBits();
7370 SDValue Src = Op.getOperand(0);
7371 SDLoc DL(Op);
7372 MVT ContainerVT = VT;
7373
7374 // We choose FP type that can represent the value if possible. Otherwise, we
7375 // use rounding to zero conversion for correct exponent of the result.
7376 // TODO: Use f16 for i8 when possible?
7377 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
7378 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
7379 FloatEltVT = MVT::f32;
7380 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
7381
7382 // Legal types should have been checked in the RISCVTargetLowering
7383 // constructor.
7384 // TODO: Splitting may make sense in some cases.
7385 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
7386 "Expected legal float type!");
7387
7388 // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
7389 // The trailing zero count is equal to log2 of this single bit value.
7390 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
7391 SDValue Neg = DAG.getNegative(Src, DL, VT);
7392 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
7393 }
7394
7395 // We have a legal FP type, convert to it.
7396 SDValue FloatVal;
7397 if (FloatVT.bitsGT(VT)) {
7398 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
7399 } else {
7400 // Use RTZ to avoid rounding influencing exponent of FloatVal.
7401 if (VT.isFixedLengthVector()) {
7402 ContainerVT = getContainerForFixedLengthVector(VT);
7403 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
7404 }
7405 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7406 SDValue RTZRM =
7407 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
7408 MVT ContainerFloatVT =
7409 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
7410 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
7411 Src, Mask, RTZRM, VL);
7412 if (VT.isFixedLengthVector())
7413 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
7414 }
7415 // Bitcast to integer and shift the exponent to the LSB.
7416 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
7417 SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
7418 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
7419
7420 // Restore back to original type. Truncation after SRL is to generate vnsrl.
7421 SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
7422 DAG.getConstant(ShiftAmt, DL, IntVT));
7423 if (IntVT.bitsLT(VT))
7424 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
7425 else if (IntVT.bitsGT(VT))
7426 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
7427
7428 // The exponent contains log2 of the value in biased form.
7429 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
7430 // For trailing zeros, we just need to subtract the bias.
7431 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
7432 return DAG.getNode(ISD::SUB, DL, VT, Exp,
7433 DAG.getConstant(ExponentBias, DL, VT));
7434
7435 // For leading zeros, we need to remove the bias and convert from log2 to
7436 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
7437 unsigned Adjust = ExponentBias + (EltSize - 1);
7438 SDValue Res =
7439 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
7440
7441 // The above result with zero input equals to Adjust which is greater than
7442 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
7443 if (Op.getOpcode() == ISD::CTLZ)
7444 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
7445
7446 return Res;
7447}
7448
7449SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7450 SelectionDAG &DAG) const {
7451 SDLoc DL(Op);
7452 MVT XLenVT = Subtarget.getXLenVT();
7453 SDValue Source = Op->getOperand(0);
7454 MVT SrcVT = Source.getSimpleValueType();
7455 SDValue Mask = Op->getOperand(1);
7456 SDValue EVL = Op->getOperand(2);
7457
7458 if (SrcVT.isFixedLengthVector()) {
7459 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
7460 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
7461 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
7462 Subtarget);
7463 SrcVT = ContainerVT;
7464 }
7465
7466 // Convert to boolean vector.
7467 if (SrcVT.getScalarType() != MVT::i1) {
7468 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
7469 SrcVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorElementCount());
7470 Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
7471 {Source, AllZero, DAG.getCondCode(ISD::SETNE),
7472 DAG.getUNDEF(SrcVT), Mask, EVL});
7473 }
7474
7475 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
7476 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
7477 // In this case, we can interpret poison as -1, so nothing to do further.
7478 return Res;
7479
7480 // Convert -1 to VL.
7481 SDValue SetCC =
7482 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
7483 Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
7484 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
7485}
7486
7487// While RVV has alignment restrictions, we should always be able to load as a
7488// legal equivalently-sized byte-typed vector instead. This method is
7489// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7490// the load is already correctly-aligned, it returns SDValue().
7491SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7492 SelectionDAG &DAG) const {
7493 auto *Load = cast<LoadSDNode>(Op);
7494 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7495
7496 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7497 Load->getMemoryVT(),
7498 *Load->getMemOperand()))
7499 return SDValue();
7500
7501 SDLoc DL(Op);
7502 MVT VT = Op.getSimpleValueType();
7503 unsigned EltSizeBits = VT.getScalarSizeInBits();
7504 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7505 "Unexpected unaligned RVV load type");
7506 MVT NewVT =
7507 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7508 assert(NewVT.isValid() &&
7509 "Expecting equally-sized RVV vector types to be legal");
7510 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7511 Load->getPointerInfo(), Load->getBaseAlign(),
7512 Load->getMemOperand()->getFlags());
7513 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7514}
7515
7516// While RVV has alignment restrictions, we should always be able to store as a
7517// legal equivalently-sized byte-typed vector instead. This method is
7518// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7519// returns SDValue() if the store is already correctly aligned.
7520SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7521 SelectionDAG &DAG) const {
7522 auto *Store = cast<StoreSDNode>(Op);
7523 assert(Store && Store->getValue().getValueType().isVector() &&
7524 "Expected vector store");
7525
7526 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7527 Store->getMemoryVT(),
7528 *Store->getMemOperand()))
7529 return SDValue();
7530
7531 SDLoc DL(Op);
7532 SDValue StoredVal = Store->getValue();
7533 MVT VT = StoredVal.getSimpleValueType();
7534 unsigned EltSizeBits = VT.getScalarSizeInBits();
7535 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7536 "Unexpected unaligned RVV store type");
7537 MVT NewVT =
7538 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7539 assert(NewVT.isValid() &&
7540 "Expecting equally-sized RVV vector types to be legal");
7541 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7542 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7543 Store->getPointerInfo(), Store->getBaseAlign(),
7544 Store->getMemOperand()->getFlags());
7545}
7546
7547// While RVV has alignment restrictions, we should always be able to load as a
7548// legal equivalently-sized byte-typed vector instead. This method is
7549// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7550// the load is already correctly-aligned, it returns SDValue().
7551SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7552 SelectionDAG &DAG) const {
7553 auto *Load = cast<VPLoadSDNode>(Op);
7554 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7555
7556 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7557 Load->getMemoryVT(),
7558 *Load->getMemOperand()))
7559 return SDValue();
7560
7561 SDValue Mask = Load->getMask();
7562
7563 // FIXME: Handled masked loads somehow.
7564 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7565 return SDValue();
7566
7567 SDLoc DL(Op);
7568 MVT VT = Op.getSimpleValueType();
7569 unsigned EltSizeBits = VT.getScalarSizeInBits();
7570 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7571 "Unexpected unaligned RVV load type");
7572 MVT NewVT =
7573 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7574 assert(NewVT.isValid() &&
7575 "Expecting equally-sized RVV vector types to be legal");
7576
7577 SDValue VL = Load->getVectorLength();
7578 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7579 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7580
7581 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7582 SDValue L = DAG.getLoadVP(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7583 DAG.getAllOnesConstant(DL, MaskVT), VL,
7584 Load->getPointerInfo(), Load->getBaseAlign(),
7585 Load->getMemOperand()->getFlags(), AAMDNodes());
7586 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7587}
7588
7589// While RVV has alignment restrictions, we should always be able to store as a
7590// legal equivalently-sized byte-typed vector instead. This method is
7591// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7592// It returns SDValue() if the store is already correctly aligned.
7593SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7594 SelectionDAG &DAG) const {
7595 auto *Store = cast<VPStoreSDNode>(Op);
7596 assert(Store && Store->getValue().getValueType().isVector() &&
7597 "Expected vector store");
7598
7599 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7600 Store->getMemoryVT(),
7601 *Store->getMemOperand()))
7602 return SDValue();
7603
7604 SDValue Mask = Store->getMask();
7605
7606 // FIXME: Handled masked stores somehow.
7607 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7608 return SDValue();
7609
7610 SDLoc DL(Op);
7611 SDValue StoredVal = Store->getValue();
7612 MVT VT = StoredVal.getSimpleValueType();
7613 unsigned EltSizeBits = VT.getScalarSizeInBits();
7614 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7615 "Unexpected unaligned RVV store type");
7616 MVT NewVT =
7617 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7618 assert(NewVT.isValid() &&
7619 "Expecting equally-sized RVV vector types to be legal");
7620
7621 SDValue VL = Store->getVectorLength();
7622 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7623 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7624
7625 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7626
7627 LocationSize Size = LocationSize::precise(NewVT.getStoreSize());
7628 MachineFunction &MF = DAG.getMachineFunction();
7629 MachineMemOperand *MMO = MF.getMachineMemOperand(
7630 Store->getPointerInfo(), Store->getMemOperand()->getFlags(), Size,
7631 Store->getBaseAlign());
7632
7633 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7634 return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7635 DAG.getPOISON(Store->getBasePtr().getValueType()),
7636 DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
7637 ISD::UNINDEXED);
7638}
7639
7640static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7641 const RISCVSubtarget &Subtarget) {
7642 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7643
7644 int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
7645
7646 // All simm32 constants should be handled by isel.
7647 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7648 // this check redundant, but small immediates are common so this check
7649 // should have better compile time.
7650 if (isInt<32>(Imm))
7651 return Op;
7652
7653 // We only need to cost the immediate, if constant pool lowering is enabled.
7654 if (!Subtarget.useConstantPoolForLargeInts())
7655 return Op;
7656
7657 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget);
7658 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7659 return Op;
7660
7661 // Optimizations below are disabled for opt size. If we're optimizing for
7662 // size, use a constant pool.
7663 if (DAG.shouldOptForSize())
7664 return SDValue();
7665
7666 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7667 // that if it will avoid a constant pool.
7668 // It will require an extra temporary register though.
7669 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7670 // low and high 32 bits are the same and bit 31 and 63 are set.
7671 unsigned ShiftAmt, AddOpc;
7672 RISCVMatInt::InstSeq SeqLo =
7673 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
7674 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7675 return Op;
7676
7677 return SDValue();
7678}
7679
7680SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7681 SelectionDAG &DAG) const {
7682 MVT VT = Op.getSimpleValueType();
7683 const APFloat &Imm = cast<ConstantFPSDNode>(Op)->getValueAPF();
7684
7685 // Can this constant be selected by a Zfa FLI instruction?
7686 bool Negate = false;
7687 int Index = getLegalZfaFPImm(Imm, VT);
7688
7689 // If the constant is negative, try negating.
7690 if (Index < 0 && Imm.isNegative()) {
7691 Index = getLegalZfaFPImm(-Imm, VT);
7692 Negate = true;
7693 }
7694
7695 // If we couldn't find a FLI lowering, fall back to generic code.
7696 if (Index < 0)
7697 return SDValue();
7698
7699 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7700 SDLoc DL(Op);
7701 SDValue Const =
7702 DAG.getNode(RISCVISD::FLI, DL, VT,
7703 DAG.getTargetConstant(Index, DL, Subtarget.getXLenVT()));
7704 if (!Negate)
7705 return Const;
7706
7707 return DAG.getNode(ISD::FNEG, DL, VT, Const);
7708}
7709
7710static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7711 SelectionDAG &DAG) {
7712
7713 unsigned IsData = Op.getConstantOperandVal(4);
7714
7715 // mips-p8700 we support data prefetch for now.
7716 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7717 return Op.getOperand(0);
7718 return Op;
7719}
7720
7721static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7722 const RISCVSubtarget &Subtarget) {
7723 SDLoc dl(Op);
7724 AtomicOrdering FenceOrdering =
7725 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
7726 SyncScope::ID FenceSSID =
7727 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
7728
7729 if (Subtarget.hasStdExtZtso()) {
7730 // The only fence that needs an instruction is a sequentially-consistent
7731 // cross-thread fence.
7732 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7733 FenceSSID == SyncScope::System)
7734 return Op;
7735
7736 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7737 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7738 }
7739
7740 // singlethread fences only synchronize with signal handlers on the same
7741 // thread and thus only need to preserve instruction order, not actually
7742 // enforce memory ordering.
7743 if (FenceSSID == SyncScope::SingleThread)
7744 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7745 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7746
7747 return Op;
7748}
7749
7750SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7751 SelectionDAG &DAG) const {
7752 SDLoc DL(Op);
7753 MVT VT = Op.getSimpleValueType();
7754 MVT XLenVT = Subtarget.getXLenVT();
7755 unsigned Check = Op.getConstantOperandVal(1);
7756 unsigned TDCMask = 0;
7757 if (Check & fcSNan)
7758 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7759 if (Check & fcQNan)
7760 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7761 if (Check & fcPosInf)
7762 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7763 if (Check & fcNegInf)
7764 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7765 if (Check & fcPosNormal)
7766 TDCMask |= RISCV::FPMASK_Positive_Normal;
7767 if (Check & fcNegNormal)
7768 TDCMask |= RISCV::FPMASK_Negative_Normal;
7769 if (Check & fcPosSubnormal)
7770 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7771 if (Check & fcNegSubnormal)
7772 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7773 if (Check & fcPosZero)
7774 TDCMask |= RISCV::FPMASK_Positive_Zero;
7775 if (Check & fcNegZero)
7776 TDCMask |= RISCV::FPMASK_Negative_Zero;
7777
7778 bool IsOneBitMask = isPowerOf2_32(TDCMask);
7779
7780 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
7781
7782 if (VT.isVector()) {
7783 SDValue Op0 = Op.getOperand(0);
7784 MVT VT0 = Op.getOperand(0).getSimpleValueType();
7785
7786 if (VT.isScalableVector()) {
7787 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7788 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
7789 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
7790 VL, Op->getFlags());
7791 if (IsOneBitMask)
7792 return DAG.getSetCC(DL, VT, FPCLASS,
7793 DAG.getConstant(TDCMask, DL, DstVT),
7794 ISD::CondCode::SETEQ);
7795 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
7796 DAG.getConstant(TDCMask, DL, DstVT));
7797 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
7798 ISD::SETNE);
7799 }
7800
7801 MVT ContainerVT0 = getContainerForFixedLengthVector(VT0);
7802 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7803 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7804 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
7805 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
7806
7807 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
7808 Mask, VL, Op->getFlags());
7809
7810 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7811 DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
7812 if (IsOneBitMask) {
7813 SDValue VMSEQ =
7814 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7815 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
7816 DAG.getUNDEF(ContainerVT), Mask, VL});
7817 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
7818 }
7819 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
7820 TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
7821
7822 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
7823 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7824 DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
7825
7826 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7827 {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
7828 DAG.getUNDEF(ContainerVT), Mask, VL});
7829 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
7830 }
7831
7832 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
7833 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
7834 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
7835 ISD::CondCode::SETNE);
7836 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
7837}
7838
7839// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7840// operations propagate nans.
7841static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7842 const RISCVSubtarget &Subtarget) {
7843 SDLoc DL(Op);
7844 MVT VT = Op.getSimpleValueType();
7845
7846 SDValue X = Op.getOperand(0);
7847 SDValue Y = Op.getOperand(1);
7848
7849 if (!VT.isVector()) {
7850 MVT XLenVT = Subtarget.getXLenVT();
7851
7852 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7853 // ensures that when one input is a nan, the other will also be a nan
7854 // allowing the nan to propagate. If both inputs are nan, this will swap the
7855 // inputs which is harmless.
7856
7857 SDValue NewY = Y;
7858 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
7859 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
7860 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
7861 }
7862
7863 SDValue NewX = X;
7864 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
7865 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
7866 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
7867 }
7868
7869 unsigned Opc =
7870 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7871 return DAG.getNode(Opc, DL, VT, NewX, NewY);
7872 }
7873
7874 // Check no NaNs before converting to fixed vector scalable.
7875 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
7876 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
7877
7878 MVT ContainerVT = VT;
7879 if (VT.isFixedLengthVector()) {
7880 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
7881 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
7882 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
7883 }
7884
7885 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7886
7887 SDValue NewY = Y;
7888 if (!XIsNeverNan) {
7889 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7890 {X, X, DAG.getCondCode(ISD::SETOEQ),
7891 DAG.getUNDEF(ContainerVT), Mask, VL});
7892 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
7893 DAG.getUNDEF(ContainerVT), VL);
7894 }
7895
7896 SDValue NewX = X;
7897 if (!YIsNeverNan) {
7898 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7899 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
7900 DAG.getUNDEF(ContainerVT), Mask, VL});
7901 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
7902 DAG.getUNDEF(ContainerVT), VL);
7903 }
7904
7905 unsigned Opc =
7906 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
7907 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
7908 DAG.getUNDEF(ContainerVT), Mask, VL);
7909 if (VT.isFixedLengthVector())
7910 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
7911 return Res;
7912}
7913
7914static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7915 const RISCVSubtarget &Subtarget) {
7916 bool IsFABS = Op.getOpcode() == ISD::FABS;
7917 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7918 "Wrong opcode for lowering FABS or FNEG.");
7919
7920 MVT XLenVT = Subtarget.getXLenVT();
7921 MVT VT = Op.getSimpleValueType();
7922 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7923
7924 SDLoc DL(Op);
7925 SDValue Fmv =
7926 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op.getOperand(0));
7927
7928 APInt Mask = IsFABS ? APInt::getSignedMaxValue(16) : APInt::getSignMask(16);
7929 Mask = Mask.sext(Subtarget.getXLen());
7930
7931 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7932 SDValue Logic =
7933 DAG.getNode(LogicOpc, DL, XLenVT, Fmv, DAG.getConstant(Mask, DL, XLenVT));
7934 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
7935}
7936
7937static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7938 const RISCVSubtarget &Subtarget) {
7939 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7940
7941 MVT XLenVT = Subtarget.getXLenVT();
7942 MVT VT = Op.getSimpleValueType();
7943 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7944
7945 SDValue Mag = Op.getOperand(0);
7946 SDValue Sign = Op.getOperand(1);
7947
7948 SDLoc DL(Op);
7949
7950 // Get sign bit into an integer value.
7951 unsigned SignSize = Sign.getValueSizeInBits();
7952 SDValue SignAsInt = [&]() {
7953 if (SignSize == Subtarget.getXLen())
7954 return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
7955 switch (SignSize) {
7956 case 16:
7957 return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
7958 case 32:
7959 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
7960 case 64: {
7961 assert(XLenVT == MVT::i32 && "Unexpected type");
7962 // Copy the upper word to integer.
7963 SignSize = 32;
7964 return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
7965 .getValue(1);
7966 }
7967 default:
7968 llvm_unreachable("Unexpected sign size");
7969 }
7970 }();
7971
7972 // Get the signbit at the right position for MagAsInt.
7973 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
7974 SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
7975 SignAsInt,
7976 DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
7977
7978 // Mask the sign bit and any bits above it. The extra bits will be dropped
7979 // when we convert back to FP.
7980 SDValue SignMask = DAG.getConstant(
7981 APInt::getSignMask(16).sext(Subtarget.getXLen()), DL, XLenVT);
7982 SDValue SignBit = DAG.getNode(ISD::AND, DL, XLenVT, SignAsInt, SignMask);
7983
7984 // Transform Mag value to integer, and clear the sign bit.
7985 SDValue MagAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Mag);
7986 SDValue ClearSignMask = DAG.getConstant(
7987 APInt::getSignedMaxValue(16).sext(Subtarget.getXLen()), DL, XLenVT);
7988 SDValue ClearedSign =
7989 DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
7990
7991 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
7992 SDNodeFlags::Disjoint);
7993
7994 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
7995}
7996
7997/// Get a RISC-V target specified VL op for a given SDNode.
7998static unsigned getRISCVVLOp(SDValue Op) {
7999#define OP_CASE(NODE) \
8000 case ISD::NODE: \
8001 return RISCVISD::NODE##_VL;
8002#define VP_CASE(NODE) \
8003 case ISD::VP_##NODE: \
8004 return RISCVISD::NODE##_VL;
8005 // clang-format off
8006 switch (Op.getOpcode()) {
8007 default:
8008 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
8009 OP_CASE(ADD)
8010 OP_CASE(SUB)
8011 OP_CASE(MUL)
8012 OP_CASE(MULHS)
8013 OP_CASE(MULHU)
8014 OP_CASE(SDIV)
8015 OP_CASE(SREM)
8016 OP_CASE(UDIV)
8017 OP_CASE(UREM)
8018 OP_CASE(SHL)
8019 OP_CASE(SRA)
8020 OP_CASE(SRL)
8021 OP_CASE(ROTL)
8022 OP_CASE(ROTR)
8023 OP_CASE(BSWAP)
8024 OP_CASE(CTTZ)
8025 OP_CASE(CTLZ)
8026 OP_CASE(CTPOP)
8027 OP_CASE(BITREVERSE)
8028 OP_CASE(CLMUL)
8029 OP_CASE(CLMULH)
8030 OP_CASE(SADDSAT)
8031 OP_CASE(UADDSAT)
8032 OP_CASE(SSUBSAT)
8033 OP_CASE(USUBSAT)
8034 OP_CASE(AVGFLOORS)
8035 OP_CASE(AVGFLOORU)
8036 OP_CASE(AVGCEILS)
8037 OP_CASE(AVGCEILU)
8038 OP_CASE(FADD)
8039 OP_CASE(FSUB)
8040 OP_CASE(FMUL)
8041 OP_CASE(FDIV)
8042 OP_CASE(FNEG)
8043 OP_CASE(FABS)
8044 OP_CASE(FCOPYSIGN)
8045 OP_CASE(FSQRT)
8046 OP_CASE(SMIN)
8047 OP_CASE(SMAX)
8048 OP_CASE(UMIN)
8049 OP_CASE(UMAX)
8050 OP_CASE(ABDS)
8051 OP_CASE(ABDU)
8052 OP_CASE(STRICT_FADD)
8053 OP_CASE(STRICT_FSUB)
8054 OP_CASE(STRICT_FMUL)
8055 OP_CASE(STRICT_FDIV)
8056 OP_CASE(STRICT_FSQRT)
8057 VP_CASE(SDIV) // VP_SDIV
8058 VP_CASE(SREM) // VP_SREM
8059 VP_CASE(UDIV) // VP_UDIV
8060 VP_CASE(UREM) // VP_UREM
8061 case ISD::CTLZ_ZERO_POISON:
8062 return RISCVISD::CTLZ_VL;
8063 case ISD::CTTZ_ZERO_POISON:
8064 return RISCVISD::CTTZ_VL;
8065 case ISD::FMA:
8066 return RISCVISD::VFMADD_VL;
8067 case ISD::STRICT_FMA:
8068 return RISCVISD::STRICT_VFMADD_VL;
8069 case ISD::AND:
8070 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8071 return RISCVISD::VMAND_VL;
8072 return RISCVISD::AND_VL;
8073 case ISD::OR:
8074 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8075 return RISCVISD::VMOR_VL;
8076 return RISCVISD::OR_VL;
8077 case ISD::XOR:
8078 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8079 return RISCVISD::VMXOR_VL;
8080 return RISCVISD::XOR_VL;
8081 case ISD::ANY_EXTEND:
8082 case ISD::ZERO_EXTEND:
8083 return RISCVISD::VZEXT_VL;
8084 case ISD::SIGN_EXTEND:
8085 return RISCVISD::VSEXT_VL;
8086 case ISD::SETCC:
8087 return RISCVISD::SETCC_VL;
8088 case ISD::VSELECT:
8089 return RISCVISD::VMERGE_VL;
8090 case ISD::VP_MERGE:
8091 return RISCVISD::VMERGE_VL;
8092 case ISD::FMINNUM:
8093 case ISD::FMINIMUMNUM:
8094 return RISCVISD::VFMIN_VL;
8095 case ISD::FMAXNUM:
8096 case ISD::FMAXIMUMNUM:
8097 return RISCVISD::VFMAX_VL;
8098 case ISD::LRINT:
8099 case ISD::LLRINT:
8100 return RISCVISD::VFCVT_RM_X_F_VL;
8101 case ISD::MASKED_UDIV:
8102 return RISCVISD::UDIV_VL;
8103 case ISD::MASKED_UREM:
8104 return RISCVISD::UREM_VL;
8105 case ISD::MASKED_SDIV:
8106 return RISCVISD::SDIV_VL;
8107 case ISD::MASKED_SREM:
8108 return RISCVISD::SREM_VL;
8109 }
8110 // clang-format on
8111#undef OP_CASE
8112#undef VP_CASE
8113}
8114
8115static bool isPromotedOpNeedingSplit(SDValue Op,
8116 const RISCVSubtarget &Subtarget,
8117 const TargetLowering &TLI) {
8118 MVT OpVT = Op.getSimpleValueType();
8119 if (!OpVT.isVector())
8120 return false;
8121 MVT EltVT = OpVT.getVectorElementType();
8122 if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
8123 !Subtarget.hasVInstructionsF16()) &&
8124 !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
8125 (!Subtarget.hasVInstructionsBF16() ||
8126 !llvm::is_contained(ZvfbfaOps, Op.getOpcode()))))
8127 return false;
8128 // Need to split when the same width f32 vector type isn't legal.
8129 return !TLI.isTypeLegal(
8130 MVT::getVectorVT(MVT::f32, OpVT.getVectorElementCount()));
8131}
8132
8133static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
8134 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
8135 SDLoc DL(Op);
8136
8137 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8138 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8139
8140 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8141 if (!Op.getOperand(j).getValueType().isVector()) {
8142 LoOperands[j] = Op.getOperand(j);
8143 HiOperands[j] = Op.getOperand(j);
8144 continue;
8145 }
8146 std::tie(LoOperands[j], HiOperands[j]) =
8147 DAG.SplitVector(Op.getOperand(j), DL);
8148 }
8149
8150 SDValue LoRes =
8151 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
8152 SDValue HiRes =
8153 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
8154
8155 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
8156}
8157
8158static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
8159 bool IsVP) {
8160 SDLoc DL(Op);
8161
8162 if (IsVP) {
8163 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
8164 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
8165 auto [EVLLo, EVLHi] =
8166 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
8167
8168 SDValue ResLo =
8169 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8170 {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
8171 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8172 {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
8173 }
8174
8175 unsigned Opcode = Op.getOpcode();
8176 unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
8177
8178 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(OpNo), DL);
8179 if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
8180 SDValue ResLo = DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
8181 Op.getOperand(0), Lo, Op->getFlags());
8182 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), ResLo, Hi,
8183 Op->getFlags());
8184 }
8185
8186 SDValue ResLo =
8187 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Lo, Op->getFlags());
8188 SDValue ResHi =
8189 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Hi, Op->getFlags());
8190 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
8191 return DAG.getNode(BaseOpc, DL, Op.getValueType(), ResLo, ResHi,
8192 Op->getFlags());
8193}
8194
8195static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
8196
8197 assert(Op->isStrictFPOpcode());
8198
8199 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
8200
8201 SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
8202 SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
8203
8204 SDLoc DL(Op);
8205
8206 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8207 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8208
8209 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8210 if (!Op.getOperand(j).getValueType().isVector()) {
8211 LoOperands[j] = Op.getOperand(j);
8212 HiOperands[j] = Op.getOperand(j);
8213 continue;
8214 }
8215 std::tie(LoOperands[j], HiOperands[j]) =
8216 DAG.SplitVector(Op.getOperand(j), DL);
8217 }
8218
8219 SDValue LoRes =
8220 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
8221 HiOperands[0] = LoRes.getValue(1);
8222 SDValue HiRes =
8223 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
8224
8225 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
8226 LoRes.getValue(0), HiRes.getValue(0));
8227 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
8228}
8229
8230SDValue
8231RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
8232 SelectionDAG &DAG) const {
8233 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8234 "Unexpected bfloat16 load lowering");
8235
8236 SDLoc DL(Op);
8237 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
8238 EVT MemVT = LD->getMemoryVT();
8239 SDValue Load = DAG.getExtLoad(
8240 ISD::ZEXTLOAD, DL, Subtarget.getXLenVT(), LD->getChain(),
8241 LD->getBasePtr(),
8242 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
8243 LD->getMemOperand());
8244 // Using mask to make bf16 nan-boxing valid when we don't have flh
8245 // instruction. -65536 would be treat as a small number and thus it can be
8246 // directly used lui to get the constant.
8247 SDValue mask = DAG.getSignedConstant(-65536, DL, Subtarget.getXLenVT());
8248 SDValue OrSixteenOne =
8249 DAG.getNode(ISD::OR, DL, Load.getValueType(), {Load, mask});
8250 SDValue ConvertedResult =
8251 DAG.getNode(RISCVISD::NDS_FMV_BF16_X, DL, MVT::bf16, OrSixteenOne);
8252 return DAG.getMergeValues({ConvertedResult, Load.getValue(1)}, DL);
8253}
8254
8255SDValue
8256RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
8257 SelectionDAG &DAG) const {
8258 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8259 "Unexpected bfloat16 store lowering");
8260
8261 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
8262 SDLoc DL(Op);
8263 SDValue FMV = DAG.getNode(RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
8264 Subtarget.getXLenVT(), ST->getValue());
8265 return DAG.getTruncStore(
8266 ST->getChain(), DL, FMV, ST->getBasePtr(),
8267 EVT::getIntegerVT(*DAG.getContext(), ST->getMemoryVT().getSizeInBits()),
8268 ST->getMemOperand());
8269}
8270
8271static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
8272 const RISCVSubtarget &Subtarget);
8273
8274static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
8275 const RISCVSubtarget &Subtarget);
8276
8277SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
8278 SelectionDAG &DAG) const {
8279 switch (Op.getOpcode()) {
8280 default:
8281 reportFatalInternalError(
8282 "Unimplemented RISCVTargetLowering::LowerOperation Case");
8283 case ISD::PREFETCH:
8284 return LowerPREFETCH(Op, Subtarget, DAG);
8285 case ISD::ATOMIC_FENCE:
8286 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
8287 case ISD::GlobalAddress:
8288 return lowerGlobalAddress(Op, DAG);
8289 case ISD::BlockAddress:
8290 return lowerBlockAddress(Op, DAG);
8291 case ISD::ConstantPool:
8292 return lowerConstantPool(Op, DAG);
8293 case ISD::JumpTable:
8294 return lowerJumpTable(Op, DAG);
8295 case ISD::GlobalTLSAddress:
8296 return lowerGlobalTLSAddress(Op, DAG);
8297 case ISD::Constant:
8298 return lowerConstant(Op, DAG, Subtarget);
8299 case ISD::ConstantFP:
8300 return lowerConstantFP(Op, DAG);
8301 case ISD::SELECT:
8302 return lowerSELECT(Op, DAG);
8303 case ISD::BRCOND:
8304 return lowerBRCOND(Op, DAG);
8305 case ISD::VASTART:
8306 return lowerVASTART(Op, DAG);
8307 case ISD::FRAMEADDR:
8308 return lowerFRAMEADDR(Op, DAG);
8309 case ISD::RETURNADDR:
8310 return lowerRETURNADDR(Op, DAG);
8311 case ISD::SHL_PARTS