LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front("addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front("ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(1)->getType();
1095 Type *Ty = VectorType::get(ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front("tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with("get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 F->getParent(), Intrinsic::vector_extract, Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with("set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 F->getParent(), Intrinsic::vector_insert, Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 F->getParent(), Intrinsic::vector_insert, Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with("rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front("sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front("ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1172 StringRef Name) {
1173 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1174 Intrinsic::ID ID =
1175 StringSwitch<Intrinsic::ID>(Name)
1176 .Case("im2col.3d",
1177 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1178 .Case("im2col.4d",
1179 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1180 .Case("im2col.5d",
1181 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1182 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1183 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1184 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1185 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1186 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1187 .Default(Intrinsic::not_intrinsic);
1188
1189 if (ID == Intrinsic::not_intrinsic)
1190 return ID;
1191
1192 // These intrinsics may need upgrade for two reasons:
1193 // (1) When the address-space of the first argument is shared[AS=3]
1194 // (and we upgrade it to use shared_cluster address-space[AS=7])
1195 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1196 NVPTXAS::ADDRESS_SPACE_SHARED)
1197 return ID;
1198
1199 // (2) When there are only two boolean flag arguments at the end:
1200 //
1201 // The last three parameters of the older version of these
1202 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1203 //
1204 // The newer version reads as:
1205 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1206 //
1207 // So, when the type of the [N-3]rd argument is "not i1", then
1208 // it is the older version and we need to upgrade.
1209 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1210 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1211 if (!ArgType->isIntegerTy(1))
1212 return ID;
1213 }
1214
1215 return Intrinsic::not_intrinsic;
1216}
1217
1218// The legacy TMA reduction intrinsics encode the reduction operator in their
1219// name, while the current ones take it as an immediate argument. Map the
1220// operator part of a legacy name to the corresponding immediate value.
1221static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1222 return StringSwitch<std::optional<unsigned>>(Name)
1223 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1224 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1225 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1226 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1227 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1228 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1229 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1230 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1231 .Default(std::nullopt);
1232}
1233
1234static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1235 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1236 return Intrinsic::not_intrinsic;
1237
1238 auto [RedOpName, ShapeName] = Name.split('.');
1239 if (!getNVPTXTMAReductionOp(RedOpName))
1240 return Intrinsic::not_intrinsic;
1241
1242 return StringSwitch<Intrinsic::ID>(ShapeName)
1243 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1244 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1245 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1246 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1247 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1248 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1249 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1250 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1251 .Default(Intrinsic::not_intrinsic);
1252}
1253
1254static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1255 StringRef Name) {
1256 if (Name.consume_front("mapa.shared.cluster"))
1257 if (F->getReturnType()->getPointerAddressSpace() ==
1258 NVPTXAS::ADDRESS_SPACE_SHARED)
1259 return Intrinsic::nvvm_mapa_shared_cluster;
1260
1261 if (Name.consume_front("cp.async.bulk.")) {
1262 Intrinsic::ID ID =
1263 StringSwitch<Intrinsic::ID>(Name)
1264 .Case("global.to.shared.cluster",
1265 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1266 .Case("shared.cta.to.cluster",
1267 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1268 .Default(Intrinsic::not_intrinsic);
1269
1270 if (ID != Intrinsic::not_intrinsic)
1271 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1272 NVPTXAS::ADDRESS_SPACE_SHARED)
1273 return ID;
1274 }
1275
1276 return Intrinsic::not_intrinsic;
1277}
1278
1279static Intrinsic::ID
1280shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1281 if (!Name.consume_front("tcgen05.commit."))
1282 return Intrinsic::not_intrinsic;
1283
1284 if (Name.consume_front("shared."))
1285 return StringSwitch<Intrinsic::ID>(Name)
1286 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1287 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1288 .Default(Intrinsic::not_intrinsic);
1289
1290 if (Name.consume_front("mc.shared.")) {
1291 // Only upgrade older i16 mc variants.
1292 if (!F->getArg(1)->getType()->isIntegerTy(16))
1293 return Intrinsic::not_intrinsic;
1294
1295 return StringSwitch<Intrinsic::ID>(Name)
1296 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1297 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1298 .Default(Intrinsic::not_intrinsic);
1299 }
1300
1301 return Intrinsic::not_intrinsic;
1302}
1303
1304static Intrinsic::ID
1305shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1306 if (F->arg_size() != 2)
1307 return Intrinsic::not_intrinsic;
1308
1309 if (Name.consume_front("tcgen05.alloc.shared.") ||
1310 Name.consume_front("tcgen05.alloc."))
1311 return StringSwitch<Intrinsic::ID>(Name)
1312 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1313 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1314 .Default(Intrinsic::not_intrinsic);
1315
1316 if (Name.consume_front("tcgen05.dealloc."))
1317 return StringSwitch<Intrinsic::ID>(Name)
1318 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1319 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1320 .Default(Intrinsic::not_intrinsic);
1321
1322 return Intrinsic::not_intrinsic;
1323}
1324
1325static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1326 if (Name.consume_front("fma.rn."))
1327 return StringSwitch<Intrinsic::ID>(Name)
1328 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1329 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1330 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1331 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1332 .Default(Intrinsic::not_intrinsic);
1333
1334 if (Name.consume_front("fmax."))
1335 return StringSwitch<Intrinsic::ID>(Name)
1336 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1337 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1338 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1339 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1340 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1341 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1342 .Case("ftz.nan.xorsign.abs.bf16",
1343 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1344 .Case("ftz.nan.xorsign.abs.bf16x2",
1345 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1346 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1347 .Case("ftz.xorsign.abs.bf16x2",
1348 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1349 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1350 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1351 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1352 .Case("nan.xorsign.abs.bf16x2",
1353 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1354 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1355 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1356 .Default(Intrinsic::not_intrinsic);
1357
1358 if (Name.consume_front("fmin."))
1359 return StringSwitch<Intrinsic::ID>(Name)
1360 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1361 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1362 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1363 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1364 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1365 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1366 .Case("ftz.nan.xorsign.abs.bf16",
1367 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1368 .Case("ftz.nan.xorsign.abs.bf16x2",
1369 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1370 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1371 .Case("ftz.xorsign.abs.bf16x2",
1372 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1373 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1374 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1375 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1376 .Case("nan.xorsign.abs.bf16x2",
1377 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1378 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1379 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1380 .Default(Intrinsic::not_intrinsic);
1381
1382 if (Name.consume_front("neg."))
1383 return StringSwitch<Intrinsic::ID>(Name)
1384 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1385 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1386 .Default(Intrinsic::not_intrinsic);
1387
1388 return Intrinsic::not_intrinsic;
1389}
1390
1391static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1392 StringRef Name) {
1393 if (!Name.consume_front("tcgen05.mma."))
1394 return Intrinsic::not_intrinsic;
1395
1396 // tcgen05.mma.ws.* variants do not need collector-b appended.
1397 if (Name.starts_with("ws"))
1398 return Intrinsic::not_intrinsic;
1399
1400 return F->getIntrinsicID();
1401}
1402
1403static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1404 return Name.consume_front("local") || Name.consume_front("shared") ||
1405 Name.consume_front("global") || Name.consume_front("constant") ||
1406 Name.consume_front("param");
1407}
1408
1409static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1410 if (!Name.consume_front("vp."))
1411 return 0;
1412 return StringSwitch<unsigned>(Name)
1413 .StartsWith("select", Instruction::Select)
1414 .StartsWith("add", Instruction::Add)
1415 .StartsWith("sub", Instruction::Sub)
1416 .StartsWith("mul", Instruction::Mul)
1417 .StartsWith("ashr", Instruction::AShr)
1418 .StartsWith("lshr", Instruction::LShr)
1419 .StartsWith("shl", Instruction::Shl)
1420 .StartsWith("or", Instruction::Or)
1421 .StartsWith("and", Instruction::And)
1422 .StartsWith("xor", Instruction::Xor)
1423 .StartsWith("fadd", Instruction::FAdd)
1424 .StartsWith("fsub", Instruction::FSub)
1425 .StartsWith("fmuladd", 0)
1426 .StartsWith("fmul", Instruction::FMul)
1427 .StartsWith("fdiv", Instruction::FDiv)
1428 .StartsWith("frem", Instruction::FRem)
1429 .StartsWith("fneg", Instruction::FNeg)
1430 .StartsWith("trunc", Instruction::Trunc)
1431 .StartsWith("zext", Instruction::ZExt)
1432 .StartsWith("sext", Instruction::SExt)
1433 .StartsWith("fptrunc", Instruction::FPTrunc)
1434 .StartsWith("fpext", Instruction::FPExt)
1435 .StartsWith("fptoui", Instruction::FPToUI)
1436 .StartsWith("fptosi", Instruction::FPToSI)
1437 .StartsWith("uitofp", Instruction::UIToFP)
1438 .StartsWith("sitofp", Instruction::SIToFP)
1439 .StartsWith("ptrtoint", Instruction::PtrToInt)
1440 .StartsWith("inttoptr", Instruction::IntToPtr)
1441 .StartsWith("icmp", Instruction::ICmp)
1442 .StartsWith("fcmp", Instruction::FCmp)
1443 .Default(0);
1444}
1445
1446static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1447 if (!Name.consume_front("vp."))
1448 return 0;
1449 return StringSwitch<Intrinsic::ID>(Name)
1450 .StartsWith("abs", Intrinsic::abs)
1451 .StartsWith("smax", Intrinsic::smax)
1452 .StartsWith("smin", Intrinsic::smin)
1453 .StartsWith("umax", Intrinsic::umax)
1454 .StartsWith("umin", Intrinsic::umin)
1455 .StartsWith("copysign", Intrinsic::copysign)
1456 .StartsWith("minnum", Intrinsic::minnum)
1457 .StartsWith("maxnum", Intrinsic::maxnum)
1458 .StartsWith("minimum", Intrinsic::minimum)
1459 .StartsWith("maximum", Intrinsic::maximum)
1460 .StartsWith("fabs", Intrinsic::fabs)
1461 .StartsWith("sqrt", Intrinsic::sqrt)
1462 .StartsWith("fma", Intrinsic::fma)
1463 .StartsWith("fmuladd", Intrinsic::fmuladd)
1464 .StartsWith("ceil", Intrinsic::ceil)
1465 .StartsWith("floor", Intrinsic::floor)
1466 .StartsWith("rint", Intrinsic::rint)
1467 .StartsWith("nearbyint", Intrinsic::nearbyint)
1468 .StartsWith("roundeven", Intrinsic::roundeven)
1469 .StartsWith("roundtozero", Intrinsic::trunc)
1470 .StartsWith("round", Intrinsic::round)
1471 .StartsWith("lrint", Intrinsic::lrint)
1472 .StartsWith("llrint", Intrinsic::llrint)
1473 .StartsWith("bitreverse", Intrinsic::bitreverse)
1474 .StartsWith("bswap", Intrinsic::bswap)
1475 .StartsWith("ctpop", Intrinsic::ctpop)
1476 .StartsWith("ctlz", Intrinsic::ctlz)
1477 .StartsWith("cttz.elts", 0)
1478 .StartsWith("cttz", Intrinsic::cttz)
1479 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1480 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1481 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1482 .StartsWith("usub.sat", Intrinsic::usub_sat)
1483 .StartsWith("fshl", Intrinsic::fshl)
1484 .StartsWith("fshr", Intrinsic::fshr)
1485 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1486 .Default(0);
1487}
1488
1492
1493static bool convertIntrinsicValidType(StringRef Name,
1494 const FunctionType *FuncTy) {
1495 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1496 if (Name.starts_with("to.fp16")) {
1497 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1498 HalfTy) &&
1499 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1500 FuncTy->getReturnType());
1501 }
1502
1503 if (Name.starts_with("from.fp16")) {
1504 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1505 HalfTy) &&
1506 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1507 FuncTy->getReturnType());
1508 }
1509
1510 return false;
1511}
1512
1513static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1514 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1515 if (IID == Intrinsic::not_intrinsic)
1516 return false;
1517
1518 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1519 if (Defaults.empty())
1520 return false;
1521
1522 // Overloaded intrinsics are out of scope for the default-arg feature
1523 // and will be supported in a follow-up.
1524 if (Intrinsic::isOverloaded(IID))
1525 return false;
1526
1527 // Get the canonical full declaration for this intrinsic.
1528 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1529
1530 // If the existing declaration already has all args, nothing to upgrade
1531 if (F->arg_size() >= FullDecl->arg_size())
1532 return false;
1533
1534 // Defaults are a contiguous trailing block, so checking the first missing
1535 // argument is enough.
1536 if (F->arg_size() < FirstDefault)
1537 return false;
1538
1539 NewFn = FullDecl;
1540 return true;
1541}
1542
1543static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1544 bool CanUpgradeDebugIntrinsicsToRecords) {
1545 assert(F && "Illegal to upgrade a non-existent Function.");
1546
1547 StringRef Name = F->getName();
1548
1549 // Quickly eliminate it, if it's not a candidate.
1550 if (!Name.consume_front("llvm.") || Name.empty())
1551 return false;
1552
1553 switch (Name[0]) {
1554 default: break;
1555 case 'a': {
1556 bool IsArm = Name.consume_front("arm.");
1557 if (IsArm || Name.consume_front("aarch64.")) {
1558 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1559 return true;
1560 break;
1561 }
1562
1563 if (Name.consume_front("amdgcn.")) {
1564 if (Name == "alignbit") {
1565 // Target specific intrinsic became redundant
1566 NewFn = Intrinsic::getOrInsertDeclaration(
1567 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1568 return true;
1569 }
1570
1571 if (Name.consume_front("atomic.")) {
1572 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1573 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1574 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1575 // and usub_sat so there's no new declaration.
1576 NewFn = nullptr;
1577 return true;
1578 }
1579 break; // No other 'amdgcn.atomic.*'
1580 }
1581
1582 switch (F->getIntrinsicID()) {
1583 default:
1584 break;
1585 // Legacy wmma iu intrinsics without the optional clamp operand.
1586 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1587 if (F->arg_size() == 7) {
1588 NewFn = nullptr;
1589 return true;
1590 }
1591 break;
1592 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1593 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1594 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1595 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1596 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1597 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1598 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1599 if (F->arg_size() == 8) {
1600 NewFn = nullptr;
1601 return true;
1602 }
1603 break;
1604 }
1605
1606 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1607 Name.consume_front("flat.atomic.")) {
1608 if (Name.starts_with("fadd") ||
1609 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1610 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1611 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1612 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1613 // declaration.
1614 NewFn = nullptr;
1615 return true;
1616 }
1617 }
1618
1619 if (Name.starts_with("ldexp.")) {
1620 // Target specific intrinsic became redundant
1621 NewFn = Intrinsic::getOrInsertDeclaration(
1622 F->getParent(), Intrinsic::ldexp,
1623 {F->getReturnType(), F->getArg(1)->getType()});
1624 return true;
1625 }
1626 break; // No other 'amdgcn.*'
1627 }
1628
1629 break;
1630 }
1631 case 'c': {
1632 if (F->arg_size() == 1) {
1633 if (Name.consume_front("convert.")) {
1634 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1635 NewFn = nullptr;
1636 return true;
1637 }
1638 }
1639
1640 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1641 .StartsWith("ctlz.", Intrinsic::ctlz)
1642 .StartsWith("cttz.", Intrinsic::cttz)
1643 .Default(Intrinsic::not_intrinsic);
1644 if (ID != Intrinsic::not_intrinsic) {
1645 rename(F);
1646 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1647 F->arg_begin()->getType());
1648 return true;
1649 }
1650 }
1651
1652 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1653 if (Name == "coro.end" &&
1654 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1655 CoroEndID = Intrinsic::coro_end;
1656 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1657 CoroEndID = Intrinsic::coro_end_async;
1658
1659 if (CoroEndID != Intrinsic::not_intrinsic) {
1660 rename(F);
1661 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1662 return true;
1663 }
1664
1665 break;
1666 }
1667 case 'd':
1668 if (Name.consume_front("dbg.")) {
1669 // Mark debug intrinsics for upgrade to new debug format.
1670 if (CanUpgradeDebugIntrinsicsToRecords) {
1671 if (Name == "addr" || Name == "value" || Name == "assign" ||
1672 Name == "declare" || Name == "label") {
1673 // There's no function to replace these with.
1674 NewFn = nullptr;
1675 // But we do want these to get upgraded.
1676 return true;
1677 }
1678 }
1679 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1680 // converted to DbgVariableRecords later.
1681 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1682 rename(F);
1683 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1684 Intrinsic::dbg_value);
1685 return true;
1686 }
1687 break; // No other 'dbg.*'.
1688 }
1689 break;
1690 case 'e':
1691 if (Name.consume_front("experimental.vector.")) {
1692 Intrinsic::ID ID =
1693 StringSwitch<Intrinsic::ID>(Name)
1694 // Skip over extract.last.active, otherwise it will be 'upgraded'
1695 // to a regular vector extract which is a different operation.
1696 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1697 .StartsWith("extract.", Intrinsic::vector_extract)
1698 .StartsWith("insert.", Intrinsic::vector_insert)
1699 .StartsWith("reverse.", Intrinsic::vector_reverse)
1700 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1701 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1702 .StartsWith("partial.reduce.add",
1703 Intrinsic::vector_partial_reduce_add)
1704 .Default(Intrinsic::not_intrinsic);
1705 if (ID != Intrinsic::not_intrinsic) {
1706 const auto *FT = F->getFunctionType();
1707 SmallVector<Type *, 2> Tys;
1708 if (ID == Intrinsic::vector_extract ||
1709 ID == Intrinsic::vector_interleave2)
1710 // Extracting overloads the return type.
1711 Tys.push_back(FT->getReturnType());
1712 if (ID != Intrinsic::vector_interleave2)
1713 Tys.push_back(FT->getParamType(0));
1714 if (ID == Intrinsic::vector_insert ||
1715 ID == Intrinsic::vector_partial_reduce_add)
1716 // Inserting overloads the inserted type.
1717 Tys.push_back(FT->getParamType(1));
1718 rename(F);
1719 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1720 return true;
1721 }
1722
1723 if (Name.consume_front("reduce.")) {
1724 SmallVector<StringRef, 2> Groups;
1725 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1726 if (R.match(Name, &Groups))
1727 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1728 .Case("add", Intrinsic::vector_reduce_add)
1729 .Case("mul", Intrinsic::vector_reduce_mul)
1730 .Case("and", Intrinsic::vector_reduce_and)
1731 .Case("or", Intrinsic::vector_reduce_or)
1732 .Case("xor", Intrinsic::vector_reduce_xor)
1733 .Case("smax", Intrinsic::vector_reduce_smax)
1734 .Case("smin", Intrinsic::vector_reduce_smin)
1735 .Case("umax", Intrinsic::vector_reduce_umax)
1736 .Case("umin", Intrinsic::vector_reduce_umin)
1737 .Case("fmax", Intrinsic::vector_reduce_fmax)
1738 .Case("fmin", Intrinsic::vector_reduce_fmin)
1739 .Default(Intrinsic::not_intrinsic);
1740
1741 bool V2 = false;
1742 if (ID == Intrinsic::not_intrinsic) {
1743 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1744 Groups.clear();
1745 V2 = true;
1746 if (R2.match(Name, &Groups))
1747 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1748 .Case("fadd", Intrinsic::vector_reduce_fadd)
1749 .Case("fmul", Intrinsic::vector_reduce_fmul)
1750 .Default(Intrinsic::not_intrinsic);
1751 }
1752 if (ID != Intrinsic::not_intrinsic) {
1753 rename(F);
1754 auto Args = F->getFunctionType()->params();
1755 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1756 {Args[V2 ? 1 : 0]});
1757 return true;
1758 }
1759 break; // No other 'expermental.vector.reduce.*'.
1760 }
1761
1762 if (Name.consume_front("splice"))
1763 return true;
1764 break; // No other 'experimental.vector.*'.
1765 }
1766 if (Name.consume_front("experimental.stepvector.")) {
1767 Intrinsic::ID ID = Intrinsic::stepvector;
1768 rename(F);
1769 NewFn = Intrinsic::getOrInsertDeclaration(
1770 F->getParent(), ID, F->getFunctionType()->getReturnType());
1771 return true;
1772 }
1773 break; // No other 'e*'.
1774 case 'f':
1775 if (Name.starts_with("flt.rounds")) {
1776 rename(F);
1777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1778 Intrinsic::get_rounding);
1779 return true;
1780 }
1781 break;
1782 case 'i':
1783 if (Name.starts_with("invariant.group.barrier")) {
1784 // Rename invariant.group.barrier to launder.invariant.group
1785 auto Args = F->getFunctionType()->params();
1786 Type* ObjectPtr[1] = {Args[0]};
1787 rename(F);
1788 NewFn = Intrinsic::getOrInsertDeclaration(
1789 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1790 return true;
1791 }
1792 break;
1793 case 'l': {
1794 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1795 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1796 if (IsLifetimeStart || IsLifetimeEnd) {
1797 if (F->arg_size() == 2) {
1798 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1799 : Intrinsic::lifetime_end;
1800 rename(F);
1801 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1802 // arguments. Use the Ptr argument to create new declaration.
1803 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1804 F->getArg(1)->getType());
1805 return true;
1806 } else if (F->arg_size() == 1 && Name == ".i64") {
1807 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1808 // Autoupgrade prior to
1809 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1810 // intrinsic with no expected calls. To allow auto-upgrade process to
1811 // delete such invalid intrinsic declaration, set NewFn = nullptr
1812 // and return true here. If there are actual calls to this intrinsic
1813 // (which is not expected), they will be deleted in
1814 // UpgradeIntrinsicCall.
1815 NewFn = nullptr;
1816 return true;
1817 }
1818 }
1819 break;
1820 }
1821 case 'm': {
1822 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1823 // alignment parameter to embedding the alignment as an attribute of
1824 // the pointer args.
1825 if (unsigned ID = StringSwitch<unsigned>(Name)
1826 .StartsWith("memcpy.", Intrinsic::memcpy)
1827 .StartsWith("memmove.", Intrinsic::memmove)
1828 .Default(0)) {
1829 if (F->arg_size() == 5) {
1830 rename(F);
1831 // Get the types of dest, src, and len
1832 ArrayRef<Type *> ParamTypes =
1833 F->getFunctionType()->params().slice(0, 3);
1834 NewFn =
1835 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1836 return true;
1837 }
1838 }
1839 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1840 rename(F);
1841 // Get the types of dest, and len
1842 const auto *FT = F->getFunctionType();
1843 Type *ParamTypes[2] = {
1844 FT->getParamType(0), // Dest
1845 FT->getParamType(2) // len
1846 };
1847 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1848 Intrinsic::memset, ParamTypes);
1849 return true;
1850 }
1851
1852 unsigned MaskedID =
1853 StringSwitch<unsigned>(Name)
1854 .StartsWith("masked.load", Intrinsic::masked_load)
1855 .StartsWith("masked.gather", Intrinsic::masked_gather)
1856 .StartsWith("masked.store", Intrinsic::masked_store)
1857 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1858 .Default(0);
1859 if (MaskedID && F->arg_size() == 4) {
1860 rename(F);
1861 if (MaskedID == Intrinsic::masked_load ||
1862 MaskedID == Intrinsic::masked_gather) {
1863 NewFn = Intrinsic::getOrInsertDeclaration(
1864 F->getParent(), MaskedID,
1865 {F->getReturnType(), F->getArg(0)->getType()});
1866 return true;
1867 }
1868 NewFn = Intrinsic::getOrInsertDeclaration(
1869 F->getParent(), MaskedID,
1870 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1871 return true;
1872 }
1873 break;
1874 }
1875 case 'n': {
1876 if (Name.consume_front("nvvm.")) {
1877 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1878 if (F->arg_size() == 1) {
1879 Intrinsic::ID IID =
1880 StringSwitch<Intrinsic::ID>(Name)
1881 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1882 .Case("clz.i", Intrinsic::ctlz)
1883 .Case("popc.i", Intrinsic::ctpop)
1884 .Default(Intrinsic::not_intrinsic);
1885 if (IID != Intrinsic::not_intrinsic) {
1886 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1887 {F->getReturnType()});
1888 return true;
1889 }
1890 } else if (F->arg_size() == 2) {
1891 Intrinsic::ID IID =
1892 StringSwitch<Intrinsic::ID>(Name)
1893 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1894 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1895 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1896 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1897 .Default(Intrinsic::not_intrinsic);
1898 if (IID != Intrinsic::not_intrinsic) {
1899 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1900 {F->getReturnType()});
1901 return true;
1902 }
1903 }
1904
1905 // Check for nvvm intrinsics that need a return type adjustment.
1906 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1907 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1908 if (IID != Intrinsic::not_intrinsic) {
1909 NewFn = nullptr;
1910 return true;
1911 }
1912 }
1913
1914 // Upgrade Distributed Shared Memory Intrinsics
1915 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1916 if (IID != Intrinsic::not_intrinsic) {
1917 rename(F);
1918 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1919 return true;
1920 }
1921
1922 // Upgrade TMA reduction intrinsics
1923 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
1924 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
1925 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
1926 if (IID != Intrinsic::not_intrinsic) {
1927 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1928 return true;
1929 }
1930
1931 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
1932 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
1933 if (IID != Intrinsic::not_intrinsic) {
1934 rename(F);
1935 NewFn = Intrinsic::getOrInsertDeclaration(
1936 F->getParent(), IID, F->getReturnType(),
1937 F->getFunctionType()->params());
1938 return true;
1939 }
1940
1941 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
1942 // tcgen05.alloc shared variants to anyptr intrinsics.
1943 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
1944 if (IID != Intrinsic::not_intrinsic) {
1945 rename(F);
1946 if (Intrinsic::isOverloaded(IID))
1947 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1948 {F->getArg(0)->getType()});
1949 else
1950 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1951 return true;
1952 }
1953
1954 // Upgrade TMA copy G2S Intrinsics
1955 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1956 if (IID != Intrinsic::not_intrinsic) {
1957 rename(F);
1958 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1959 return true;
1960 }
1961
1962 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
1963 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
1964 if (IID != Intrinsic::not_intrinsic) {
1965 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1966 return NewFn != F;
1967 }
1968
1969 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1970 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1971 //
1972 // TODO: We could add lohi.i2d.
1973 bool Expand = false;
1974 if (Name.consume_front("abs."))
1975 // nvvm.abs.{i,ii}
1976 Expand =
1977 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1978 else if (Name.consume_front("fabs."))
1979 // nvvm.fabs.{f,ftz.f,d}
1980 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1981 else if (Name.consume_front("ex2.approx."))
1982 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1983 Expand =
1984 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1985 else if (Name.consume_front("atomic.load."))
1986 // nvvm.atomic.load.add.{f32,f64}.p
1987 // nvvm.atomic.load.{inc,dec}.32.p
1988 Expand = StringSwitch<bool>(Name)
1989 .StartsWith("add.f32.p", true)
1990 .StartsWith("add.f64.p", true)
1991 .StartsWith("inc.32.p", true)
1992 .StartsWith("dec.32.p", true)
1993 .Default(false);
1994 else if (Name.consume_front("atomic."))
1995 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
1996 // nvvm.atomic.cas.gen.i.{cta,sys}
1997 Expand = StringSwitch<bool>(Name)
1998 .StartsWith("add.gen.", true)
1999 .StartsWith("exch.gen.", true)
2000 .StartsWith("max.gen.", true)
2001 .StartsWith("min.gen.", true)
2002 .StartsWith("inc.gen.", true)
2003 .StartsWith("dec.gen.", true)
2004 .StartsWith("and.gen.", true)
2005 .StartsWith("or.gen.", true)
2006 .StartsWith("xor.gen.", true)
2007 .StartsWith("cas.gen.", true)
2008 .Default(false);
2009 else if (Name.consume_front("bitcast."))
2010 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2011 Expand =
2012 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2013 else if (Name.consume_front("rotate."))
2014 // nvvm.rotate.{b32,b64,right.b64}
2015 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2016 else if (Name.consume_front("ptr.gen.to."))
2017 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2018 Expand = consumeNVVMPtrAddrSpace(Name);
2019 else if (Name.consume_front("ptr."))
2020 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2021 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2022 else if (Name.consume_front("ldg.global."))
2023 // nvvm.ldg.global.{i,p,f}
2024 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2025 Name.starts_with("p."));
2026 else
2027 Expand = StringSwitch<bool>(Name)
2028 .Case("barrier0", true)
2029 .Case("barrier.n", true)
2030 .Case("barrier.sync.cnt", true)
2031 .Case("barrier.sync", true)
2032 .Case("barrier", true)
2033 .Case("bar.sync", true)
2034 .Case("barrier0.popc", true)
2035 .Case("barrier0.and", true)
2036 .Case("barrier0.or", true)
2037 .Case("clz.ll", true)
2038 .Case("popc.ll", true)
2039 .Case("h2f", true)
2040 .Case("swap.lo.hi.b64", true)
2041 .Case("tanh.approx.f32", true)
2042 .Default(false);
2043
2044 if (Expand) {
2045 NewFn = nullptr;
2046 return true;
2047 }
2048 break; // No other 'nvvm.*'.
2049 }
2050 break;
2051 }
2052 case 'o':
2053 if (Name.starts_with("objectsize.")) {
2054 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2055 if (F->arg_size() == 2 || F->arg_size() == 3) {
2056 rename(F);
2057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2058 Intrinsic::objectsize, Tys);
2059 return true;
2060 }
2061 }
2062 break;
2063
2064 case 'p':
2065 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2066 rename(F);
2067 NewFn = Intrinsic::getOrInsertDeclaration(
2068 F->getParent(), Intrinsic::ptr_annotation,
2069 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2070 return true;
2071 }
2072 break;
2073
2074 case 'r': {
2075 if (Name.consume_front("riscv.")) {
2076 Intrinsic::ID ID;
2077 ID = StringSwitch<Intrinsic::ID>(Name)
2078 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2079 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2080 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2081 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2082 .Default(Intrinsic::not_intrinsic);
2083 if (ID != Intrinsic::not_intrinsic) {
2084 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2085 rename(F);
2086 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2087 return true;
2088 }
2089 break; // No other applicable upgrades.
2090 }
2091
2092 ID = StringSwitch<Intrinsic::ID>(Name)
2093 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2094 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2095 .Default(Intrinsic::not_intrinsic);
2096 if (ID != Intrinsic::not_intrinsic) {
2097 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2098 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2099 rename(F);
2100 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2101 return true;
2102 }
2103 break; // No other applicable upgrades.
2104 }
2105
2106 ID = StringSwitch<Intrinsic::ID>(Name)
2107 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2108 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2109 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2110 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2111 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2112 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2113 .Default(Intrinsic::not_intrinsic);
2114 if (ID != Intrinsic::not_intrinsic) {
2115 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2116 rename(F);
2117 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2118 return true;
2119 }
2120 break; // No other applicable upgrades.
2121 }
2122
2123 // Replace llvm.riscv.clmul with llvm.clmul.
2124 if (Name == "clmul.i32" || Name == "clmul.i64") {
2125 NewFn = Intrinsic::getOrInsertDeclaration(
2126 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2127 return true;
2128 }
2129
2130 break; // No other 'riscv.*' intrinsics
2131 }
2132 } break;
2133
2134 case 's':
2135 if (Name == "stackprotectorcheck") {
2136 NewFn = nullptr;
2137 return true;
2138 }
2139 break;
2140
2141 case 't':
2142 if (Name == "thread.pointer") {
2143 NewFn = Intrinsic::getOrInsertDeclaration(
2144 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2145 return true;
2146 }
2147 break;
2148
2149 case 'v': {
2150 if (Name == "var.annotation" && F->arg_size() == 4) {
2151 rename(F);
2152 NewFn = Intrinsic::getOrInsertDeclaration(
2153 F->getParent(), Intrinsic::var_annotation,
2154 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2155 return true;
2156 }
2157 if (Name.consume_front("vector.splice")) {
2158 if (Name.starts_with(".left") || Name.starts_with(".right"))
2159 break;
2160 return true;
2161 }
2162 if (shouldUpgradeVPIntrinsic(Name))
2163 return true;
2164 break;
2165 }
2166
2167 case 'w':
2168 if (Name.consume_front("wasm.")) {
2169 Intrinsic::ID ID =
2170 StringSwitch<Intrinsic::ID>(Name)
2171 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2172 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2173 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2174 .Default(Intrinsic::not_intrinsic);
2175 if (ID != Intrinsic::not_intrinsic) {
2176 rename(F);
2177 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2178 F->getReturnType());
2179 return true;
2180 }
2181
2182 if (Name.consume_front("dot.i8x16.i7x16.")) {
2183 ID = StringSwitch<Intrinsic::ID>(Name)
2184 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2185 .Case("add.signed",
2186 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2187 .Default(Intrinsic::not_intrinsic);
2188 if (ID != Intrinsic::not_intrinsic) {
2189 rename(F);
2190 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2191 return true;
2192 }
2193 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2194 }
2195 break; // No other 'wasm.*'.
2196 }
2197 break;
2198
2199 case 'x':
2200 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2201 return true;
2202 }
2203
2204 auto *ST = dyn_cast<StructType>(F->getReturnType());
2205 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2206 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2207 // Replace return type with literal non-packed struct. Only do this for
2208 // intrinsics declared to return a struct, not for intrinsics with
2209 // overloaded return type, in which case the exact struct type will be
2210 // mangled into the name.
2211 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2212 FunctionType *FT = F->getFunctionType();
2213 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2214 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2215 std::string Name = F->getName().str();
2216 rename(F);
2217 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2218 Name, F->getParent());
2219
2220 // The new function may also need remangling.
2221 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2222 NewFn = *Result;
2223 return true;
2224 }
2225 }
2226
2227 // Remangle our intrinsic since we upgrade the mangling
2228 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2229 if (Result != std::nullopt) {
2230 NewFn = *Result;
2231 return true;
2232 }
2233
2234 // This may not belong here. This function is effectively being overloaded
2235 // to both detect an intrinsic which needs upgrading, and to provide the
2236 // upgraded form of the intrinsic. We should perhaps have two separate
2237 // functions for this.
2238 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
2239 return true;
2240
2241 return false;
2242}
2243
2244bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2245 bool CanUpgradeDebugIntrinsicsToRecords) {
2246 NewFn = nullptr;
2247 bool Upgraded =
2248 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2249
2250 // Upgrade intrinsic attributes. This does not change the function.
2251 if (NewFn)
2252 F = NewFn;
2253 if (Intrinsic::ID id = F->getIntrinsicID()) {
2254 // Only do this if the intrinsic signature is valid.
2255 SmallVector<Type *> OverloadTys;
2256 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2257 F->setAttributes(
2258 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2259 }
2260 return Upgraded;
2261}
2262
2263GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2264 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2265 GV->getName() == "llvm.global_dtors")) ||
2266 !GV->hasInitializer())
2267 return nullptr;
2268 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2269 if (!ATy)
2270 return nullptr;
2271 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2272 if (!STy || STy->getNumElements() != 2)
2273 return nullptr;
2274
2275 LLVMContext &C = GV->getContext();
2276 IRBuilder<> IRB(C);
2277 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2278 IRB.getPtrTy());
2279 Constant *Init = GV->getInitializer();
2280 unsigned N = Init->getNumOperands();
2281 std::vector<Constant *> NewCtors(N);
2282 for (unsigned i = 0; i != N; ++i) {
2283 auto Ctor = cast<Constant>(Init->getOperand(i));
2284 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2285 Ctor->getAggregateElement(1),
2286 ConstantPointerNull::get(IRB.getPtrTy()));
2287 }
2288 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2289
2290 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2291 NewInit, GV->getName());
2292}
2293
2294// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2295// to byte shuffles.
2296static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2297 unsigned Shift) {
2298 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2299 unsigned NumElts = ResultTy->getNumElements() * 8;
2300
2301 // Bitcast from a 64-bit element type to a byte element type.
2302 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2303 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2304
2305 // We'll be shuffling in zeroes.
2306 Value *Res = Constant::getNullValue(VecTy);
2307
2308 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2309 // we'll just return the zero vector.
2310 if (Shift < 16) {
2311 int Idxs[64];
2312 // 256/512-bit version is split into 2/4 16-byte lanes.
2313 for (unsigned l = 0; l != NumElts; l += 16)
2314 for (unsigned i = 0; i != 16; ++i) {
2315 unsigned Idx = NumElts + i - Shift;
2316 if (Idx < NumElts)
2317 Idx -= NumElts - 16; // end of lane, switch operand.
2318 Idxs[l + i] = Idx + l;
2319 }
2320
2321 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2322 }
2323
2324 // Bitcast back to a 64-bit element type.
2325 return Builder.CreateBitCast(Res, ResultTy, "cast");
2326}
2327
2328// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2329// to byte shuffles.
2330static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2331 unsigned Shift) {
2332 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2333 unsigned NumElts = ResultTy->getNumElements() * 8;
2334
2335 // Bitcast from a 64-bit element type to a byte element type.
2336 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2337 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2338
2339 // We'll be shuffling in zeroes.
2340 Value *Res = Constant::getNullValue(VecTy);
2341
2342 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2343 // we'll just return the zero vector.
2344 if (Shift < 16) {
2345 int Idxs[64];
2346 // 256/512-bit version is split into 2/4 16-byte lanes.
2347 for (unsigned l = 0; l != NumElts; l += 16)
2348 for (unsigned i = 0; i != 16; ++i) {
2349 unsigned Idx = i + Shift;
2350 if (Idx >= 16)
2351 Idx += NumElts - 16; // end of lane, switch operand.
2352 Idxs[l + i] = Idx + l;
2353 }
2354
2355 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2356 }
2357
2358 // Bitcast back to a 64-bit element type.
2359 return Builder.CreateBitCast(Res, ResultTy, "cast");
2360}
2361
2362static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2363 unsigned NumElts) {
2364 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2365 llvm::VectorType *MaskTy = FixedVectorType::get(
2366 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2367 Mask = Builder.CreateBitCast(Mask, MaskTy);
2368
2369 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2370 // i8 and we need to extract down to the right number of elements.
2371 if (NumElts <= 4) {
2372 int Indices[4];
2373 for (unsigned i = 0; i != NumElts; ++i)
2374 Indices[i] = i;
2375 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2376 "extract");
2377 }
2378
2379 return Mask;
2380}
2381
2382static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2383 Value *Op1) {
2384 // If the mask is all ones just emit the first operation.
2385 if (const auto *C = dyn_cast<Constant>(Mask))
2386 if (C->isAllOnesValue())
2387 return Op0;
2388
2389 Mask = getX86MaskVec(Builder, Mask,
2390 cast<FixedVectorType>(Op0->getType())->getNumElements());
2391 return Builder.CreateSelect(Mask, Op0, Op1);
2392}
2393
2394static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2395 Value *Op1) {
2396 // If the mask is all ones just emit the first operation.
2397 if (const auto *C = dyn_cast<Constant>(Mask))
2398 if (C->isAllOnesValue())
2399 return Op0;
2400
2401 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2402 Mask->getType()->getIntegerBitWidth());
2403 Mask = Builder.CreateBitCast(Mask, MaskTy);
2404 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2405 return Builder.CreateSelect(Mask, Op0, Op1);
2406}
2407
2408// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2409// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2410// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2411static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2412 Value *Op1, Value *Shift,
2413 Value *Passthru, Value *Mask,
2414 bool IsVALIGN) {
2415 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2416
2417 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2418 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2419 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2420 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2421
2422 // Mask the immediate for VALIGN.
2423 if (IsVALIGN)
2424 ShiftVal &= (NumElts - 1);
2425
2426 // If palignr is shifting the pair of vectors more than the size of two
2427 // lanes, emit zero.
2428 if (ShiftVal >= 32)
2429 return llvm::Constant::getNullValue(Op0->getType());
2430
2431 // If palignr is shifting the pair of input vectors more than one lane,
2432 // but less than two lanes, convert to shifting in zeroes.
2433 if (ShiftVal > 16) {
2434 ShiftVal -= 16;
2435 Op1 = Op0;
2436 Op0 = llvm::Constant::getNullValue(Op0->getType());
2437 }
2438
2439 int Indices[64];
2440 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2441 for (unsigned l = 0; l < NumElts; l += 16) {
2442 for (unsigned i = 0; i != 16; ++i) {
2443 unsigned Idx = ShiftVal + i;
2444 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2445 Idx += NumElts - 16; // End of lane, switch operand.
2446 Indices[l + i] = Idx + l;
2447 }
2448 }
2449
2450 Value *Align = Builder.CreateShuffleVector(
2451 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2452
2453 return emitX86Select(Builder, Mask, Align, Passthru);
2454}
2455
2456static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2457 bool ZeroMask, bool IndexForm) {
2458 Type *Ty = CI.getType();
2459 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2460 unsigned EltWidth = Ty->getScalarSizeInBits();
2461 bool IsFloat = Ty->isFPOrFPVectorTy();
2462 Intrinsic::ID IID;
2463 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2464 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2465 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2466 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2467 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2468 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2469 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2470 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2471 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2472 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2473 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2474 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2475 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2476 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2477 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2478 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2479 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2480 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2481 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2482 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2483 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2484 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2485 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2486 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2487 else if (VecWidth == 128 && EltWidth == 16)
2488 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2489 else if (VecWidth == 256 && EltWidth == 16)
2490 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2491 else if (VecWidth == 512 && EltWidth == 16)
2492 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2493 else if (VecWidth == 128 && EltWidth == 8)
2494 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2495 else if (VecWidth == 256 && EltWidth == 8)
2496 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2497 else if (VecWidth == 512 && EltWidth == 8)
2498 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2499 else
2500 llvm_unreachable("Unexpected intrinsic");
2501
2502 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2503 CI.getArgOperand(2) };
2504
2505 // If this isn't index form we need to swap operand 0 and 1.
2506 if (!IndexForm)
2507 std::swap(Args[0], Args[1]);
2508
2509 Value *V = Builder.CreateIntrinsic(IID, Args);
2510 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2511 : Builder.CreateBitCast(CI.getArgOperand(1),
2512 Ty);
2513 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2514}
2515
2516static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2517 Intrinsic::ID IID) {
2518 Type *Ty = CI.getType();
2519 Value *Op0 = CI.getOperand(0);
2520 Value *Op1 = CI.getOperand(1);
2521 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2522
2523 if (CI.arg_size() == 4) { // For masked intrinsics.
2524 Value *VecSrc = CI.getOperand(2);
2525 Value *Mask = CI.getOperand(3);
2526 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2527 }
2528 return Res;
2529}
2530
2531static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2532 bool IsRotateRight) {
2533 Type *Ty = CI.getType();
2534 Value *Src = CI.getArgOperand(0);
2535 Value *Amt = CI.getArgOperand(1);
2536
2537 // Amount may be scalar immediate, in which case create a splat vector.
2538 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2539 // we only care about the lowest log2 bits anyway.
2540 if (Amt->getType() != Ty) {
2541 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2542 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2543 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2544 }
2545
2546 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2547 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2548
2549 if (CI.arg_size() == 4) { // For masked intrinsics.
2550 Value *VecSrc = CI.getOperand(2);
2551 Value *Mask = CI.getOperand(3);
2552 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2553 }
2554 return Res;
2555}
2556
2557static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2558 bool IsSigned) {
2559 Type *Ty = CI.getType();
2560 Value *LHS = CI.getArgOperand(0);
2561 Value *RHS = CI.getArgOperand(1);
2562
2563 CmpInst::Predicate Pred;
2564 switch (Imm) {
2565 case 0x0:
2566 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2567 break;
2568 case 0x1:
2569 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2570 break;
2571 case 0x2:
2572 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2573 break;
2574 case 0x3:
2575 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2576 break;
2577 case 0x4:
2578 Pred = ICmpInst::ICMP_EQ;
2579 break;
2580 case 0x5:
2581 Pred = ICmpInst::ICMP_NE;
2582 break;
2583 case 0x6:
2584 return Constant::getNullValue(Ty); // FALSE
2585 case 0x7:
2586 return Constant::getAllOnesValue(Ty); // TRUE
2587 default:
2588 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2589 }
2590
2591 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2592 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2593 return Ext;
2594}
2595
2596static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2597 bool IsShiftRight, bool ZeroMask) {
2598 Type *Ty = CI.getType();
2599 Value *Op0 = CI.getArgOperand(0);
2600 Value *Op1 = CI.getArgOperand(1);
2601 Value *Amt = CI.getArgOperand(2);
2602
2603 if (IsShiftRight)
2604 std::swap(Op0, Op1);
2605
2606 // Amount may be scalar immediate, in which case create a splat vector.
2607 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2608 // we only care about the lowest log2 bits anyway.
2609 if (Amt->getType() != Ty) {
2610 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2611 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2612 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2613 }
2614
2615 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2616 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2617
2618 unsigned NumArgs = CI.arg_size();
2619 if (NumArgs >= 4) { // For masked intrinsics.
2620 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2621 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2622 CI.getArgOperand(0);
2623 Value *Mask = CI.getOperand(NumArgs - 1);
2624 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2625 }
2626 return Res;
2627}
2628
2629static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2630 Value *Mask, bool Aligned) {
2631 const Align Alignment =
2632 Aligned
2633 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2634 : Align(1);
2635
2636 // If the mask is all ones just emit a regular store.
2637 if (const auto *C = dyn_cast<Constant>(Mask))
2638 if (C->isAllOnesValue())
2639 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2640
2641 // Convert the mask from an integer type to a vector of i1.
2642 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2643 Mask = getX86MaskVec(Builder, Mask, NumElts);
2644 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2645}
2646
2647static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2648 Value *Passthru, Value *Mask, bool Aligned) {
2649 Type *ValTy = Passthru->getType();
2650 const Align Alignment =
2651 Aligned
2652 ? Align(
2653 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2654 8)
2655 : Align(1);
2656
2657 // If the mask is all ones just emit a regular store.
2658 if (const auto *C = dyn_cast<Constant>(Mask))
2659 if (C->isAllOnesValue())
2660 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2661
2662 // Convert the mask from an integer type to a vector of i1.
2663 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2664 Mask = getX86MaskVec(Builder, Mask, NumElts);
2665 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2666}
2667
2668static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2669 Type *Ty = CI.getType();
2670 Value *Op0 = CI.getArgOperand(0);
2671 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2672 {Op0, Builder.getInt1(false)});
2673 if (CI.arg_size() == 3)
2674 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2675 return Res;
2676}
2677
2678static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2679 Type *Ty = CI.getType();
2680
2681 // Arguments have a vXi32 type so cast to vXi64.
2682 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2683 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2684
2685 if (IsSigned) {
2686 // Shift left then arithmetic shift right.
2687 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2688 LHS = Builder.CreateShl(LHS, ShiftAmt);
2689 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2690 RHS = Builder.CreateShl(RHS, ShiftAmt);
2691 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2692 } else {
2693 // Clear the upper bits.
2694 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2695 LHS = Builder.CreateAnd(LHS, Mask);
2696 RHS = Builder.CreateAnd(RHS, Mask);
2697 }
2698
2699 Value *Res = Builder.CreateMul(LHS, RHS);
2700
2701 if (CI.arg_size() == 4)
2702 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2703
2704 return Res;
2705}
2706
2707// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2708static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2709 Value *Mask) {
2710 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2711 if (Mask) {
2712 const auto *C = dyn_cast<Constant>(Mask);
2713 if (!C || !C->isAllOnesValue())
2714 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2715 }
2716
2717 if (NumElts < 8) {
2718 int Indices[8];
2719 for (unsigned i = 0; i != NumElts; ++i)
2720 Indices[i] = i;
2721 for (unsigned i = NumElts; i != 8; ++i)
2722 Indices[i] = NumElts + i % NumElts;
2723 Vec = Builder.CreateShuffleVector(Vec,
2724 Constant::getNullValue(Vec->getType()),
2725 Indices);
2726 }
2727 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2728}
2729
2730static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2731 unsigned CC, bool Signed) {
2732 Value *Op0 = CI.getArgOperand(0);
2733 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2734
2735 Value *Cmp;
2736 if (CC == 3) {
2737 Cmp = Constant::getNullValue(
2738 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2739 } else if (CC == 7) {
2740 Cmp = Constant::getAllOnesValue(
2741 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2742 } else {
2743 ICmpInst::Predicate Pred;
2744 switch (CC) {
2745 default: llvm_unreachable("Unknown condition code");
2746 case 0: Pred = ICmpInst::ICMP_EQ; break;
2747 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2748 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2749 case 4: Pred = ICmpInst::ICMP_NE; break;
2750 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2751 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2752 }
2753 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2754 }
2755
2756 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2757
2758 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2759}
2760
2761// Replace a masked intrinsic with an older unmasked intrinsic.
2762static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2763 Intrinsic::ID IID) {
2764 Value *Rep =
2765 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2766 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2767}
2768
2769static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2770 Value* A = CI.getArgOperand(0);
2771 Value* B = CI.getArgOperand(1);
2772 Value* Src = CI.getArgOperand(2);
2773 Value* Mask = CI.getArgOperand(3);
2774
2775 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2776 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2777 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2778 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2779 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2780 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2781}
2782
2783static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2784 Value* Op = CI.getArgOperand(0);
2785 Type* ReturnOp = CI.getType();
2786 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2787 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2788 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2789}
2790
2791// Replace intrinsic with unmasked version and a select.
2792static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2793 CallBase &CI, Value *&Rep) {
2794 Name = Name.substr(12); // Remove avx512.mask.
2795
2796 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2797 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2798 Intrinsic::ID IID;
2799 if (Name.starts_with("max.p")) {
2800 if (VecWidth == 128 && EltWidth == 32)
2801 IID = Intrinsic::x86_sse_max_ps;
2802 else if (VecWidth == 128 && EltWidth == 64)
2803 IID = Intrinsic::x86_sse2_max_pd;
2804 else if (VecWidth == 256 && EltWidth == 32)
2805 IID = Intrinsic::x86_avx_max_ps_256;
2806 else if (VecWidth == 256 && EltWidth == 64)
2807 IID = Intrinsic::x86_avx_max_pd_256;
2808 else
2809 llvm_unreachable("Unexpected intrinsic");
2810 } else if (Name.starts_with("min.p")) {
2811 if (VecWidth == 128 && EltWidth == 32)
2812 IID = Intrinsic::x86_sse_min_ps;
2813 else if (VecWidth == 128 && EltWidth == 64)
2814 IID = Intrinsic::x86_sse2_min_pd;
2815 else if (VecWidth == 256 && EltWidth == 32)
2816 IID = Intrinsic::x86_avx_min_ps_256;
2817 else if (VecWidth == 256 && EltWidth == 64)
2818 IID = Intrinsic::x86_avx_min_pd_256;
2819 else
2820 llvm_unreachable("Unexpected intrinsic");
2821 } else if (Name.starts_with("pshuf.b.")) {
2822 if (VecWidth == 128)
2823 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2824 else if (VecWidth == 256)
2825 IID = Intrinsic::x86_avx2_pshuf_b;
2826 else if (VecWidth == 512)
2827 IID = Intrinsic::x86_avx512_pshuf_b_512;
2828 else
2829 llvm_unreachable("Unexpected intrinsic");
2830 } else if (Name.starts_with("pmul.hr.sw.")) {
2831 if (VecWidth == 128)
2832 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2833 else if (VecWidth == 256)
2834 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2835 else if (VecWidth == 512)
2836 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2837 else
2838 llvm_unreachable("Unexpected intrinsic");
2839 } else if (Name.starts_with("pmulh.w.")) {
2840 if (VecWidth == 128)
2841 IID = Intrinsic::x86_sse2_pmulh_w;
2842 else if (VecWidth == 256)
2843 IID = Intrinsic::x86_avx2_pmulh_w;
2844 else if (VecWidth == 512)
2845 IID = Intrinsic::x86_avx512_pmulh_w_512;
2846 else
2847 llvm_unreachable("Unexpected intrinsic");
2848 } else if (Name.starts_with("pmulhu.w.")) {
2849 if (VecWidth == 128)
2850 IID = Intrinsic::x86_sse2_pmulhu_w;
2851 else if (VecWidth == 256)
2852 IID = Intrinsic::x86_avx2_pmulhu_w;
2853 else if (VecWidth == 512)
2854 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2855 else
2856 llvm_unreachable("Unexpected intrinsic");
2857 } else if (Name.starts_with("pmaddw.d.")) {
2858 if (VecWidth == 128)
2859 IID = Intrinsic::x86_sse2_pmadd_wd;
2860 else if (VecWidth == 256)
2861 IID = Intrinsic::x86_avx2_pmadd_wd;
2862 else if (VecWidth == 512)
2863 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2864 else
2865 llvm_unreachable("Unexpected intrinsic");
2866 } else if (Name.starts_with("pmaddubs.w.")) {
2867 if (VecWidth == 128)
2868 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2869 else if (VecWidth == 256)
2870 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2871 else if (VecWidth == 512)
2872 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2873 else
2874 llvm_unreachable("Unexpected intrinsic");
2875 } else if (Name.starts_with("packsswb.")) {
2876 if (VecWidth == 128)
2877 IID = Intrinsic::x86_sse2_packsswb_128;
2878 else if (VecWidth == 256)
2879 IID = Intrinsic::x86_avx2_packsswb;
2880 else if (VecWidth == 512)
2881 IID = Intrinsic::x86_avx512_packsswb_512;
2882 else
2883 llvm_unreachable("Unexpected intrinsic");
2884 } else if (Name.starts_with("packssdw.")) {
2885 if (VecWidth == 128)
2886 IID = Intrinsic::x86_sse2_packssdw_128;
2887 else if (VecWidth == 256)
2888 IID = Intrinsic::x86_avx2_packssdw;
2889 else if (VecWidth == 512)
2890 IID = Intrinsic::x86_avx512_packssdw_512;
2891 else
2892 llvm_unreachable("Unexpected intrinsic");
2893 } else if (Name.starts_with("packuswb.")) {
2894 if (VecWidth == 128)
2895 IID = Intrinsic::x86_sse2_packuswb_128;
2896 else if (VecWidth == 256)
2897 IID = Intrinsic::x86_avx2_packuswb;
2898 else if (VecWidth == 512)
2899 IID = Intrinsic::x86_avx512_packuswb_512;
2900 else
2901 llvm_unreachable("Unexpected intrinsic");
2902 } else if (Name.starts_with("packusdw.")) {
2903 if (VecWidth == 128)
2904 IID = Intrinsic::x86_sse41_packusdw;
2905 else if (VecWidth == 256)
2906 IID = Intrinsic::x86_avx2_packusdw;
2907 else if (VecWidth == 512)
2908 IID = Intrinsic::x86_avx512_packusdw_512;
2909 else
2910 llvm_unreachable("Unexpected intrinsic");
2911 } else if (Name.starts_with("vpermilvar.")) {
2912 if (VecWidth == 128 && EltWidth == 32)
2913 IID = Intrinsic::x86_avx_vpermilvar_ps;
2914 else if (VecWidth == 128 && EltWidth == 64)
2915 IID = Intrinsic::x86_avx_vpermilvar_pd;
2916 else if (VecWidth == 256 && EltWidth == 32)
2917 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2918 else if (VecWidth == 256 && EltWidth == 64)
2919 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2920 else if (VecWidth == 512 && EltWidth == 32)
2921 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2922 else if (VecWidth == 512 && EltWidth == 64)
2923 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2924 else
2925 llvm_unreachable("Unexpected intrinsic");
2926 } else if (Name == "cvtpd2dq.256") {
2927 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2928 } else if (Name == "cvtpd2ps.256") {
2929 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2930 } else if (Name == "cvttpd2dq.256") {
2931 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2932 } else if (Name == "cvttps2dq.128") {
2933 IID = Intrinsic::x86_sse2_cvttps2dq;
2934 } else if (Name == "cvttps2dq.256") {
2935 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2936 } else if (Name.starts_with("permvar.")) {
2937 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2938 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2939 IID = Intrinsic::x86_avx2_permps;
2940 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2941 IID = Intrinsic::x86_avx2_permd;
2942 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2943 IID = Intrinsic::x86_avx512_permvar_df_256;
2944 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2945 IID = Intrinsic::x86_avx512_permvar_di_256;
2946 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2947 IID = Intrinsic::x86_avx512_permvar_sf_512;
2948 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2949 IID = Intrinsic::x86_avx512_permvar_si_512;
2950 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2951 IID = Intrinsic::x86_avx512_permvar_df_512;
2952 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2953 IID = Intrinsic::x86_avx512_permvar_di_512;
2954 else if (VecWidth == 128 && EltWidth == 16)
2955 IID = Intrinsic::x86_avx512_permvar_hi_128;
2956 else if (VecWidth == 256 && EltWidth == 16)
2957 IID = Intrinsic::x86_avx512_permvar_hi_256;
2958 else if (VecWidth == 512 && EltWidth == 16)
2959 IID = Intrinsic::x86_avx512_permvar_hi_512;
2960 else if (VecWidth == 128 && EltWidth == 8)
2961 IID = Intrinsic::x86_avx512_permvar_qi_128;
2962 else if (VecWidth == 256 && EltWidth == 8)
2963 IID = Intrinsic::x86_avx512_permvar_qi_256;
2964 else if (VecWidth == 512 && EltWidth == 8)
2965 IID = Intrinsic::x86_avx512_permvar_qi_512;
2966 else
2967 llvm_unreachable("Unexpected intrinsic");
2968 } else if (Name.starts_with("dbpsadbw.")) {
2969 if (VecWidth == 128)
2970 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2971 else if (VecWidth == 256)
2972 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2973 else if (VecWidth == 512)
2974 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2975 else
2976 llvm_unreachable("Unexpected intrinsic");
2977 } else if (Name.starts_with("pmultishift.qb.")) {
2978 if (VecWidth == 128)
2979 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2980 else if (VecWidth == 256)
2981 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2982 else if (VecWidth == 512)
2983 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2984 else
2985 llvm_unreachable("Unexpected intrinsic");
2986 } else if (Name.starts_with("conflict.")) {
2987 if (Name[9] == 'd' && VecWidth == 128)
2988 IID = Intrinsic::x86_avx512_conflict_d_128;
2989 else if (Name[9] == 'd' && VecWidth == 256)
2990 IID = Intrinsic::x86_avx512_conflict_d_256;
2991 else if (Name[9] == 'd' && VecWidth == 512)
2992 IID = Intrinsic::x86_avx512_conflict_d_512;
2993 else if (Name[9] == 'q' && VecWidth == 128)
2994 IID = Intrinsic::x86_avx512_conflict_q_128;
2995 else if (Name[9] == 'q' && VecWidth == 256)
2996 IID = Intrinsic::x86_avx512_conflict_q_256;
2997 else if (Name[9] == 'q' && VecWidth == 512)
2998 IID = Intrinsic::x86_avx512_conflict_q_512;
2999 else
3000 llvm_unreachable("Unexpected intrinsic");
3001 } else if (Name.starts_with("pavg.")) {
3002 if (Name[5] == 'b' && VecWidth == 128)
3003 IID = Intrinsic::x86_sse2_pavg_b;
3004 else if (Name[5] == 'b' && VecWidth == 256)
3005 IID = Intrinsic::x86_avx2_pavg_b;
3006 else if (Name[5] == 'b' && VecWidth == 512)
3007 IID = Intrinsic::x86_avx512_pavg_b_512;
3008 else if (Name[5] == 'w' && VecWidth == 128)
3009 IID = Intrinsic::x86_sse2_pavg_w;
3010 else if (Name[5] == 'w' && VecWidth == 256)
3011 IID = Intrinsic::x86_avx2_pavg_w;
3012 else if (Name[5] == 'w' && VecWidth == 512)
3013 IID = Intrinsic::x86_avx512_pavg_w_512;
3014 else
3015 llvm_unreachable("Unexpected intrinsic");
3016 } else
3017 return false;
3018
3019 SmallVector<Value *, 4> Args(CI.args());
3020 Args.pop_back();
3021 Args.pop_back();
3022 Rep = Builder.CreateIntrinsic(IID, Args);
3023 unsigned NumArgs = CI.arg_size();
3024 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3025 CI.getArgOperand(NumArgs - 2));
3026 return true;
3027}
3028
3029/// Upgrade comment in call to inline asm that represents an objc retain release
3030/// marker.
3031void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3032 size_t Pos;
3033 if (AsmStr->find("mov\tfp") == 0 &&
3034 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3035 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3036 AsmStr->replace(Pos, 1, ";");
3037 }
3038}
3039
3040static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3041 Function *F, IRBuilder<> &Builder) {
3042 Value *Rep = nullptr;
3043
3044 if (Name == "abs.i" || Name == "abs.ll") {
3045 Value *Arg = CI->getArgOperand(0);
3046 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3047 {Arg, Builder.getTrue()},
3048 /*FMFSource=*/nullptr, "abs");
3049 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3050 Type *Ty = (Name == "abs.bf16")
3051 ? Builder.getBFloatTy()
3052 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3053 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3054 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3055 Rep = Builder.CreateBitCast(Abs, CI->getType());
3056 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3057 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3058 : Intrinsic::nvvm_fabs;
3059 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3060 } else if (Name.consume_front("ex2.approx.")) {
3061 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3062 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3063 : Intrinsic::nvvm_ex2_approx;
3064 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3065 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3066 Name.starts_with("atomic.load.add.f64.p")) {
3067 Value *Ptr = CI->getArgOperand(0);
3068 Value *Val = CI->getArgOperand(1);
3069 Rep = Builder.CreateAtomicRMW(
3070 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3071 CI->getContext().getOrInsertSyncScopeID("device"));
3072 // The default scope for atomic.load.* intrinsics is device
3073 // (= gpu scope in ptx), but the default LLVM atomic scope is
3074 // "system"
3075 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3076 Name.starts_with("atomic.load.dec.32.p")) {
3077 Value *Ptr = CI->getArgOperand(0);
3078 Value *Val = CI->getArgOperand(1);
3079 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3080 : AtomicRMWInst::UDecWrap;
3081 Rep = Builder.CreateAtomicRMW(
3082 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3083 CI->getContext().getOrInsertSyncScopeID("device"));
3084 // See comment above.
3085 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3086 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3087 StringRef Op = Name.substr(StringRef("atomic.").size());
3088 Value *Ptr = CI->getArgOperand(0);
3089 Value *Val = CI->getArgOperand(1);
3090 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3091 Op.contains(".cta.") ? "block" : "");
3092 if (Op.starts_with("cas.")) {
3093 Value *New = CI->getArgOperand(2);
3094 Value *Pair = Builder.CreateAtomicCmpXchg(
3095 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3096 AtomicOrdering::Monotonic, SSID);
3097 Rep = Builder.CreateExtractValue(Pair, 0);
3098 } else {
3099 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3100 // because we were actually missing those intrinsics!
3101 AtomicRMWInst::BinOp BinOp =
3102 StringSwitch<AtomicRMWInst::BinOp>(Op)
3103 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3104 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3105 .StartsWith("exch.", AtomicRMWInst::Xchg)
3106 .StartsWith("max.", AtomicRMWInst::Max)
3107 .StartsWith("min.", AtomicRMWInst::Min)
3108 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3109 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3110 .StartsWith("and.", AtomicRMWInst::And)
3111 .StartsWith("or.", AtomicRMWInst::Or)
3112 .StartsWith("xor.", AtomicRMWInst::Xor)
3113 .Default(AtomicRMWInst::BAD_BINOP);
3114 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3115 "unexpected nvvm scoped atomic intrinsic");
3116 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3117 AtomicOrdering::Monotonic, SSID);
3118 }
3119 } else if (Name == "clz.ll") {
3120 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3121 Value *Arg = CI->getArgOperand(0);
3122 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3123 {Arg, Builder.getFalse()},
3124 /*FMFSource=*/nullptr, "ctlz");
3125 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3126 } else if (Name == "popc.ll") {
3127 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3128 // i64.
3129 Value *Arg = CI->getArgOperand(0);
3130 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3131 Arg, /*FMFSource=*/nullptr, "ctpop");
3132 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3133 } else if (Name == "h2f") {
3134 Value *Cast =
3135 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3136 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3137 } else if (Name.consume_front("bitcast.") &&
3138 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3139 Name == "d2ll")) {
3140 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3141 } else if (Name == "rotate.b32") {
3142 Value *Arg = CI->getOperand(0);
3143 Value *ShiftAmt = CI->getOperand(1);
3144 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3145 {Arg, Arg, ShiftAmt});
3146 } else if (Name == "rotate.b64") {
3147 Type *Int64Ty = Builder.getInt64Ty();
3148 Value *Arg = CI->getOperand(0);
3149 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3150 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3151 {Arg, Arg, ZExtShiftAmt});
3152 } else if (Name == "rotate.right.b64") {
3153 Type *Int64Ty = Builder.getInt64Ty();
3154 Value *Arg = CI->getOperand(0);
3155 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3156 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3157 {Arg, Arg, ZExtShiftAmt});
3158 } else if (Name == "swap.lo.hi.b64") {
3159 Type *Int64Ty = Builder.getInt64Ty();
3160 Value *Arg = CI->getOperand(0);
3161 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3162 {Arg, Arg, Builder.getInt64(32)});
3163 } else if ((Name.consume_front("ptr.gen.to.") &&
3164 consumeNVVMPtrAddrSpace(Name)) ||
3165 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3166 Name.starts_with(".to.gen"))) {
3167 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3168 } else if (Name.consume_front("ldg.global")) {
3169 Value *Ptr = CI->getArgOperand(0);
3170 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3171 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3172 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3173 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3174 MDNode *MD = MDNode::get(Builder.getContext(), {});
3175 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3176 return LD;
3177 } else if (Name == "tanh.approx.f32") {
3178 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3179 FastMathFlags FMF;
3180 FMF.setApproxFunc();
3181 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3182 FMF);
3183 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3184 Value *Arg =
3185 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3186 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3187 {}, {Arg});
3188 } else if (Name == "barrier") {
3189 Rep = Builder.CreateIntrinsic(
3190 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3191 {CI->getArgOperand(0), CI->getArgOperand(1)});
3192 } else if (Name == "barrier.sync") {
3193 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3194 {CI->getArgOperand(0)});
3195 } else if (Name == "barrier.sync.cnt") {
3196 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3197 {CI->getArgOperand(0), CI->getArgOperand(1)});
3198 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3199 Name == "barrier0.or") {
3200 Value *C = CI->getArgOperand(0);
3201 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3202
3203 Intrinsic::ID IID =
3204 StringSwitch<Intrinsic::ID>(Name)
3205 .Case("barrier0.popc",
3206 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3207 .Case("barrier0.and",
3208 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3209 .Case("barrier0.or",
3210 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3211 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3212 Rep = Builder.CreateZExt(Bar, CI->getType());
3213 } else {
3214 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3215 if (IID != Intrinsic::not_intrinsic &&
3216 !F->getReturnType()->getScalarType()->isBFloatTy()) {
3217 rename(F);
3218 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3219 SmallVector<Value *, 2> Args;
3220 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3221 Value *Arg = CI->getArgOperand(I);
3222 Type *OldType = Arg->getType();
3223 Type *NewType = NewFn->getArg(I)->getType();
3224 Args.push_back(
3225 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3226 ? Builder.CreateBitCast(Arg, NewType)
3227 : Arg);
3228 }
3229 Rep = Builder.CreateCall(NewFn, Args);
3230 if (F->getReturnType()->isIntegerTy())
3231 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3232 }
3233 }
3234
3235 return Rep;
3236}
3237
3238static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3239 IRBuilder<> &Builder) {
3240 LLVMContext &C = F->getContext();
3241 Value *Rep = nullptr;
3242
3243 if (Name.starts_with("sse4a.movnt.")) {
3244 SmallVector<Metadata *, 1> Elts;
3245 Elts.push_back(
3246 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3247 MDNode *Node = MDNode::get(C, Elts);
3248
3249 Value *Arg0 = CI->getArgOperand(0);
3250 Value *Arg1 = CI->getArgOperand(1);
3251
3252 // Nontemporal (unaligned) store of the 0'th element of the float/double
3253 // vector.
3254 Value *Extract =
3255 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3256
3257 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3258 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3259 } else if (Name.starts_with("avx.movnt.") ||
3260 Name.starts_with("avx512.storent.")) {
3261 SmallVector<Metadata *, 1> Elts;
3262 Elts.push_back(
3263 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3264 MDNode *Node = MDNode::get(C, Elts);
3265
3266 Value *Arg0 = CI->getArgOperand(0);
3267 Value *Arg1 = CI->getArgOperand(1);
3268
3269 StoreInst *SI = Builder.CreateAlignedStore(
3270 Arg1, Arg0,
3271 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3272 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3273 } else if (Name == "sse2.storel.dq") {
3274 Value *Arg0 = CI->getArgOperand(0);
3275 Value *Arg1 = CI->getArgOperand(1);
3276
3277 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3278 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3279 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3280 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3281 } else if (Name.starts_with("sse.storeu.") ||
3282 Name.starts_with("sse2.storeu.") ||
3283 Name.starts_with("avx.storeu.")) {
3284 Value *Arg0 = CI->getArgOperand(0);
3285 Value *Arg1 = CI->getArgOperand(1);
3286 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3287 } else if (Name == "avx512.mask.store.ss") {
3288 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3289 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3290 Mask, false);
3291 } else if (Name.starts_with("avx512.mask.store")) {
3292 // "avx512.mask.storeu." or "avx512.mask.store."
3293 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3294 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3295 CI->getArgOperand(2), Aligned);
3296 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3297 // Upgrade packed integer vector compare intrinsics to compare instructions.
3298 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3299 bool CmpEq = Name[9] == 'e';
3300 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3301 CI->getArgOperand(0), CI->getArgOperand(1));
3302 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3303 } else if (Name.starts_with("avx512.broadcastm")) {
3304 Type *ExtTy = Type::getInt32Ty(C);
3305 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3306 ExtTy = Type::getInt64Ty(C);
3307 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3308 ExtTy->getPrimitiveSizeInBits();
3309 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3310 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3311 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3312 Value *Vec = CI->getArgOperand(0);
3313 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3314 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3315 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3316 } else if (Name.starts_with("avx.sqrt.p") ||
3317 Name.starts_with("sse2.sqrt.p") ||
3318 Name.starts_with("sse.sqrt.p")) {
3319 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3320 {CI->getArgOperand(0)});
3321 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3322 if (CI->arg_size() == 4 &&
3323 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3324 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3325 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3326 : Intrinsic::x86_avx512_sqrt_pd_512;
3327
3328 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3329 Rep = Builder.CreateIntrinsic(IID, Args);
3330 } else {
3331 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3332 {CI->getArgOperand(0)});
3333 }
3334 Rep =
3335 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3336 } else if (Name.starts_with("avx512.ptestm") ||
3337 Name.starts_with("avx512.ptestnm")) {
3338 Value *Op0 = CI->getArgOperand(0);
3339 Value *Op1 = CI->getArgOperand(1);
3340 Value *Mask = CI->getArgOperand(2);
3341 Rep = Builder.CreateAnd(Op0, Op1);
3342 llvm::Type *Ty = Op0->getType();
3343 Value *Zero = llvm::Constant::getNullValue(Ty);
3344 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3345 ? ICmpInst::ICMP_NE
3346 : ICmpInst::ICMP_EQ;
3347 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3348 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3349 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3350 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3351 ->getNumElements();
3352 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3353 Rep =
3354 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3355 } else if (Name.starts_with("avx512.kunpck")) {
3356 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3357 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3358 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3359 int Indices[64];
3360 for (unsigned i = 0; i != NumElts; ++i)
3361 Indices[i] = i;
3362
3363 // First extract half of each vector. This gives better codegen than
3364 // doing it in a single shuffle.
3365 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3366 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3367 // Concat the vectors.
3368 // NOTE: Operands have to be swapped to match intrinsic definition.
3369 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3370 Rep = Builder.CreateBitCast(Rep, CI->getType());
3371 } else if (Name == "avx512.kand.w") {
3372 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3373 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3374 Rep = Builder.CreateAnd(LHS, RHS);
3375 Rep = Builder.CreateBitCast(Rep, CI->getType());
3376 } else if (Name == "avx512.kandn.w") {
3377 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3378 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3379 LHS = Builder.CreateNot(LHS);
3380 Rep = Builder.CreateAnd(LHS, RHS);
3381 Rep = Builder.CreateBitCast(Rep, CI->getType());
3382 } else if (Name == "avx512.kor.w") {
3383 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3384 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3385 Rep = Builder.CreateOr(LHS, RHS);
3386 Rep = Builder.CreateBitCast(Rep, CI->getType());
3387 } else if (Name == "avx512.kxor.w") {
3388 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3389 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3390 Rep = Builder.CreateXor(LHS, RHS);
3391 Rep = Builder.CreateBitCast(Rep, CI->getType());
3392 } else if (Name == "avx512.kxnor.w") {
3393 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3394 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3395 LHS = Builder.CreateNot(LHS);
3396 Rep = Builder.CreateXor(LHS, RHS);
3397 Rep = Builder.CreateBitCast(Rep, CI->getType());
3398 } else if (Name == "avx512.knot.w") {
3399 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3400 Rep = Builder.CreateNot(Rep);
3401 Rep = Builder.CreateBitCast(Rep, CI->getType());
3402 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3403 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3404 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3405 Rep = Builder.CreateOr(LHS, RHS);
3406 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3407 Value *C;
3408 if (Name[14] == 'c')
3409 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3410 else
3411 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3412 Rep = Builder.CreateICmpEQ(Rep, C);
3413 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3414 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3415 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3416 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3417 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3418 Type *I32Ty = Type::getInt32Ty(C);
3419 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3420 ConstantInt::get(I32Ty, 0));
3421 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3422 ConstantInt::get(I32Ty, 0));
3423 Value *EltOp;
3424 if (Name.contains(".add."))
3425 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3426 else if (Name.contains(".sub."))
3427 EltOp = Builder.CreateFSub(Elt0, Elt1);
3428 else if (Name.contains(".mul."))
3429 EltOp = Builder.CreateFMul(Elt0, Elt1);
3430 else
3431 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3432 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3433 ConstantInt::get(I32Ty, 0));
3434 } else if (Name.starts_with("avx512.mask.pcmp")) {
3435 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3436 bool CmpEq = Name[16] == 'e';
3437 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3438 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3439 Type *OpTy = CI->getArgOperand(0)->getType();
3440 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3441 Intrinsic::ID IID;
3442 switch (VecWidth) {
3443 default:
3444 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3445 break;
3446 case 128:
3447 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3448 break;
3449 case 256:
3450 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3451 break;
3452 case 512:
3453 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3454 break;
3455 }
3456
3457 Rep =
3458 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3459 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3460 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3461 Type *OpTy = CI->getArgOperand(0)->getType();
3462 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3463 unsigned EltWidth = OpTy->getScalarSizeInBits();
3464 Intrinsic::ID IID;
3465 if (VecWidth == 128 && EltWidth == 32)
3466 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3467 else if (VecWidth == 256 && EltWidth == 32)
3468 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3469 else if (VecWidth == 512 && EltWidth == 32)
3470 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3471 else if (VecWidth == 128 && EltWidth == 64)
3472 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3473 else if (VecWidth == 256 && EltWidth == 64)
3474 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3475 else if (VecWidth == 512 && EltWidth == 64)
3476 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3477 else
3478 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3479
3480 Rep =
3481 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3482 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3483 } else if (Name.starts_with("avx512.cmp.p")) {
3484 SmallVector<Value *, 4> Args(CI->args());
3485 Type *OpTy = Args[0]->getType();
3486 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3487 unsigned EltWidth = OpTy->getScalarSizeInBits();
3488 Intrinsic::ID IID;
3489 if (VecWidth == 128 && EltWidth == 32)
3490 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3491 else if (VecWidth == 256 && EltWidth == 32)
3492 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3493 else if (VecWidth == 512 && EltWidth == 32)
3494 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3495 else if (VecWidth == 128 && EltWidth == 64)
3496 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3497 else if (VecWidth == 256 && EltWidth == 64)
3498 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3499 else if (VecWidth == 512 && EltWidth == 64)
3500 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3501 else
3502 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3503
3504 Value *Mask = Constant::getAllOnesValue(CI->getType());
3505 if (VecWidth == 512)
3506 std::swap(Mask, Args.back());
3507 Args.push_back(Mask);
3508
3509 Rep = Builder.CreateIntrinsic(IID, Args);
3510 } else if (Name.starts_with("avx512.mask.cmp.")) {
3511 // Integer compare intrinsics.
3512 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3513 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3514 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3515 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3516 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3517 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3518 Name.starts_with("avx512.cvtw2mask.") ||
3519 Name.starts_with("avx512.cvtd2mask.") ||
3520 Name.starts_with("avx512.cvtq2mask.")) {
3521 Value *Op = CI->getArgOperand(0);
3522 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3523 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3524 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3525 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3526 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3527 Name.starts_with("avx512.mask.pabs")) {
3528 Rep = upgradeAbs(Builder, *CI);
3529 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3530 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3531 Name.starts_with("avx512.mask.pmaxs")) {
3532 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3533 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3534 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3535 Name.starts_with("avx512.mask.pmaxu")) {
3536 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3537 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3538 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3539 Name.starts_with("avx512.mask.pmins")) {
3540 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3541 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3542 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3543 Name.starts_with("avx512.mask.pminu")) {
3544 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3545 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3546 Name == "avx512.pmulu.dq.512" ||
3547 Name.starts_with("avx512.mask.pmulu.dq.")) {
3548 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3549 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3550 Name == "avx512.pmul.dq.512" ||
3551 Name.starts_with("avx512.mask.pmul.dq.")) {
3552 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3553 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3554 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3555 Rep =
3556 Builder.CreateSIToFP(CI->getArgOperand(1),
3557 cast<VectorType>(CI->getType())->getElementType());
3558 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3559 } else if (Name == "avx512.cvtusi2sd") {
3560 Rep =
3561 Builder.CreateUIToFP(CI->getArgOperand(1),
3562 cast<VectorType>(CI->getType())->getElementType());
3563 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3564 } else if (Name == "sse2.cvtss2sd") {
3565 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3566 Rep = Builder.CreateFPExt(
3567 Rep, cast<VectorType>(CI->getType())->getElementType());
3568 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3569 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3570 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3571 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3572 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3573 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3574 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3575 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3576 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3577 Name == "avx512.mask.cvtqq2ps.256" ||
3578 Name == "avx512.mask.cvtqq2ps.512" ||
3579 Name == "avx512.mask.cvtuqq2ps.256" ||
3580 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3581 Name == "avx.cvt.ps2.pd.256" ||
3582 Name == "avx512.mask.cvtps2pd.128" ||
3583 Name == "avx512.mask.cvtps2pd.256") {
3584 auto *DstTy = cast<FixedVectorType>(CI->getType());
3585 Rep = CI->getArgOperand(0);
3586 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3587
3588 unsigned NumDstElts = DstTy->getNumElements();
3589 if (NumDstElts < SrcTy->getNumElements()) {
3590 assert(NumDstElts == 2 && "Unexpected vector size");
3591 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3592 }
3593
3594 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3595 bool IsUnsigned = Name.contains("cvtu");
3596 if (IsPS2PD)
3597 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3598 else if (CI->arg_size() == 4 &&
3599 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3600 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3601 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3602 : Intrinsic::x86_avx512_sitofp_round;
3603 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3604 {Rep, CI->getArgOperand(3)});
3605 } else {
3606 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3607 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3608 }
3609
3610 if (CI->arg_size() >= 3)
3611 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3612 CI->getArgOperand(1));
3613 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3614 Name.starts_with("vcvtph2ps.")) {
3615 auto *DstTy = cast<FixedVectorType>(CI->getType());
3616 Rep = CI->getArgOperand(0);
3617 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3618 unsigned NumDstElts = DstTy->getNumElements();
3619 if (NumDstElts != SrcTy->getNumElements()) {
3620 assert(NumDstElts == 4 && "Unexpected vector size");
3621 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3622 }
3623 Rep = Builder.CreateBitCast(
3624 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3625 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3626 if (CI->arg_size() >= 3)
3627 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3628 CI->getArgOperand(1));
3629 } else if (Name.starts_with("avx512.mask.load")) {
3630 // "avx512.mask.loadu." or "avx512.mask.load."
3631 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3632 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3633 CI->getArgOperand(2), Aligned);
3634 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3635 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3636 auto *PtrTy = CI->getOperand(0)->getType();
3637 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3638 ResultTy->getNumElements());
3639 Rep = Builder.CreateIntrinsic(
3640 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3641 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3642 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3643 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3644 auto *PtrTy = CI->getArgOperand(0)->getType();
3645 Value *MaskVec =
3646 getX86MaskVec(Builder, CI->getArgOperand(2),
3647 cast<FixedVectorType>(ResultTy)->getNumElements());
3648 Rep = Builder.CreateIntrinsic(
3649 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3650 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3651 } else if (Name.starts_with("avx512.mask.compress.") ||
3652 Name.starts_with("avx512.mask.expand.")) {
3653 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3654
3655 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3656 ResultTy->getNumElements());
3657
3658 bool IsCompress = Name[12] == 'c';
3659 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3660 : Intrinsic::x86_avx512_mask_expand;
3661 Rep = Builder.CreateIntrinsic(
3662 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3663 } else if (Name.starts_with("xop.vpcom")) {
3664 bool IsSigned;
3665 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3666 Name.ends_with("uq"))
3667 IsSigned = false;
3668 else if (Name.ends_with("b") || Name.ends_with("w") ||
3669 Name.ends_with("d") || Name.ends_with("q"))
3670 IsSigned = true;
3671 else
3672 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3673
3674 unsigned Imm;
3675 if (CI->arg_size() == 3) {
3676 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3677 } else {
3678 Name = Name.substr(9); // strip off "xop.vpcom"
3679 if (Name.starts_with("lt"))
3680 Imm = 0;
3681 else if (Name.starts_with("le"))
3682 Imm = 1;
3683 else if (Name.starts_with("gt"))
3684 Imm = 2;
3685 else if (Name.starts_with("ge"))
3686 Imm = 3;
3687 else if (Name.starts_with("eq"))
3688 Imm = 4;
3689 else if (Name.starts_with("ne"))
3690 Imm = 5;
3691 else if (Name.starts_with("false"))
3692 Imm = 6;
3693 else if (Name.starts_with("true"))
3694 Imm = 7;
3695 else
3696 llvm_unreachable("Unknown condition");
3697 }
3698
3699 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3700 } else if (Name.starts_with("xop.vpcmov")) {
3701 Value *Sel = CI->getArgOperand(2);
3702 Value *NotSel = Builder.CreateNot(Sel);
3703 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3704 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3705 Rep = Builder.CreateOr(Sel0, Sel1);
3706 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3707 Name.starts_with("avx512.mask.prol")) {
3708 Rep = upgradeX86Rotate(Builder, *CI, false);
3709 } else if (Name.starts_with("avx512.pror") ||
3710 Name.starts_with("avx512.mask.pror")) {
3711 Rep = upgradeX86Rotate(Builder, *CI, true);
3712 } else if (Name.starts_with("avx512.vpshld.") ||
3713 Name.starts_with("avx512.mask.vpshld") ||
3714 Name.starts_with("avx512.maskz.vpshld")) {
3715 bool ZeroMask = Name[11] == 'z';
3716 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3717 } else if (Name.starts_with("avx512.vpshrd.") ||
3718 Name.starts_with("avx512.mask.vpshrd") ||
3719 Name.starts_with("avx512.maskz.vpshrd")) {
3720 bool ZeroMask = Name[11] == 'z';
3721 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3722 } else if (Name == "sse42.crc32.64.8") {
3723 Value *Trunc0 =
3724 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3725 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3726 {Trunc0, CI->getArgOperand(1)});
3727 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3728 } else if (Name.starts_with("avx.vbroadcast.s") ||
3729 Name.starts_with("avx512.vbroadcast.s")) {
3730 // Replace broadcasts with a series of insertelements.
3731 auto *VecTy = cast<FixedVectorType>(CI->getType());
3732 Type *EltTy = VecTy->getElementType();
3733 unsigned EltNum = VecTy->getNumElements();
3734 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3735 Type *I32Ty = Type::getInt32Ty(C);
3736 Rep = PoisonValue::get(VecTy);
3737 for (unsigned I = 0; I < EltNum; ++I)
3738 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3739 } else if (Name.starts_with("sse41.pmovsx") ||
3740 Name.starts_with("sse41.pmovzx") ||
3741 Name.starts_with("avx2.pmovsx") ||
3742 Name.starts_with("avx2.pmovzx") ||
3743 Name.starts_with("avx512.mask.pmovsx") ||
3744 Name.starts_with("avx512.mask.pmovzx")) {
3745 auto *DstTy = cast<FixedVectorType>(CI->getType());
3746 unsigned NumDstElts = DstTy->getNumElements();
3747
3748 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3749 SmallVector<int, 8> ShuffleMask(NumDstElts);
3750 for (unsigned i = 0; i != NumDstElts; ++i)
3751 ShuffleMask[i] = i;
3752
3753 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3754
3755 bool DoSext = Name.contains("pmovsx");
3756 Rep =
3757 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3758 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3759 if (CI->arg_size() == 3)
3760 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3761 CI->getArgOperand(1));
3762 } else if (Name == "avx512.mask.pmov.qd.256" ||
3763 Name == "avx512.mask.pmov.qd.512" ||
3764 Name == "avx512.mask.pmov.wb.256" ||
3765 Name == "avx512.mask.pmov.wb.512") {
3766 Type *Ty = CI->getArgOperand(1)->getType();
3767 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3768 Rep =
3769 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3770 } else if (Name.starts_with("avx.vbroadcastf128") ||
3771 Name == "avx2.vbroadcasti128") {
3772 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3773 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3774 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3775 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3776 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3777 if (NumSrcElts == 2)
3778 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3779 else
3780 Rep = Builder.CreateShuffleVector(Load,
3781 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3782 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3783 Name.starts_with("avx512.mask.shuf.f")) {
3784 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3785 Type *VT = CI->getType();
3786 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3787 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3788 unsigned ControlBitsMask = NumLanes - 1;
3789 unsigned NumControlBits = NumLanes / 2;
3790 SmallVector<int, 8> ShuffleMask(0);
3791
3792 for (unsigned l = 0; l != NumLanes; ++l) {
3793 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3794 // We actually need the other source.
3795 if (l >= NumLanes / 2)
3796 LaneMask += NumLanes;
3797 for (unsigned i = 0; i != NumElementsInLane; ++i)
3798 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3799 }
3800 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3801 CI->getArgOperand(1), ShuffleMask);
3802 Rep =
3803 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3804 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3805 Name.starts_with("avx512.mask.broadcasti")) {
3806 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3807 ->getNumElements();
3808 unsigned NumDstElts =
3809 cast<FixedVectorType>(CI->getType())->getNumElements();
3810
3811 SmallVector<int, 8> ShuffleMask(NumDstElts);
3812 for (unsigned i = 0; i != NumDstElts; ++i)
3813 ShuffleMask[i] = i % NumSrcElts;
3814
3815 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3816 CI->getArgOperand(0), ShuffleMask);
3817 Rep =
3818 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3819 } else if (Name.starts_with("avx2.pbroadcast") ||
3820 Name.starts_with("avx2.vbroadcast") ||
3821 Name.starts_with("avx512.pbroadcast") ||
3822 Name.starts_with("avx512.mask.broadcast.s")) {
3823 // Replace vp?broadcasts with a vector shuffle.
3824 Value *Op = CI->getArgOperand(0);
3825 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3826 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3827 SmallVector<int, 8> M;
3828 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3829 Rep = Builder.CreateShuffleVector(Op, M);
3830
3831 if (CI->arg_size() == 3)
3832 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3833 CI->getArgOperand(1));
3834 } else if (Name.starts_with("sse2.padds.") ||
3835 Name.starts_with("avx2.padds.") ||
3836 Name.starts_with("avx512.padds.") ||
3837 Name.starts_with("avx512.mask.padds.")) {
3838 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3839 } else if (Name.starts_with("sse2.psubs.") ||
3840 Name.starts_with("avx2.psubs.") ||
3841 Name.starts_with("avx512.psubs.") ||
3842 Name.starts_with("avx512.mask.psubs.")) {
3843 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3844 } else if (Name.starts_with("sse2.paddus.") ||
3845 Name.starts_with("avx2.paddus.") ||
3846 Name.starts_with("avx512.mask.paddus.")) {
3847 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3848 } else if (Name.starts_with("sse2.psubus.") ||
3849 Name.starts_with("avx2.psubus.") ||
3850 Name.starts_with("avx512.mask.psubus.")) {
3851 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3852 } else if (Name.starts_with("avx512.mask.palignr.")) {
3853 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3854 CI->getArgOperand(1), CI->getArgOperand(2),
3855 CI->getArgOperand(3), CI->getArgOperand(4),
3856 false);
3857 } else if (Name.starts_with("avx512.mask.valign.")) {
3858 Rep = upgradeX86ALIGNIntrinsics(
3859 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3860 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3861 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3862 // 128/256-bit shift left specified in bits.
3863 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3864 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3865 Shift / 8); // Shift is in bits.
3866 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3867 // 128/256-bit shift right specified in bits.
3868 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3869 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3870 Shift / 8); // Shift is in bits.
3871 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3872 Name == "avx512.psll.dq.512") {
3873 // 128/256/512-bit shift left specified in bytes.
3874 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3875 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3876 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3877 Name == "avx512.psrl.dq.512") {
3878 // 128/256/512-bit shift right specified in bytes.
3879 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3880 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3881 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3882 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3883 Name.starts_with("avx2.pblendd.")) {
3884 Value *Op0 = CI->getArgOperand(0);
3885 Value *Op1 = CI->getArgOperand(1);
3886 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3887 auto *VecTy = cast<FixedVectorType>(CI->getType());
3888 unsigned NumElts = VecTy->getNumElements();
3889
3890 SmallVector<int, 16> Idxs(NumElts);
3891 for (unsigned i = 0; i != NumElts; ++i)
3892 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3893
3894 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3895 } else if (Name.starts_with("avx.vinsertf128.") ||
3896 Name == "avx2.vinserti128" ||
3897 Name.starts_with("avx512.mask.insert")) {
3898 Value *Op0 = CI->getArgOperand(0);
3899 Value *Op1 = CI->getArgOperand(1);
3900 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3901 unsigned DstNumElts =
3902 cast<FixedVectorType>(CI->getType())->getNumElements();
3903 unsigned SrcNumElts =
3904 cast<FixedVectorType>(Op1->getType())->getNumElements();
3905 unsigned Scale = DstNumElts / SrcNumElts;
3906
3907 // Mask off the high bits of the immediate value; hardware ignores those.
3908 Imm = Imm % Scale;
3909
3910 // Extend the second operand into a vector the size of the destination.
3911 SmallVector<int, 8> Idxs(DstNumElts);
3912 for (unsigned i = 0; i != SrcNumElts; ++i)
3913 Idxs[i] = i;
3914 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3915 Idxs[i] = SrcNumElts;
3916 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3917
3918 // Insert the second operand into the first operand.
3919
3920 // Note that there is no guarantee that instruction lowering will actually
3921 // produce a vinsertf128 instruction for the created shuffles. In
3922 // particular, the 0 immediate case involves no lane changes, so it can
3923 // be handled as a blend.
3924
3925 // Example of shuffle mask for 32-bit elements:
3926 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3927 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3928
3929 // First fill with identify mask.
3930 for (unsigned i = 0; i != DstNumElts; ++i)
3931 Idxs[i] = i;
3932 // Then replace the elements where we need to insert.
3933 for (unsigned i = 0; i != SrcNumElts; ++i)
3934 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3935 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3936
3937 // If the intrinsic has a mask operand, handle that.
3938 if (CI->arg_size() == 5)
3939 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3940 CI->getArgOperand(3));
3941 } else if (Name.starts_with("avx.vextractf128.") ||
3942 Name == "avx2.vextracti128" ||
3943 Name.starts_with("avx512.mask.vextract")) {
3944 Value *Op0 = CI->getArgOperand(0);
3945 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3946 unsigned DstNumElts =
3947 cast<FixedVectorType>(CI->getType())->getNumElements();
3948 unsigned SrcNumElts =
3949 cast<FixedVectorType>(Op0->getType())->getNumElements();
3950 unsigned Scale = SrcNumElts / DstNumElts;
3951
3952 // Mask off the high bits of the immediate value; hardware ignores those.
3953 Imm = Imm % Scale;
3954
3955 // Get indexes for the subvector of the input vector.
3956 SmallVector<int, 8> Idxs(DstNumElts);
3957 for (unsigned i = 0; i != DstNumElts; ++i) {
3958 Idxs[i] = i + (Imm * DstNumElts);
3959 }
3960 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3961
3962 // If the intrinsic has a mask operand, handle that.
3963 if (CI->arg_size() == 4)
3964 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3965 CI->getArgOperand(2));
3966 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3967 Name.starts_with("avx512.mask.perm.di.")) {
3968 Value *Op0 = CI->getArgOperand(0);
3969 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3970 auto *VecTy = cast<FixedVectorType>(CI->getType());
3971 unsigned NumElts = VecTy->getNumElements();
3972
3973 SmallVector<int, 8> Idxs(NumElts);
3974 for (unsigned i = 0; i != NumElts; ++i)
3975 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3976
3977 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3978
3979 if (CI->arg_size() == 4)
3980 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3981 CI->getArgOperand(2));
3982 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3983 // The immediate permute control byte looks like this:
3984 // [1:0] - select 128 bits from sources for low half of destination
3985 // [2] - ignore
3986 // [3] - zero low half of destination
3987 // [5:4] - select 128 bits from sources for high half of destination
3988 // [6] - ignore
3989 // [7] - zero high half of destination
3990
3991 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3992
3993 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3994 unsigned HalfSize = NumElts / 2;
3995 SmallVector<int, 8> ShuffleMask(NumElts);
3996
3997 // Determine which operand(s) are actually in use for this instruction.
3998 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3999 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4000
4001 // If needed, replace operands based on zero mask.
4002 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4003 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4004
4005 // Permute low half of result.
4006 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4007 for (unsigned i = 0; i < HalfSize; ++i)
4008 ShuffleMask[i] = StartIndex + i;
4009
4010 // Permute high half of result.
4011 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4012 for (unsigned i = 0; i < HalfSize; ++i)
4013 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4014
4015 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4016
4017 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4018 Name.starts_with("avx512.mask.vpermil.p") ||
4019 Name.starts_with("avx512.mask.pshuf.d.")) {
4020 Value *Op0 = CI->getArgOperand(0);
4021 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4022 auto *VecTy = cast<FixedVectorType>(CI->getType());
4023 unsigned NumElts = VecTy->getNumElements();
4024 // Calculate the size of each index in the immediate.
4025 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4026 unsigned IdxMask = ((1 << IdxSize) - 1);
4027
4028 SmallVector<int, 8> Idxs(NumElts);
4029 // Lookup the bits for this element, wrapping around the immediate every
4030 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4031 // to offset by the first index of each group.
4032 for (unsigned i = 0; i != NumElts; ++i)
4033 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4034
4035 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4036
4037 if (CI->arg_size() == 4)
4038 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4039 CI->getArgOperand(2));
4040 } else if (Name == "sse2.pshufl.w" ||
4041 Name.starts_with("avx512.mask.pshufl.w.")) {
4042 Value *Op0 = CI->getArgOperand(0);
4043 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4044 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4045
4046 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4047 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4048
4049 SmallVector<int, 16> Idxs(NumElts);
4050 for (unsigned l = 0; l != NumElts; l += 8) {
4051 for (unsigned i = 0; i != 4; ++i)
4052 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4053 for (unsigned i = 4; i != 8; ++i)
4054 Idxs[i + l] = i + l;
4055 }
4056
4057 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4058
4059 if (CI->arg_size() == 4)
4060 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4061 CI->getArgOperand(2));
4062 } else if (Name == "sse2.pshufh.w" ||
4063 Name.starts_with("avx512.mask.pshufh.w.")) {
4064 Value *Op0 = CI->getArgOperand(0);
4065 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4066 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4067
4068 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4069 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4070
4071 SmallVector<int, 16> Idxs(NumElts);
4072 for (unsigned l = 0; l != NumElts; l += 8) {
4073 for (unsigned i = 0; i != 4; ++i)
4074 Idxs[i + l] = i + l;
4075 for (unsigned i = 0; i != 4; ++i)
4076 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4077 }
4078
4079 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4080
4081 if (CI->arg_size() == 4)
4082 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4083 CI->getArgOperand(2));
4084 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4085 Value *Op0 = CI->getArgOperand(0);
4086 Value *Op1 = CI->getArgOperand(1);
4087 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4088 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4089
4090 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4091 unsigned HalfLaneElts = NumLaneElts / 2;
4092
4093 SmallVector<int, 16> Idxs(NumElts);
4094 for (unsigned i = 0; i != NumElts; ++i) {
4095 // Base index is the starting element of the lane.
4096 Idxs[i] = i - (i % NumLaneElts);
4097 // If we are half way through the lane switch to the other source.
4098 if ((i % NumLaneElts) >= HalfLaneElts)
4099 Idxs[i] += NumElts;
4100 // Now select the specific element. By adding HalfLaneElts bits from
4101 // the immediate. Wrapping around the immediate every 8-bits.
4102 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4103 }
4104
4105 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4106
4107 Rep =
4108 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4109 } else if (Name.starts_with("avx512.mask.movddup") ||
4110 Name.starts_with("avx512.mask.movshdup") ||
4111 Name.starts_with("avx512.mask.movsldup")) {
4112 Value *Op0 = CI->getArgOperand(0);
4113 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4114 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4115
4116 unsigned Offset = 0;
4117 if (Name.starts_with("avx512.mask.movshdup."))
4118 Offset = 1;
4119
4120 SmallVector<int, 16> Idxs(NumElts);
4121 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4122 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4123 Idxs[i + l + 0] = i + l + Offset;
4124 Idxs[i + l + 1] = i + l + Offset;
4125 }
4126
4127 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4128
4129 Rep =
4130 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4131 } else if (Name.starts_with("avx512.mask.punpckl") ||
4132 Name.starts_with("avx512.mask.unpckl.")) {
4133 Value *Op0 = CI->getArgOperand(0);
4134 Value *Op1 = CI->getArgOperand(1);
4135 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4136 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4137
4138 SmallVector<int, 64> Idxs(NumElts);
4139 for (int l = 0; l != NumElts; l += NumLaneElts)
4140 for (int i = 0; i != NumLaneElts; ++i)
4141 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4142
4143 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4144
4145 Rep =
4146 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4147 } else if (Name.starts_with("avx512.mask.punpckh") ||
4148 Name.starts_with("avx512.mask.unpckh.")) {
4149 Value *Op0 = CI->getArgOperand(0);
4150 Value *Op1 = CI->getArgOperand(1);
4151 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4152 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4153
4154 SmallVector<int, 64> Idxs(NumElts);
4155 for (int l = 0; l != NumElts; l += NumLaneElts)
4156 for (int i = 0; i != NumLaneElts; ++i)
4157 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4158
4159 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4160
4161 Rep =
4162 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4163 } else if (Name.starts_with("avx512.mask.and.") ||
4164 Name.starts_with("avx512.mask.pand.")) {
4165 VectorType *FTy = cast<VectorType>(CI->getType());
4166 VectorType *ITy = VectorType::getInteger(FTy);
4167 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4168 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4169 Rep = Builder.CreateBitCast(Rep, FTy);
4170 Rep =
4171 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4172 } else if (Name.starts_with("avx512.mask.andn.") ||
4173 Name.starts_with("avx512.mask.pandn.")) {
4174 VectorType *FTy = cast<VectorType>(CI->getType());
4175 VectorType *ITy = VectorType::getInteger(FTy);
4176 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4177 Rep = Builder.CreateAnd(Rep,
4178 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4179 Rep = Builder.CreateBitCast(Rep, FTy);
4180 Rep =
4181 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4182 } else if (Name.starts_with("avx512.mask.or.") ||
4183 Name.starts_with("avx512.mask.por.")) {
4184 VectorType *FTy = cast<VectorType>(CI->getType());
4185 VectorType *ITy = VectorType::getInteger(FTy);
4186 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4187 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4188 Rep = Builder.CreateBitCast(Rep, FTy);
4189 Rep =
4190 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4191 } else if (Name.starts_with("avx512.mask.xor.") ||
4192 Name.starts_with("avx512.mask.pxor.")) {
4193 VectorType *FTy = cast<VectorType>(CI->getType());
4194 VectorType *ITy = VectorType::getInteger(FTy);
4195 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4196 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4197 Rep = Builder.CreateBitCast(Rep, FTy);
4198 Rep =
4199 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4200 } else if (Name.starts_with("avx512.mask.padd.")) {
4201 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4202 Rep =
4203 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4204 } else if (Name.starts_with("avx512.mask.psub.")) {
4205 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4206 Rep =
4207 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4208 } else if (Name.starts_with("avx512.mask.pmull.")) {
4209 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4210 Rep =
4211 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4212 } else if (Name.starts_with("avx512.mask.add.p")) {
4213 if (Name.ends_with(".512")) {
4214 Intrinsic::ID IID;
4215 if (Name[17] == 's')
4216 IID = Intrinsic::x86_avx512_add_ps_512;
4217 else
4218 IID = Intrinsic::x86_avx512_add_pd_512;
4219
4220 Rep = Builder.CreateIntrinsic(
4221 IID,
4222 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4223 } else {
4224 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4225 }
4226 Rep =
4227 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4228 } else if (Name.starts_with("avx512.mask.div.p")) {
4229 if (Name.ends_with(".512")) {
4230 Intrinsic::ID IID;
4231 if (Name[17] == 's')
4232 IID = Intrinsic::x86_avx512_div_ps_512;
4233 else
4234 IID = Intrinsic::x86_avx512_div_pd_512;
4235
4236 Rep = Builder.CreateIntrinsic(
4237 IID,
4238 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4239 } else {
4240 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4241 }
4242 Rep =
4243 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4244 } else if (Name.starts_with("avx512.mask.mul.p")) {
4245 if (Name.ends_with(".512")) {
4246 Intrinsic::ID IID;
4247 if (Name[17] == 's')
4248 IID = Intrinsic::x86_avx512_mul_ps_512;
4249 else
4250 IID = Intrinsic::x86_avx512_mul_pd_512;
4251
4252 Rep = Builder.CreateIntrinsic(
4253 IID,
4254 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4255 } else {
4256 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4257 }
4258 Rep =
4259 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4260 } else if (Name.starts_with("avx512.mask.sub.p")) {
4261 if (Name.ends_with(".512")) {
4262 Intrinsic::ID IID;
4263 if (Name[17] == 's')
4264 IID = Intrinsic::x86_avx512_sub_ps_512;
4265 else
4266 IID = Intrinsic::x86_avx512_sub_pd_512;
4267
4268 Rep = Builder.CreateIntrinsic(
4269 IID,
4270 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4271 } else {
4272 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4273 }
4274 Rep =
4275 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4276 } else if ((Name.starts_with("avx512.mask.max.p") ||
4277 Name.starts_with("avx512.mask.min.p")) &&
4278 Name.drop_front(18) == ".512") {
4279 bool IsDouble = Name[17] == 'd';
4280 bool IsMin = Name[13] == 'i';
4281 static const Intrinsic::ID MinMaxTbl[2][2] = {
4282 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4283 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4284 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4285
4286 Rep = Builder.CreateIntrinsic(
4287 IID,
4288 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4289 Rep =
4290 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4291 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4292 Rep =
4293 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4294 {CI->getArgOperand(0), Builder.getInt1(false)});
4295 Rep =
4296 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4297 } else if (Name.starts_with("avx512.mask.psll")) {
4298 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4299 bool IsVariable = Name[16] == 'v';
4300 char Size = Name[16] == '.' ? Name[17]
4301 : Name[17] == '.' ? Name[18]
4302 : Name[18] == '.' ? Name[19]
4303 : Name[20];
4304
4305 Intrinsic::ID IID;
4306 if (IsVariable && Name[17] != '.') {
4307 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4308 IID = Intrinsic::x86_avx2_psllv_q;
4309 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4310 IID = Intrinsic::x86_avx2_psllv_q_256;
4311 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4312 IID = Intrinsic::x86_avx2_psllv_d;
4313 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4314 IID = Intrinsic::x86_avx2_psllv_d_256;
4315 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4316 IID = Intrinsic::x86_avx512_psllv_w_128;
4317 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4318 IID = Intrinsic::x86_avx512_psllv_w_256;
4319 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4320 IID = Intrinsic::x86_avx512_psllv_w_512;
4321 else
4322 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4323 } else if (Name.ends_with(".128")) {
4324 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4325 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4326 : Intrinsic::x86_sse2_psll_d;
4327 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4328 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4329 : Intrinsic::x86_sse2_psll_q;
4330 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4331 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4332 : Intrinsic::x86_sse2_psll_w;
4333 else
4334 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4335 } else if (Name.ends_with(".256")) {
4336 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4337 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4338 : Intrinsic::x86_avx2_psll_d;
4339 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4340 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4341 : Intrinsic::x86_avx2_psll_q;
4342 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4343 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4344 : Intrinsic::x86_avx2_psll_w;
4345 else
4346 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4347 } else {
4348 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4349 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4350 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4351 : Intrinsic::x86_avx512_psll_d_512;
4352 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4353 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4354 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4355 : Intrinsic::x86_avx512_psll_q_512;
4356 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4357 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4358 : Intrinsic::x86_avx512_psll_w_512;
4359 else
4360 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4361 }
4362
4363 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4364 } else if (Name.starts_with("avx512.mask.psrl")) {
4365 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4366 bool IsVariable = Name[16] == 'v';
4367 char Size = Name[16] == '.' ? Name[17]
4368 : Name[17] == '.' ? Name[18]
4369 : Name[18] == '.' ? Name[19]
4370 : Name[20];
4371
4372 Intrinsic::ID IID;
4373 if (IsVariable && Name[17] != '.') {
4374 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4375 IID = Intrinsic::x86_avx2_psrlv_q;
4376 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4377 IID = Intrinsic::x86_avx2_psrlv_q_256;
4378 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4379 IID = Intrinsic::x86_avx2_psrlv_d;
4380 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4381 IID = Intrinsic::x86_avx2_psrlv_d_256;
4382 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4383 IID = Intrinsic::x86_avx512_psrlv_w_128;
4384 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4385 IID = Intrinsic::x86_avx512_psrlv_w_256;
4386 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4387 IID = Intrinsic::x86_avx512_psrlv_w_512;
4388 else
4389 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4390 } else if (Name.ends_with(".128")) {
4391 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4392 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4393 : Intrinsic::x86_sse2_psrl_d;
4394 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4395 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4396 : Intrinsic::x86_sse2_psrl_q;
4397 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4398 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4399 : Intrinsic::x86_sse2_psrl_w;
4400 else
4401 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4402 } else if (Name.ends_with(".256")) {
4403 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4404 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4405 : Intrinsic::x86_avx2_psrl_d;
4406 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4407 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4408 : Intrinsic::x86_avx2_psrl_q;
4409 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4410 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4411 : Intrinsic::x86_avx2_psrl_w;
4412 else
4413 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4414 } else {
4415 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4416 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4417 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4418 : Intrinsic::x86_avx512_psrl_d_512;
4419 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4420 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4421 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4422 : Intrinsic::x86_avx512_psrl_q_512;
4423 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4424 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4425 : Intrinsic::x86_avx512_psrl_w_512;
4426 else
4427 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4428 }
4429
4430 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4431 } else if (Name.starts_with("avx512.mask.psra")) {
4432 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4433 bool IsVariable = Name[16] == 'v';
4434 char Size = Name[16] == '.' ? Name[17]
4435 : Name[17] == '.' ? Name[18]
4436 : Name[18] == '.' ? Name[19]
4437 : Name[20];
4438
4439 Intrinsic::ID IID;
4440 if (IsVariable && Name[17] != '.') {
4441 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4442 IID = Intrinsic::x86_avx2_psrav_d;
4443 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4444 IID = Intrinsic::x86_avx2_psrav_d_256;
4445 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4446 IID = Intrinsic::x86_avx512_psrav_w_128;
4447 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4448 IID = Intrinsic::x86_avx512_psrav_w_256;
4449 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4450 IID = Intrinsic::x86_avx512_psrav_w_512;
4451 else
4452 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4453 } else if (Name.ends_with(".128")) {
4454 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4455 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4456 : Intrinsic::x86_sse2_psra_d;
4457 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4458 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4459 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4460 : Intrinsic::x86_avx512_psra_q_128;
4461 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4462 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4463 : Intrinsic::x86_sse2_psra_w;
4464 else
4465 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4466 } else if (Name.ends_with(".256")) {
4467 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4468 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4469 : Intrinsic::x86_avx2_psra_d;
4470 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4471 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4472 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4473 : Intrinsic::x86_avx512_psra_q_256;
4474 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4475 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4476 : Intrinsic::x86_avx2_psra_w;
4477 else
4478 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4479 } else {
4480 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4481 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4482 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4483 : Intrinsic::x86_avx512_psra_d_512;
4484 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4485 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4486 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4487 : Intrinsic::x86_avx512_psra_q_512;
4488 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4489 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4490 : Intrinsic::x86_avx512_psra_w_512;
4491 else
4492 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4493 }
4494
4495 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4496 } else if (Name.starts_with("avx512.mask.move.s")) {
4497 Rep = upgradeMaskedMove(Builder, *CI);
4498 } else if (Name.starts_with("avx512.cvtmask2")) {
4499 Rep = upgradeMaskToInt(Builder, *CI);
4500 } else if (Name.ends_with(".movntdqa")) {
4501 MDNode *Node = MDNode::get(
4502 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4503
4504 LoadInst *LI = Builder.CreateAlignedLoad(
4505 CI->getType(), CI->getArgOperand(0),
4506 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4507 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4508 Rep = LI;
4509 } else if (Name.starts_with("fma.vfmadd.") ||
4510 Name.starts_with("fma.vfmsub.") ||
4511 Name.starts_with("fma.vfnmadd.") ||
4512 Name.starts_with("fma.vfnmsub.")) {
4513 bool NegMul = Name[6] == 'n';
4514 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4515 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4516
4517 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4518 CI->getArgOperand(2)};
4519
4520 if (IsScalar) {
4521 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4522 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4523 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4524 }
4525
4526 if (NegMul && !IsScalar)
4527 Ops[0] = Builder.CreateFNeg(Ops[0]);
4528 if (NegMul && IsScalar)
4529 Ops[1] = Builder.CreateFNeg(Ops[1]);
4530 if (NegAcc)
4531 Ops[2] = Builder.CreateFNeg(Ops[2]);
4532
4533 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4534
4535 if (IsScalar)
4536 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4537 } else if (Name.starts_with("fma4.vfmadd.s")) {
4538 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4539 CI->getArgOperand(2)};
4540
4541 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4542 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4543 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4544
4545 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4546
4547 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4548 Rep, (uint64_t)0);
4549 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4550 Name.starts_with("avx512.maskz.vfmadd.s") ||
4551 Name.starts_with("avx512.mask3.vfmadd.s") ||
4552 Name.starts_with("avx512.mask3.vfmsub.s") ||
4553 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4554 bool IsMask3 = Name[11] == '3';
4555 bool IsMaskZ = Name[11] == 'z';
4556 // Drop the "avx512.mask." to make it easier.
4557 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4558 bool NegMul = Name[2] == 'n';
4559 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4560
4561 Value *A = CI->getArgOperand(0);
4562 Value *B = CI->getArgOperand(1);
4563 Value *C = CI->getArgOperand(2);
4564
4565 if (NegMul && (IsMask3 || IsMaskZ))
4566 A = Builder.CreateFNeg(A);
4567 if (NegMul && !(IsMask3 || IsMaskZ))
4568 B = Builder.CreateFNeg(B);
4569 if (NegAcc)
4570 C = Builder.CreateFNeg(C);
4571
4572 A = Builder.CreateExtractElement(A, (uint64_t)0);
4573 B = Builder.CreateExtractElement(B, (uint64_t)0);
4574 C = Builder.CreateExtractElement(C, (uint64_t)0);
4575
4576 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4577 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4578 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4579
4580 Intrinsic::ID IID;
4581 if (Name.back() == 'd')
4582 IID = Intrinsic::x86_avx512_vfmadd_f64;
4583 else
4584 IID = Intrinsic::x86_avx512_vfmadd_f32;
4585 Rep = Builder.CreateIntrinsic(IID, Ops);
4586 } else {
4587 Rep = Builder.CreateFMA(A, B, C);
4588 }
4589
4590 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4591 : IsMask3 ? C
4592 : A;
4593
4594 // For Mask3 with NegAcc, we need to create a new extractelement that
4595 // avoids the negation above.
4596 if (NegAcc && IsMask3)
4597 PassThru =
4598 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4599
4600 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4601 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4602 (uint64_t)0);
4603 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4604 Name.starts_with("avx512.mask.vfnmadd.p") ||
4605 Name.starts_with("avx512.mask.vfnmsub.p") ||
4606 Name.starts_with("avx512.mask3.vfmadd.p") ||
4607 Name.starts_with("avx512.mask3.vfmsub.p") ||
4608 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4609 Name.starts_with("avx512.maskz.vfmadd.p")) {
4610 bool IsMask3 = Name[11] == '3';
4611 bool IsMaskZ = Name[11] == 'z';
4612 // Drop the "avx512.mask." to make it easier.
4613 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4614 bool NegMul = Name[2] == 'n';
4615 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4616
4617 Value *A = CI->getArgOperand(0);
4618 Value *B = CI->getArgOperand(1);
4619 Value *C = CI->getArgOperand(2);
4620
4621 if (NegMul && (IsMask3 || IsMaskZ))
4622 A = Builder.CreateFNeg(A);
4623 if (NegMul && !(IsMask3 || IsMaskZ))
4624 B = Builder.CreateFNeg(B);
4625 if (NegAcc)
4626 C = Builder.CreateFNeg(C);
4627
4628 if (CI->arg_size() == 5 &&
4629 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4630 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4631 Intrinsic::ID IID;
4632 // Check the character before ".512" in string.
4633 if (Name[Name.size() - 5] == 's')
4634 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4635 else
4636 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4637
4638 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4639 } else {
4640 Rep = Builder.CreateFMA(A, B, C);
4641 }
4642
4643 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4644 : IsMask3 ? CI->getArgOperand(2)
4645 : CI->getArgOperand(0);
4646
4647 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4648 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4649 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4650 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4651 Intrinsic::ID IID;
4652 if (VecWidth == 128 && EltWidth == 32)
4653 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4654 else if (VecWidth == 256 && EltWidth == 32)
4655 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4656 else if (VecWidth == 128 && EltWidth == 64)
4657 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4658 else if (VecWidth == 256 && EltWidth == 64)
4659 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4660 else
4661 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4662
4663 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4664 CI->getArgOperand(2)};
4665 Ops[2] = Builder.CreateFNeg(Ops[2]);
4666 Rep = Builder.CreateIntrinsic(IID, Ops);
4667 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4668 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4669 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4670 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4671 bool IsMask3 = Name[11] == '3';
4672 bool IsMaskZ = Name[11] == 'z';
4673 // Drop the "avx512.mask." to make it easier.
4674 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4675 bool IsSubAdd = Name[3] == 's';
4676 if (CI->arg_size() == 5) {
4677 Intrinsic::ID IID;
4678 // Check the character before ".512" in string.
4679 if (Name[Name.size() - 5] == 's')
4680 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4681 else
4682 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4683
4684 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4685 CI->getArgOperand(2), CI->getArgOperand(4)};
4686 if (IsSubAdd)
4687 Ops[2] = Builder.CreateFNeg(Ops[2]);
4688
4689 Rep = Builder.CreateIntrinsic(IID, Ops);
4690 } else {
4691 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4692
4693 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4694 CI->getArgOperand(2)};
4695
4696 Function *FMA = Intrinsic::getOrInsertDeclaration(
4697 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4698 Value *Odd = Builder.CreateCall(FMA, Ops);
4699 Ops[2] = Builder.CreateFNeg(Ops[2]);
4700 Value *Even = Builder.CreateCall(FMA, Ops);
4701
4702 if (IsSubAdd)
4703 std::swap(Even, Odd);
4704
4705 SmallVector<int, 32> Idxs(NumElts);
4706 for (int i = 0; i != NumElts; ++i)
4707 Idxs[i] = i + (i % 2) * NumElts;
4708
4709 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4710 }
4711
4712 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4713 : IsMask3 ? CI->getArgOperand(2)
4714 : CI->getArgOperand(0);
4715
4716 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4717 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4718 Name.starts_with("avx512.maskz.pternlog.")) {
4719 bool ZeroMask = Name[11] == 'z';
4720 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4721 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4722 Intrinsic::ID IID;
4723 if (VecWidth == 128 && EltWidth == 32)
4724 IID = Intrinsic::x86_avx512_pternlog_d_128;
4725 else if (VecWidth == 256 && EltWidth == 32)
4726 IID = Intrinsic::x86_avx512_pternlog_d_256;
4727 else if (VecWidth == 512 && EltWidth == 32)
4728 IID = Intrinsic::x86_avx512_pternlog_d_512;
4729 else if (VecWidth == 128 && EltWidth == 64)
4730 IID = Intrinsic::x86_avx512_pternlog_q_128;
4731 else if (VecWidth == 256 && EltWidth == 64)
4732 IID = Intrinsic::x86_avx512_pternlog_q_256;
4733 else if (VecWidth == 512 && EltWidth == 64)
4734 IID = Intrinsic::x86_avx512_pternlog_q_512;
4735 else
4736 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4737
4738 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4739 CI->getArgOperand(2), CI->getArgOperand(3)};
4740 Rep = Builder.CreateIntrinsic(IID, Args);
4741 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4742 : CI->getArgOperand(0);
4743 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4744 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4745 Name.starts_with("avx512.maskz.vpmadd52")) {
4746 bool ZeroMask = Name[11] == 'z';
4747 bool High = Name[20] == 'h' || Name[21] == 'h';
4748 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4749 Intrinsic::ID IID;
4750 if (VecWidth == 128 && !High)
4751 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4752 else if (VecWidth == 256 && !High)
4753 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4754 else if (VecWidth == 512 && !High)
4755 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4756 else if (VecWidth == 128 && High)
4757 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4758 else if (VecWidth == 256 && High)
4759 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4760 else if (VecWidth == 512 && High)
4761 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4762 else
4763 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4764
4765 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4766 CI->getArgOperand(2)};
4767 Rep = Builder.CreateIntrinsic(IID, Args);
4768 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4769 : CI->getArgOperand(0);
4770 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4771 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4772 Name.starts_with("avx512.mask.vpermt2var.") ||
4773 Name.starts_with("avx512.maskz.vpermt2var.")) {
4774 bool ZeroMask = Name[11] == 'z';
4775 bool IndexForm = Name[17] == 'i';
4776 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4777 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4778 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4779 Name.starts_with("avx512.mask.vpdpbusds.") ||
4780 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4781 bool ZeroMask = Name[11] == 'z';
4782 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4783 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4784 Intrinsic::ID IID;
4785 if (VecWidth == 128 && !IsSaturating)
4786 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4787 else if (VecWidth == 256 && !IsSaturating)
4788 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4789 else if (VecWidth == 512 && !IsSaturating)
4790 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4791 else if (VecWidth == 128 && IsSaturating)
4792 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4793 else if (VecWidth == 256 && IsSaturating)
4794 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4795 else if (VecWidth == 512 && IsSaturating)
4796 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4797 else
4798 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4799
4800 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4801 CI->getArgOperand(2)};
4802
4803 // Input arguments types were incorrectly set to vectors of i32 before but
4804 // they should be vectors of i8. Insert bit cast when encountering the old
4805 // types
4806 if (Args[1]->getType()->isVectorTy() &&
4807 cast<VectorType>(Args[1]->getType())
4808 ->getElementType()
4809 ->isIntegerTy(32) &&
4810 Args[2]->getType()->isVectorTy() &&
4811 cast<VectorType>(Args[2]->getType())
4812 ->getElementType()
4813 ->isIntegerTy(32)) {
4814 Type *NewArgType = nullptr;
4815 if (VecWidth == 128)
4816 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4817 else if (VecWidth == 256)
4818 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4819 else if (VecWidth == 512)
4820 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4821 else
4822 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4823 CI);
4824
4825 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4826 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4827 }
4828
4829 Rep = Builder.CreateIntrinsic(IID, Args);
4830 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4831 : CI->getArgOperand(0);
4832 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4833 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4834 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4835 Name.starts_with("avx512.mask.vpdpwssds.") ||
4836 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4837 bool ZeroMask = Name[11] == 'z';
4838 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4839 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4840 Intrinsic::ID IID;
4841 if (VecWidth == 128 && !IsSaturating)
4842 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4843 else if (VecWidth == 256 && !IsSaturating)
4844 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4845 else if (VecWidth == 512 && !IsSaturating)
4846 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4847 else if (VecWidth == 128 && IsSaturating)
4848 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4849 else if (VecWidth == 256 && IsSaturating)
4850 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4851 else if (VecWidth == 512 && IsSaturating)
4852 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4853 else
4854 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4855
4856 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4857 CI->getArgOperand(2)};
4858
4859 // Input arguments types were incorrectly set to vectors of i32 before but
4860 // they should be vectors of i16. Insert bit cast when encountering the old
4861 // types
4862 if (Args[1]->getType()->isVectorTy() &&
4863 cast<VectorType>(Args[1]->getType())
4864 ->getElementType()
4865 ->isIntegerTy(32) &&
4866 Args[2]->getType()->isVectorTy() &&
4867 cast<VectorType>(Args[2]->getType())
4868 ->getElementType()
4869 ->isIntegerTy(32)) {
4870 Type *NewArgType = nullptr;
4871 if (VecWidth == 128)
4872 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4873 else if (VecWidth == 256)
4874 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4875 else if (VecWidth == 512)
4876 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4877 else
4878 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4879 CI);
4880
4881 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4882 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4883 }
4884
4885 Rep = Builder.CreateIntrinsic(IID, Args);
4886 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4887 : CI->getArgOperand(0);
4888 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4889 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4890 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4891 Name == "subborrow.u32" || Name == "subborrow.u64") {
4892 Intrinsic::ID IID;
4893 if (Name[0] == 'a' && Name.back() == '2')
4894 IID = Intrinsic::x86_addcarry_32;
4895 else if (Name[0] == 'a' && Name.back() == '4')
4896 IID = Intrinsic::x86_addcarry_64;
4897 else if (Name[0] == 's' && Name.back() == '2')
4898 IID = Intrinsic::x86_subborrow_32;
4899 else if (Name[0] == 's' && Name.back() == '4')
4900 IID = Intrinsic::x86_subborrow_64;
4901 else
4902 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4903
4904 // Make a call with 3 operands.
4905 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4906 CI->getArgOperand(2)};
4907 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4908
4909 // Extract the second result and store it.
4910 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4911 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4912 // Replace the original call result with the first result of the new call.
4913 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4914
4915 CI->replaceAllUsesWith(CF);
4916 Rep = nullptr;
4917 } else if (Name.starts_with("avx512.mask.") &&
4918 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4919 // Rep will be updated by the call in the condition.
4920 } else if (Name.starts_with("bmi.pdep.")) {
4921 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
4922 } else if (Name.starts_with("bmi.pext.")) {
4923 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
4924 } else
4925 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4926
4927 return Rep;
4928}
4929
4930static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4931 Function *F, IRBuilder<> &Builder) {
4932 if (Name.starts_with("neon.bfcvt")) {
4933 if (Name.starts_with("neon.bfcvtn2")) {
4934 SmallVector<int, 32> LoMask(4);
4935 std::iota(LoMask.begin(), LoMask.end(), 0);
4936 SmallVector<int, 32> ConcatMask(8);
4937 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4938 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4939 Value *Trunc =
4940 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4941 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4942 } else if (Name.starts_with("neon.bfcvtn")) {
4943 SmallVector<int, 32> ConcatMask(8);
4944 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4945 Type *V4BF16 =
4946 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4947 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4948 dbgs() << "Trunc: " << *Trunc << "\n";
4949 return Builder.CreateShuffleVector(
4950 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4951 } else {
4952 return Builder.CreateFPTrunc(CI->getOperand(0),
4953 Type::getBFloatTy(F->getContext()));
4954 }
4955 } else if (Name.starts_with("sve.fcvt")) {
4956 Intrinsic::ID NewID =
4957 StringSwitch<Intrinsic::ID>(Name)
4958 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4959 .Case("sve.fcvtnt.bf16f32",
4960 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4961 .Default(Intrinsic::not_intrinsic);
4962 if (NewID == Intrinsic::not_intrinsic)
4963 llvm_unreachable("Unhandled Intrinsic!");
4964
4965 SmallVector<Value *, 3> Args(CI->args());
4966
4967 // The original intrinsics incorrectly used a predicate based on the
4968 // smallest element type rather than the largest.
4969 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4970 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4971
4972 if (Args[1]->getType() != BadPredTy)
4973 llvm_unreachable("Unexpected predicate type!");
4974
4975 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4976 BadPredTy, Args[1]);
4977 Args[1] = Builder.CreateIntrinsic(
4978 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4979
4980 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
4981 CI->getName());
4982 }
4983
4984 if (Name == "neon.vcvtfp2hf")
4985 return Builder.CreateBitCast(
4986 Builder.CreateFPTrunc(
4987 CI->getOperand(0),
4988 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4989 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
4990 if (Name == "neon.vcvthf2fp")
4991 return Builder.CreateFPExt(
4992 Builder.CreateBitCast(
4993 CI->getOperand(0),
4994 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4995 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
4996
4997 llvm_unreachable("Unhandled Intrinsic!");
4998}
4999
5000static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5001 IRBuilder<> &Builder) {
5002 if (Name == "mve.vctp64.old") {
5003 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5004 // correct type.
5005 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5006 CI->getArgOperand(0),
5007 /*FMFSource=*/nullptr, CI->getName());
5008 Value *C1 = Builder.CreateIntrinsic(
5009 Intrinsic::arm_mve_pred_v2i,
5010 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5011 return Builder.CreateIntrinsic(
5012 Intrinsic::arm_mve_pred_i2v,
5013 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5014 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5015 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5016 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5017 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5018 Name ==
5019 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5020 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5021 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5022 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5023 Name ==
5024 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5025 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5026 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5027 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5028 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5029 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5030 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5031 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5032 std::vector<Type *> Tys;
5033 unsigned ID = CI->getIntrinsicID();
5034 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5035 switch (ID) {
5036 case Intrinsic::arm_mve_mull_int_predicated:
5037 case Intrinsic::arm_mve_vqdmull_predicated:
5038 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5039 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5040 break;
5041 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5042 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5043 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5044 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5045 V2I1Ty};
5046 break;
5047 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5048 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5049 CI->getOperand(1)->getType(), V2I1Ty};
5050 break;
5051 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5052 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5053 CI->getOperand(2)->getType(), V2I1Ty};
5054 break;
5055 case Intrinsic::arm_cde_vcx1q_predicated:
5056 case Intrinsic::arm_cde_vcx1qa_predicated:
5057 case Intrinsic::arm_cde_vcx2q_predicated:
5058 case Intrinsic::arm_cde_vcx2qa_predicated:
5059 case Intrinsic::arm_cde_vcx3q_predicated:
5060 case Intrinsic::arm_cde_vcx3qa_predicated:
5061 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5062 break;
5063 default:
5064 llvm_unreachable("Unhandled Intrinsic!");
5065 }
5066
5067 std::vector<Value *> Ops;
5068 for (Value *Op : CI->args()) {
5069 Type *Ty = Op->getType();
5070 if (Ty->getScalarSizeInBits() == 1) {
5071 Value *C1 = Builder.CreateIntrinsic(
5072 Intrinsic::arm_mve_pred_v2i,
5073 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5074 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5075 }
5076 Ops.push_back(Op);
5077 }
5078
5079 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5080 CI->getName());
5081 }
5082 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5083}
5084
5085// These are expected to have the arguments:
5086// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5087//
5088// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5089//
5090static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5091 Function *F, IRBuilder<> &Builder) {
5092 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5093 // for compatibility.
5094 auto UpgradeLegacyWMMAIUIntrinsicCall =
5095 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5096 ArrayRef<Type *> OverloadTys) -> Value * {
5097 // Prepare arguments, append clamp=0 for compatibility
5098 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5099 Args.push_back(Builder.getFalse());
5100
5101 // Insert the declaration for the right overload types
5102 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5103 F->getParent(), F->getIntrinsicID(), OverloadTys);
5104
5105 // Copy operand bundles if any
5106 SmallVector<OperandBundleDef, 1> Bundles;
5107 CI->getOperandBundlesAsDefs(Bundles);
5108
5109 // Create the new call and copy calling properties
5110 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5111 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5112 NewCall->setCallingConv(CI->getCallingConv());
5113 NewCall->setAttributes(CI->getAttributes());
5114 NewCall->copyMetadata(*CI);
5115 return NewCall;
5116 };
5117
5118 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5119 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5120 "intrinsic should have 7 arguments");
5121 Type *T1 = CI->getArgOperand(4)->getType();
5122 Type *T2 = CI->getArgOperand(1)->getType();
5123 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5124 }
5125 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5126 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5127 "intrinsic should have 8 arguments");
5128 Type *T1 = CI->getArgOperand(4)->getType();
5129 Type *T2 = CI->getArgOperand(1)->getType();
5130 Type *T3 = CI->getArgOperand(3)->getType();
5131 Type *T4 = CI->getArgOperand(5)->getType();
5132 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5133 }
5134
5135 switch (F->getIntrinsicID()) {
5136 default:
5137 break;
5138 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5139 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5140 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5141 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5142 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5143 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5144 // Drop src0 and src1 modifiers.
5145 const Value *Op0 = CI->getArgOperand(0);
5146 const Value *Op2 = CI->getArgOperand(2);
5147 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5148 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5149 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5150 if (!ModA->isZero() || !ModB->isZero())
5151 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5152
5153 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5154 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5155 Args.push_back(CI->getArgOperand(I));
5156
5157 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5158 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5159 Overloads.push_back(Args[3]->getType());
5160 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5161 F->getParent(), F->getIntrinsicID(), Overloads);
5162
5163 SmallVector<OperandBundleDef, 1> Bundles;
5164 CI->getOperandBundlesAsDefs(Bundles);
5165
5166 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5167 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5168 NewCall->setCallingConv(CI->getCallingConv());
5169 NewCall->setAttributes(CI->getAttributes());
5170 NewCall->copyMetadata(*CI);
5171 NewCall->takeName(CI);
5172 return NewCall;
5173 }
5174 }
5175
5176 AtomicRMWInst::BinOp RMWOp =
5177 StringSwitch<AtomicRMWInst::BinOp>(Name)
5178 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5179 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5180 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5181 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5182 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5183 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5184 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5185 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5186 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5187 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5188 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5189 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5190 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5191
5192 unsigned NumOperands = CI->getNumOperands();
5193 if (NumOperands < 3) // Malformed bitcode.
5194 return nullptr;
5195
5196 Value *Ptr = CI->getArgOperand(0);
5197 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5198 if (!PtrTy) // Malformed.
5199 return nullptr;
5200
5201 Value *Val = CI->getArgOperand(1);
5202 if (Val->getType() != CI->getType()) // Malformed.
5203 return nullptr;
5204
5205 ConstantInt *OrderArg = nullptr;
5206 bool IsVolatile = false;
5207
5208 // These should have 5 arguments (plus the callee). A separate version of the
5209 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5210 if (NumOperands > 3)
5211 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5212
5213 // Ignore scope argument at 3
5214
5215 if (NumOperands > 5) {
5216 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5217 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5218 }
5219
5220 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5221 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5222 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5223 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5224 Order = AtomicOrdering::SequentiallyConsistent;
5225
5226 LLVMContext &Ctx = F->getContext();
5227
5228 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5229 Type *RetTy = CI->getType();
5230 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5231 if (VT->getElementType()->isIntegerTy(16)) {
5232 VectorType *AsBF16 =
5233 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5234 Val = Builder.CreateBitCast(Val, AsBF16);
5235 }
5236 }
5237
5238 // The scope argument never really worked correctly. Use agent as the most
5239 // conservative option which should still always produce the instruction.
5240 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5241 AtomicRMWInst *RMW =
5242 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5243
5244 unsigned AddrSpace = PtrTy->getAddressSpace();
5245 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5246 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5247 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5248 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5249 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5250 }
5251
5252 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5253 MDBuilder MDB(F->getContext());
5254 MDNode *RangeNotPrivate =
5255 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5256 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5257 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5258 }
5259
5260 if (IsVolatile)
5261 RMW->setVolatile(true);
5262
5263 return Builder.CreateBitCast(RMW, RetTy);
5264}
5265
5266/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5267/// plain MDNode, as it's the verifier's job to check these are the correct
5268/// types later.
5269static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5270 if (Op < CI->arg_size()) {
5271 if (MetadataAsValue *MAV =
5272 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5273 Metadata *MD = MAV->getMetadata();
5274 return dyn_cast_if_present<MDNode>(MD);
5275 }
5276 }
5277 return nullptr;
5278}
5279
5280/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5281static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5282 if (Op < CI->arg_size())
5283 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5284 return MAV->getMetadata();
5285 return nullptr;
5286}
5287
5288/// Convert debug intrinsic calls to non-instruction debug records.
5289/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5290/// \p CI - The debug intrinsic call.
5291static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5292 DbgRecord *DR = nullptr;
5293 if (Name == "label") {
5294 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5295 } else if (Name == "assign") {
5296 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5297 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5298 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5299 unwrapMAVMetadataOp(CI, 4),
5300 /*The address is a Value ref, it will be stored as a Metadata */
5301 unwrapMAVOp(CI, 5));
5302 } else if (Name == "declare") {
5303 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5304 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5305 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5306 } else if (Name == "addr") {
5307 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5308 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5309 // Don't try to add something to the expression if it's not an expression.
5310 // Instead, allow the verifier to fail later.
5311 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5312 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5313 }
5314 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5315 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5316 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5317 } else if (Name == "value") {
5318 // An old version of dbg.value had an extra offset argument.
5319 unsigned VarOp = 1;
5320 unsigned ExprOp = 2;
5321 if (CI->arg_size() == 4) {
5322 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5323 // Nonzero offset dbg.values get dropped without a replacement.
5324 if (!Offset || !Offset->isNullValue())
5325 return;
5326 VarOp = 2;
5327 ExprOp = 3;
5328 }
5329 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5330 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5331 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5332 nullptr);
5333 }
5334 DR->setDebugLoc(CI->getDebugLoc());
5335 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5336 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5337}
5338
5339static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5340 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5341 if (!Offset)
5342 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5343 int64_t OffsetVal = Offset->getSExtValue();
5344 return Builder.CreateIntrinsic(OffsetVal >= 0
5345 ? Intrinsic::vector_splice_left
5346 : Intrinsic::vector_splice_right,
5347 CI->getType(),
5348 {CI->getArgOperand(0), CI->getArgOperand(1),
5349 Builder.getInt32(std::abs(OffsetVal))});
5350}
5351
5352static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5353 Function *F, IRBuilder<> &Builder) {
5354 if (Name.starts_with("to.fp16")) {
5355 Value *Cast =
5356 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5357 return Builder.CreateBitCast(Cast, CI->getType());
5358 }
5359
5360 if (Name.starts_with("from.fp16")) {
5361 Value *Cast =
5362 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5363 return Builder.CreateFPExt(Cast, CI->getType());
5364 }
5365
5366 return nullptr;
5367}
5368
5369static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5370 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5371 if (!MD || !isa<MDString>(MD))
5372 return ICmpInst::BAD_ICMP_PREDICATE;
5373 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5374 .Case("eq", ICmpInst::ICMP_EQ)
5375 .Case("ne", ICmpInst::ICMP_NE)
5376 .Case("ugt", ICmpInst::ICMP_UGT)
5377 .Case("uge", ICmpInst::ICMP_UGE)
5378 .Case("ult", ICmpInst::ICMP_ULT)
5379 .Case("ule", ICmpInst::ICMP_ULE)
5380 .Case("sgt", ICmpInst::ICMP_SGT)
5381 .Case("sge", ICmpInst::ICMP_SGE)
5382 .Case("slt", ICmpInst::ICMP_SLT)
5383 .Case("sle", ICmpInst::ICMP_SLE)
5384 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5385}
5386
5387static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5388 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5389 if (!MD || !isa<MDString>(MD))
5390 return FCmpInst::BAD_FCMP_PREDICATE;
5391 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5392 .Case("oeq", FCmpInst::FCMP_OEQ)
5393 .Case("ogt", FCmpInst::FCMP_OGT)
5394 .Case("oge", FCmpInst::FCMP_OGE)
5395 .Case("olt", FCmpInst::FCMP_OLT)
5396 .Case("ole", FCmpInst::FCMP_OLE)
5397 .Case("one", FCmpInst::FCMP_ONE)
5398 .Case("ord", FCmpInst::FCMP_ORD)
5399 .Case("uno", FCmpInst::FCMP_UNO)
5400 .Case("ueq", FCmpInst::FCMP_UEQ)
5401 .Case("ugt", FCmpInst::FCMP_UGT)
5402 .Case("uge", FCmpInst::FCMP_UGE)
5403 .Case("ult", FCmpInst::FCMP_ULT)
5404 .Case("ule", FCmpInst::FCMP_ULE)
5405 .Case("une", FCmpInst::FCMP_UNE)
5406 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5407}
5408
5409static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5410 IRBuilder<> &Builder) {
5411 Value *Rep;
5412 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5413 if (Opcode && Instruction::isUnaryOp(Opcode))
5414 Rep =
5415 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5416 else if (Opcode && Instruction::isBinaryOp(Opcode))
5417 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5418 CI->getArgOperand(0), CI->getArgOperand(1));
5419 else if (Opcode && Instruction::isCast(Opcode))
5420 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5421 CI->getType());
5422 else if (Opcode == Instruction::ICmp)
5423 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5424 CI->getArgOperand(0), CI->getArgOperand(1));
5425 else if (Opcode == Instruction::FCmp)
5426 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5427 CI->getArgOperand(0), CI->getArgOperand(1));
5428 else if (Opcode == Instruction::Select)
5429 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5430 CI->getArgOperand(2));
5431 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5432 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5433 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5434 } else
5435 llvm_unreachable("Unexpected vp intrinsic");
5436 Rep->takeName(CI);
5437 return Rep;
5438}
5439
5440static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5441 IRBuilder<> &Builder) {
5442 Intrinsic::ID IID = NewFn->getIntrinsicID();
5443
5444 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5445 if (Defaults.empty())
5446 return false;
5447
5448 unsigned OldArgCount = CI->arg_size();
5449 unsigned NewArgCount = NewFn->arg_size();
5450
5451 // If the caller already supplied all arguments (or more), nothing to do.
5452 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5453 if (OldArgCount >= NewArgCount)
5454 return false;
5455
5456 // Start with the existing arguments from the old call.
5457 SmallVector<Value *, 8> NewArgs(CI->args());
5458
5459 // Defaults are a contiguous trailing block, so checking the first missing
5460 // argument is enough.
5461 if (OldArgCount < FirstDefault)
5462 return false;
5463
5464 // Fill in each missing trailing argument from the table.
5465 FunctionType *NewFT = NewFn->getFunctionType();
5466 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5467 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5468 "missing argument outside the default range");
5469 Type *ParamTy = NewFT->getParamType(Idx);
5470
5471 // Only integer types are supported (i1, i8, i16, i32, i64).
5472 if (!ParamTy->isIntegerTy())
5473 return false;
5474 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5475 }
5476
5477 // Preserve operand bundles by creating the call with them.
5478 SmallVector<OperandBundleDef, 1> OpBundles;
5479 CI->getOperandBundlesAsDefs(OpBundles);
5480 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5481
5482 NewCall->takeName(CI);
5483 NewCall->setCallingConv(CI->getCallingConv());
5484 NewCall->copyMetadata(*CI);
5485 if (auto *OldCI = dyn_cast<CallInst>(CI))
5486 NewCall->setTailCallKind(OldCI->getTailCallKind());
5487
5488 CI->replaceAllUsesWith(NewCall);
5489 CI->eraseFromParent();
5490 return true;
5491}
5492
5493/// Upgrade a call to an old intrinsic. All argument and return casting must be
5494/// provided to seamlessly integrate with existing context.
5495void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5496 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5497 // checks the callee's function type matches. It's likely we need to handle
5498 // type changes here.
5499 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5500 if (!F)
5501 return;
5502
5503 LLVMContext &C = CI->getContext();
5504 IRBuilder<> Builder(C);
5505 if (isa<FPMathOperator>(CI))
5506 Builder.setFastMathFlags(CI->getFastMathFlags());
5507 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5508
5509 if (!NewFn) {
5510 // Get the Function's name.
5511 StringRef Name = F->getName();
5512 if (!Name.consume_front("llvm."))
5513 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5514
5515 bool IsX86 = Name.consume_front("x86.");
5516 bool IsNVVM = Name.consume_front("nvvm.");
5517 bool IsAArch64 = Name.consume_front("aarch64.");
5518 bool IsARM = Name.consume_front("arm.");
5519 bool IsAMDGCN = Name.consume_front("amdgcn.");
5520 bool IsDbg = Name.consume_front("dbg.");
5521 bool IsOldSplice =
5522 (Name.consume_front("experimental.vector.splice") ||
5523 Name.consume_front("vector.splice")) &&
5524 !(Name.starts_with(".left") || Name.starts_with(".right"));
5525 Value *Rep = nullptr;
5526
5527 if (!IsX86 && Name == "stackprotectorcheck") {
5528 Rep = nullptr;
5529 } else if (IsNVVM) {
5530 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5531 } else if (IsX86) {
5532 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5533 } else if (IsAArch64) {
5534 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5535 } else if (IsARM) {
5536 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5537 } else if (IsAMDGCN) {
5538 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5539 } else if (IsDbg) {
5540 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5541 } else if (IsOldSplice) {
5542 Rep = upgradeVectorSplice(CI, Builder);
5543 } else if (Name.consume_front("convert.")) {
5544 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5545 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5546 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5547 Rep = nullptr;
5548 } else if (shouldUpgradeVPIntrinsic(Name)) {
5549 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5550 } else {
5551 llvm_unreachable("Unknown function for CallBase upgrade.");
5552 }
5553
5554 if (Rep)
5555 CI->replaceAllUsesWith(Rep);
5556 CI->eraseFromParent();
5557 return;
5558 }
5559
5560 const auto &DefaultCase = [&]() -> void {
5561 if (F == NewFn)
5562 return;
5563
5564 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5565 // Handle generic mangling change.
5566 assert(
5567 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5568 "Unknown function for CallBase upgrade and isn't just a name change");
5569 CI->setCalledFunction(NewFn);
5570 return;
5571 }
5572
5573 // This must be an upgrade from a named to a literal struct.
5574 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5575 assert(OldST != NewFn->getReturnType() &&
5576 "Return type must have changed");
5577 assert(OldST->getNumElements() ==
5578 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5579 "Must have same number of elements");
5580
5581 SmallVector<Value *> Args(CI->args());
5582 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5583 NewCI->setAttributes(CI->getAttributes());
5584 Value *Res = PoisonValue::get(OldST);
5585 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5586 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5587 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5588 }
5589 CI->replaceAllUsesWith(Res);
5590 CI->eraseFromParent();
5591 return;
5592 }
5593
5594 // We're probably about to produce something invalid. Let the verifier catch
5595 // it instead of dying here.
5596 CI->setCalledOperand(
5597 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5598 return;
5599 };
5600 CallInst *NewCall = nullptr;
5601 switch (NewFn->getIntrinsicID()) {
5602 default: {
5603 // Last resort: try the data-driven default-arg upgrade.
5604 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5605 // in its .td definition, without needing a dedicated case.
5606 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5607 return;
5608 DefaultCase();
5609 return;
5610 }
5611 case Intrinsic::arm_neon_vst1:
5612 case Intrinsic::arm_neon_vst2:
5613 case Intrinsic::arm_neon_vst3:
5614 case Intrinsic::arm_neon_vst4:
5615 case Intrinsic::arm_neon_vst2lane:
5616 case Intrinsic::arm_neon_vst3lane:
5617 case Intrinsic::arm_neon_vst4lane: {
5618 SmallVector<Value *, 4> Args(CI->args());
5619 NewCall = Builder.CreateCall(NewFn, Args);
5620 break;
5621 }
5622 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5623 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5624 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5625 LLVMContext &Ctx = F->getParent()->getContext();
5626 SmallVector<Value *, 4> Args(CI->args());
5627 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5628 cast<ConstantInt>(Args[3])->getZExtValue());
5629 NewCall = Builder.CreateCall(NewFn, Args);
5630 break;
5631 }
5632 case Intrinsic::aarch64_sve_ld3_sret:
5633 case Intrinsic::aarch64_sve_ld4_sret:
5634 case Intrinsic::aarch64_sve_ld2_sret: {
5635 // Is this a trivial remangle of the name to support ptr address spaces?
5636 if (isa<StructType>(F->getReturnType())) {
5637 DefaultCase();
5638 return;
5639 }
5640
5641 StringRef Name = F->getName();
5642 Name = Name.substr(5);
5643 unsigned N = StringSwitch<unsigned>(Name)
5644 .StartsWith("aarch64.sve.ld2", 2)
5645 .StartsWith("aarch64.sve.ld3", 3)
5646 .StartsWith("aarch64.sve.ld4", 4)
5647 .Default(0);
5648 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5649 unsigned MinElts = RetTy->getMinNumElements() / N;
5650 SmallVector<Value *, 2> Args(CI->args());
5651 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5652 Value *Ret = llvm::PoisonValue::get(RetTy);
5653 for (unsigned I = 0; I < N; I++) {
5654 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5655 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5656 }
5657 NewCall = dyn_cast<CallInst>(Ret);
5658 break;
5659 }
5660
5661 case Intrinsic::coro_end_async:
5662 case Intrinsic::coro_end: {
5663 SmallVector<Value *, 3> Args(CI->args());
5664 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5665 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5666 NewCall = Builder.CreateCall(NewFn, Args);
5667
5668 if (!CI->getType()->isVoidTy()) {
5669 if (!CI->use_empty()) {
5670 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5671 CI->getModule(), Intrinsic::coro_is_in_ramp);
5672 Value *InRamp = Builder.CreateCall(IsInRamp);
5673 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5674 }
5675 CI->eraseFromParent();
5676 return;
5677 }
5678
5679 break;
5680 }
5681
5682 case Intrinsic::vector_extract: {
5683 StringRef Name = F->getName();
5684 Name = Name.substr(5); // Strip llvm
5685 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5686 DefaultCase();
5687 return;
5688 }
5689 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5690 unsigned MinElts = RetTy->getMinNumElements();
5691 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5692 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5693 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5694 break;
5695 }
5696
5697 case Intrinsic::vector_insert: {
5698 StringRef Name = F->getName();
5699 Name = Name.substr(5);
5700 if (!Name.starts_with("aarch64.sve.tuple")) {
5701 DefaultCase();
5702 return;
5703 }
5704 if (Name.starts_with("aarch64.sve.tuple.set")) {
5705 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5706 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5707 Value *NewIdx =
5708 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5709 NewCall = Builder.CreateCall(
5710 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5711 break;
5712 }
5713 if (Name.starts_with("aarch64.sve.tuple.create")) {
5714 unsigned N = StringSwitch<unsigned>(Name)
5715 .StartsWith("aarch64.sve.tuple.create2", 2)
5716 .StartsWith("aarch64.sve.tuple.create3", 3)
5717 .StartsWith("aarch64.sve.tuple.create4", 4)
5718 .Default(0);
5719 assert(N > 1 && "Create is expected to be between 2-4");
5720 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5721 Value *Ret = llvm::PoisonValue::get(RetTy);
5722 unsigned MinElts = RetTy->getMinNumElements() / N;
5723 for (unsigned I = 0; I < N; I++) {
5724 Value *V = CI->getArgOperand(I);
5725 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5726 }
5727 NewCall = dyn_cast<CallInst>(Ret);
5728 }
5729 break;
5730 }
5731
5732 case Intrinsic::arm_neon_bfdot:
5733 case Intrinsic::arm_neon_bfmmla:
5734 case Intrinsic::arm_neon_bfmlalb:
5735 case Intrinsic::arm_neon_bfmlalt:
5736 case Intrinsic::aarch64_neon_bfdot:
5737 case Intrinsic::aarch64_neon_bfmmla:
5738 case Intrinsic::aarch64_neon_bfmlalb:
5739 case Intrinsic::aarch64_neon_bfmlalt: {
5740 SmallVector<Value *, 3> Args;
5741 assert(CI->arg_size() == 3 &&
5742 "Mismatch between function args and call args");
5743 size_t OperandWidth =
5744 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5745 assert((OperandWidth == 64 || OperandWidth == 128) &&
5746 "Unexpected operand width");
5747 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5748 auto Iter = CI->args().begin();
5749 Args.push_back(*Iter++);
5750 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5751 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5752 NewCall = Builder.CreateCall(NewFn, Args);
5753 break;
5754 }
5755
5756 case Intrinsic::bitreverse:
5757 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5758 break;
5759
5760 case Intrinsic::ctlz:
5761 case Intrinsic::cttz: {
5762 if (CI->arg_size() != 1) {
5763 DefaultCase();
5764 return;
5765 }
5766
5767 NewCall =
5768 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5769 break;
5770 }
5771
5772 case Intrinsic::objectsize: {
5773 Value *NullIsUnknownSize =
5774 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5775 Value *Dynamic =
5776 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5777 NewCall = Builder.CreateCall(
5778 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5779 break;
5780 }
5781
5782 case Intrinsic::ctpop:
5783 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5784 break;
5785 case Intrinsic::dbg_value: {
5786 StringRef Name = F->getName();
5787 Name = Name.substr(5); // Strip llvm.
5788 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5789 if (Name.starts_with("dbg.addr")) {
5790 DIExpression *Expr = cast<DIExpression>(
5791 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5792 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5793 NewCall =
5794 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5795 MetadataAsValue::get(C, Expr)});
5796 break;
5797 }
5798
5799 // Upgrade from the old version that had an extra offset argument.
5800 assert(CI->arg_size() == 4);
5801 // Drop nonzero offsets instead of attempting to upgrade them.
5802 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5803 if (Offset->isNullValue()) {
5804 NewCall = Builder.CreateCall(
5805 NewFn,
5806 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5807 break;
5808 }
5809 CI->eraseFromParent();
5810 return;
5811 }
5812
5813 case Intrinsic::ptr_annotation:
5814 // Upgrade from versions that lacked the annotation attribute argument.
5815 if (CI->arg_size() != 4) {
5816 DefaultCase();
5817 return;
5818 }
5819
5820 // Create a new call with an added null annotation attribute argument.
5821 NewCall = Builder.CreateCall(
5822 NewFn,
5823 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5824 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5825 NewCall->takeName(CI);
5826 CI->replaceAllUsesWith(NewCall);
5827 CI->eraseFromParent();
5828 return;
5829
5830 case Intrinsic::var_annotation:
5831 // Upgrade from versions that lacked the annotation attribute argument.
5832 if (CI->arg_size() != 4) {
5833 DefaultCase();
5834 return;
5835 }
5836 // Create a new call with an added null annotation attribute argument.
5837 NewCall = Builder.CreateCall(
5838 NewFn,
5839 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5840 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5841 NewCall->takeName(CI);
5842 CI->replaceAllUsesWith(NewCall);
5843 CI->eraseFromParent();
5844 return;
5845
5846 case Intrinsic::riscv_aes32dsi:
5847 case Intrinsic::riscv_aes32dsmi:
5848 case Intrinsic::riscv_aes32esi:
5849 case Intrinsic::riscv_aes32esmi:
5850 case Intrinsic::riscv_sm4ks:
5851 case Intrinsic::riscv_sm4ed: {
5852 // The last argument to these intrinsics used to be i8 and changed to i32.
5853 // The type overload for sm4ks and sm4ed was removed.
5854 Value *Arg2 = CI->getArgOperand(2);
5855 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5856 return;
5857
5858 Value *Arg0 = CI->getArgOperand(0);
5859 Value *Arg1 = CI->getArgOperand(1);
5860 if (CI->getType()->isIntegerTy(64)) {
5861 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5862 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5863 }
5864
5865 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5866 cast<ConstantInt>(Arg2)->getZExtValue());
5867
5868 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5869 Value *Res = NewCall;
5870 if (Res->getType() != CI->getType())
5871 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5872 NewCall->takeName(CI);
5873 CI->replaceAllUsesWith(Res);
5874 CI->eraseFromParent();
5875 return;
5876 }
5877 case Intrinsic::nvvm_mapa_shared_cluster: {
5878 // Create a new call with the correct address space.
5879 NewCall =
5880 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5881 Value *Res = NewCall;
5882 Res = Builder.CreateAddrSpaceCast(
5883 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5884 NewCall->takeName(CI);
5885 CI->replaceAllUsesWith(Res);
5886 CI->eraseFromParent();
5887 return;
5888 }
5889 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5890 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5891 // Create a new call with the correct address space.
5892 SmallVector<Value *, 4> Args(CI->args());
5893 Args[0] = Builder.CreateAddrSpaceCast(
5894 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5895
5896 NewCall = Builder.CreateCall(NewFn, Args);
5897 NewCall->takeName(CI);
5898 CI->replaceAllUsesWith(NewCall);
5899 CI->eraseFromParent();
5900 return;
5901 }
5902 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5903 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5904 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5905 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5906 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5907 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5908 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5909 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5910 SmallVector<Value *, 16> Args(CI->args());
5911
5912 // Create AddrSpaceCast to shared_cluster if needed.
5913 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5914 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5915 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5916 Args[0] = Builder.CreateAddrSpaceCast(
5917 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5918
5919 // Attach the flag argument for cta_group, with a
5920 // default value of 0. This handles case (2) in
5921 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5922 size_t NumArgs = CI->arg_size();
5923 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5924 if (!FlagArg->getType()->isIntegerTy(1))
5925 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5926
5927 NewCall = Builder.CreateCall(NewFn, Args);
5928 NewCall->takeName(CI);
5929 CI->replaceAllUsesWith(NewCall);
5930 CI->eraseFromParent();
5931 return;
5932 }
5933 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
5934 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
5935 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
5936 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
5937 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
5938 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
5939 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
5940 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
5941 StringRef Name = F->getName();
5942 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
5943 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
5944
5945 SmallVector<Value *, 16> Args(CI->args());
5946 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
5947 NewCall = Builder.CreateCall(NewFn, Args);
5948 break;
5949 }
5950 case Intrinsic::nvvm_tcgen05_mma_shared:
5951 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5952 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5953 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
5954 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
5955 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
5956 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
5957 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
5958 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
5959 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
5960 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
5961 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
5962 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
5963 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
5964 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
5965 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
5966 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
5967 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
5968 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
5969 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
5970 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
5971 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
5972 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
5973 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
5974 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
5975 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
5976 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
5977 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
5978 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
5979 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
5980 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
5981 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
5982 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
5983 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
5984 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
5985 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
5986 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
5987 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
5988 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
5989 case Intrinsic::
5990 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
5991 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
5992 case Intrinsic::
5993 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
5994 case Intrinsic::nvvm_tcgen05_mma_tensor:
5995 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
5996 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
5997 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
5998 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
5999 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
6000 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
6001 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
6002 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
6003 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
6004 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
6005 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
6006 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
6007 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
6008 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
6009 case Intrinsic::
6010 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
6011 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
6012 case Intrinsic::
6013 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
6014 SmallVector<Value *, 12> Args(CI->args());
6015 Args.push_back(Builder.getInt32(0)); // collector_usage_b = discard(0)
6016 NewCall = Builder.CreateCall(NewFn, Args);
6017 break;
6018 }
6019 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6020 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6021 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6022 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6023 NewCall =
6024 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6025 Builder.getFalse()});
6026 break;
6027 case Intrinsic::riscv_sha256sig0:
6028 case Intrinsic::riscv_sha256sig1:
6029 case Intrinsic::riscv_sha256sum0:
6030 case Intrinsic::riscv_sha256sum1:
6031 case Intrinsic::riscv_sm3p0:
6032 case Intrinsic::riscv_sm3p1: {
6033 // The last argument to these intrinsics used to be i8 and changed to i32.
6034 // The type overload for sm4ks and sm4ed was removed.
6035 if (!CI->getType()->isIntegerTy(64))
6036 return;
6037
6038 Value *Arg =
6039 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6040
6041 NewCall = Builder.CreateCall(NewFn, Arg);
6042 Value *Res =
6043 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6044 NewCall->takeName(CI);
6045 CI->replaceAllUsesWith(Res);
6046 CI->eraseFromParent();
6047 return;
6048 }
6049
6050 case Intrinsic::x86_xop_vfrcz_ss:
6051 case Intrinsic::x86_xop_vfrcz_sd:
6052 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6053 break;
6054
6055 case Intrinsic::x86_xop_vpermil2pd:
6056 case Intrinsic::x86_xop_vpermil2ps:
6057 case Intrinsic::x86_xop_vpermil2pd_256:
6058 case Intrinsic::x86_xop_vpermil2ps_256: {
6059 SmallVector<Value *, 4> Args(CI->args());
6060 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6061 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6062 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6063 NewCall = Builder.CreateCall(NewFn, Args);
6064 break;
6065 }
6066
6067 case Intrinsic::x86_sse41_ptestc:
6068 case Intrinsic::x86_sse41_ptestz:
6069 case Intrinsic::x86_sse41_ptestnzc: {
6070 // The arguments for these intrinsics used to be v4f32, and changed
6071 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6072 // So, the only thing required is a bitcast for both arguments.
6073 // First, check the arguments have the old type.
6074 Value *Arg0 = CI->getArgOperand(0);
6075 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6076 return;
6077
6078 // Old intrinsic, add bitcasts
6079 Value *Arg1 = CI->getArgOperand(1);
6080
6081 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6082
6083 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6084 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6085
6086 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6087 break;
6088 }
6089
6090 case Intrinsic::x86_rdtscp: {
6091 // This used to take 1 arguments. If we have no arguments, it is already
6092 // upgraded.
6093 if (CI->getNumOperands() == 0)
6094 return;
6095
6096 NewCall = Builder.CreateCall(NewFn);
6097 // Extract the second result and store it.
6098 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6099 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6100 // Replace the original call result with the first result of the new call.
6101 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6102
6103 NewCall->takeName(CI);
6104 CI->replaceAllUsesWith(TSC);
6105 CI->eraseFromParent();
6106 return;
6107 }
6108
6109 case Intrinsic::x86_sse41_insertps:
6110 case Intrinsic::x86_sse41_dppd:
6111 case Intrinsic::x86_sse41_dpps:
6112 case Intrinsic::x86_sse41_mpsadbw:
6113 case Intrinsic::x86_avx_dp_ps_256:
6114 case Intrinsic::x86_avx2_mpsadbw: {
6115 // Need to truncate the last argument from i32 to i8 -- this argument models
6116 // an inherently 8-bit immediate operand to these x86 instructions.
6117 SmallVector<Value *, 4> Args(CI->args());
6118
6119 // Replace the last argument with a trunc.
6120 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6121 NewCall = Builder.CreateCall(NewFn, Args);
6122 break;
6123 }
6124
6125 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6126 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6127 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6128 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6129 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6130 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6131 SmallVector<Value *, 4> Args(CI->args());
6132 unsigned NumElts =
6133 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6134 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6135
6136 NewCall = Builder.CreateCall(NewFn, Args);
6137 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6138
6139 NewCall->takeName(CI);
6140 CI->replaceAllUsesWith(Res);
6141 CI->eraseFromParent();
6142 return;
6143 }
6144
6145 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6146 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6147 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6148 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6149 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6150 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6151 SmallVector<Value *, 4> Args(CI->args());
6152 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6153 if (NewFn->getIntrinsicID() ==
6154 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6155 Args[1] = Builder.CreateBitCast(
6156 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6157
6158 NewCall = Builder.CreateCall(NewFn, Args);
6159 Value *Res = Builder.CreateBitCast(
6160 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6161
6162 NewCall->takeName(CI);
6163 CI->replaceAllUsesWith(Res);
6164 CI->eraseFromParent();
6165 return;
6166 }
6167 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6168 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6169 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6170 SmallVector<Value *, 4> Args(CI->args());
6171 unsigned NumElts =
6172 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6173 Args[1] = Builder.CreateBitCast(
6174 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6175 Args[2] = Builder.CreateBitCast(
6176 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6177
6178 NewCall = Builder.CreateCall(NewFn, Args);
6179 break;
6180 }
6181
6182 case Intrinsic::thread_pointer: {
6183 NewCall = Builder.CreateCall(NewFn, {});
6184 break;
6185 }
6186
6187 case Intrinsic::memcpy:
6188 case Intrinsic::memmove:
6189 case Intrinsic::memset: {
6190 // We have to make sure that the call signature is what we're expecting.
6191 // We only want to change the old signatures by removing the alignment arg:
6192 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6193 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6194 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6195 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6196 // Note: i8*'s in the above can be any pointer type
6197 if (CI->arg_size() != 5) {
6198 DefaultCase();
6199 return;
6200 }
6201 // Remove alignment argument (3), and add alignment attributes to the
6202 // dest/src pointers.
6203 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6204 CI->getArgOperand(2), CI->getArgOperand(4)};
6205 NewCall = Builder.CreateCall(NewFn, Args);
6206 AttributeList OldAttrs = CI->getAttributes();
6207 AttributeList NewAttrs = AttributeList::get(
6208 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6209 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6210 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6211 NewCall->setAttributes(NewAttrs);
6212 auto *MemCI = cast<MemIntrinsic>(NewCall);
6213 // All mem intrinsics support dest alignment.
6214 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6215 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6216 // Memcpy/Memmove also support source alignment.
6217 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6218 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6219 break;
6220 }
6221
6222 case Intrinsic::masked_load:
6223 case Intrinsic::masked_gather:
6224 case Intrinsic::masked_store:
6225 case Intrinsic::masked_scatter: {
6226 if (CI->arg_size() != 4) {
6227 DefaultCase();
6228 return;
6229 }
6230
6231 auto GetMaybeAlign = [](Value *Op) {
6232 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6233 uint64_t Val = CI->getZExtValue();
6234 if (Val == 0)
6235 return MaybeAlign();
6236 if (isPowerOf2_64(Val))
6237 return MaybeAlign(Val);
6238 }
6239 reportFatalUsageError("Invalid alignment argument");
6240 };
6241 auto GetAlign = [&](Value *Op) {
6242 MaybeAlign Align = GetMaybeAlign(Op);
6243 if (Align)
6244 return *Align;
6245 reportFatalUsageError("Invalid zero alignment argument");
6246 };
6247
6248 const DataLayout &DL = CI->getDataLayout();
6249 switch (NewFn->getIntrinsicID()) {
6250 case Intrinsic::masked_load:
6251 NewCall = Builder.CreateMaskedLoad(
6252 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6253 CI->getArgOperand(2), CI->getArgOperand(3));
6254 break;
6255 case Intrinsic::masked_gather:
6256 NewCall = Builder.CreateMaskedGather(
6257 CI->getType(), CI->getArgOperand(0),
6258 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6259 CI->getType()->getScalarType()),
6260 CI->getArgOperand(2), CI->getArgOperand(3));
6261 break;
6262 case Intrinsic::masked_store:
6263 NewCall = Builder.CreateMaskedStore(
6264 CI->getArgOperand(0), CI->getArgOperand(1),
6265 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6266 break;
6267 case Intrinsic::masked_scatter:
6268 NewCall = Builder.CreateMaskedScatter(
6269 CI->getArgOperand(0), CI->getArgOperand(1),
6270 DL.getValueOrABITypeAlignment(
6271 GetMaybeAlign(CI->getArgOperand(2)),
6272 CI->getArgOperand(0)->getType()->getScalarType()),
6273 CI->getArgOperand(3));
6274 break;
6275 default:
6276 llvm_unreachable("Unexpected intrinsic ID");
6277 }
6278 // Previous metadata is still valid.
6279 NewCall->copyMetadata(*CI);
6280 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6281 break;
6282 }
6283
6284 case Intrinsic::lifetime_start:
6285 case Intrinsic::lifetime_end: {
6286 if (CI->arg_size() != 2) {
6287 DefaultCase();
6288 return;
6289 }
6290
6291 Value *Ptr = CI->getArgOperand(1);
6292 // Try to strip pointer casts, such that the lifetime works on an alloca.
6293 Ptr = Ptr->stripPointerCasts();
6294 if (isa<AllocaInst>(Ptr)) {
6295 // Don't use NewFn, as we might have looked through an addrspacecast.
6296 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6297 NewCall = Builder.CreateLifetimeStart(Ptr);
6298 else
6299 NewCall = Builder.CreateLifetimeEnd(Ptr);
6300 break;
6301 }
6302
6303 // Otherwise remove the lifetime marker.
6304 CI->eraseFromParent();
6305 return;
6306 }
6307
6308 case Intrinsic::x86_avx512_vpdpbusd_128:
6309 case Intrinsic::x86_avx512_vpdpbusd_256:
6310 case Intrinsic::x86_avx512_vpdpbusd_512:
6311 case Intrinsic::x86_avx512_vpdpbusds_128:
6312 case Intrinsic::x86_avx512_vpdpbusds_256:
6313 case Intrinsic::x86_avx512_vpdpbusds_512:
6314 case Intrinsic::x86_avx2_vpdpbssd_128:
6315 case Intrinsic::x86_avx2_vpdpbssd_256:
6316 case Intrinsic::x86_avx10_vpdpbssd_512:
6317 case Intrinsic::x86_avx2_vpdpbssds_128:
6318 case Intrinsic::x86_avx2_vpdpbssds_256:
6319 case Intrinsic::x86_avx10_vpdpbssds_512:
6320 case Intrinsic::x86_avx2_vpdpbsud_128:
6321 case Intrinsic::x86_avx2_vpdpbsud_256:
6322 case Intrinsic::x86_avx10_vpdpbsud_512:
6323 case Intrinsic::x86_avx2_vpdpbsuds_128:
6324 case Intrinsic::x86_avx2_vpdpbsuds_256:
6325 case Intrinsic::x86_avx10_vpdpbsuds_512:
6326 case Intrinsic::x86_avx2_vpdpbuud_128:
6327 case Intrinsic::x86_avx2_vpdpbuud_256:
6328 case Intrinsic::x86_avx10_vpdpbuud_512:
6329 case Intrinsic::x86_avx2_vpdpbuuds_128:
6330 case Intrinsic::x86_avx2_vpdpbuuds_256:
6331 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6332 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6333 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6334 CI->getArgOperand(2)};
6335 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6336 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6337 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6338
6339 NewCall = Builder.CreateCall(NewFn, Args);
6340 break;
6341 }
6342 case Intrinsic::x86_avx512_vpdpwssd_128:
6343 case Intrinsic::x86_avx512_vpdpwssd_256:
6344 case Intrinsic::x86_avx512_vpdpwssd_512:
6345 case Intrinsic::x86_avx512_vpdpwssds_128:
6346 case Intrinsic::x86_avx512_vpdpwssds_256:
6347 case Intrinsic::x86_avx512_vpdpwssds_512:
6348 case Intrinsic::x86_avx2_vpdpwsud_128:
6349 case Intrinsic::x86_avx2_vpdpwsud_256:
6350 case Intrinsic::x86_avx10_vpdpwsud_512:
6351 case Intrinsic::x86_avx2_vpdpwsuds_128:
6352 case Intrinsic::x86_avx2_vpdpwsuds_256:
6353 case Intrinsic::x86_avx10_vpdpwsuds_512:
6354 case Intrinsic::x86_avx2_vpdpwusd_128:
6355 case Intrinsic::x86_avx2_vpdpwusd_256:
6356 case Intrinsic::x86_avx10_vpdpwusd_512:
6357 case Intrinsic::x86_avx2_vpdpwusds_128:
6358 case Intrinsic::x86_avx2_vpdpwusds_256:
6359 case Intrinsic::x86_avx10_vpdpwusds_512:
6360 case Intrinsic::x86_avx2_vpdpwuud_128:
6361 case Intrinsic::x86_avx2_vpdpwuud_256:
6362 case Intrinsic::x86_avx10_vpdpwuud_512:
6363 case Intrinsic::x86_avx2_vpdpwuuds_128:
6364 case Intrinsic::x86_avx2_vpdpwuuds_256:
6365 case Intrinsic::x86_avx10_vpdpwuuds_512:
6366 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6367 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6368 CI->getArgOperand(2)};
6369 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6370 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6371 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6372
6373 NewCall = Builder.CreateCall(NewFn, Args);
6374 break;
6375 }
6376 assert(NewCall && "Should have either set this variable or returned through "
6377 "the default case");
6378 NewCall->takeName(CI);
6379 CI->replaceAllUsesWith(NewCall);
6380 CI->eraseFromParent();
6381}
6382
6383void llvm::UpgradeCallsToIntrinsic(Function *F) {
6384 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6385
6386 // Check if this function should be upgraded and get the replacement function
6387 // if there is one.
6388 Function *NewFn;
6389 if (UpgradeIntrinsicFunction(F, NewFn)) {
6390 // Replace all users of the old function with the new function or new
6391 // instructions. This is not a range loop because the call is deleted.
6392 for (User *U : make_early_inc_range(F->users()))
6393 if (CallBase *CB = dyn_cast<CallBase>(U))
6394 UpgradeIntrinsicCall(CB, NewFn);
6395
6396 // Remove old function, no longer used, from the module.
6397 if (F != NewFn)
6398 F->eraseFromParent();
6399 }
6400}
6401
6402MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6403 const unsigned NumOperands = MD.getNumOperands();
6404 if (NumOperands == 0)
6405 return &MD; // Invalid, punt to a verifier error.
6406
6407 // Check if the tag uses struct-path aware TBAA format.
6408 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6409 return &MD;
6410
6411 auto &Context = MD.getContext();
6412 if (NumOperands == 3) {
6413 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6414 MDNode *ScalarType = MDNode::get(Context, Elts);
6415 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6416 Metadata *Elts2[] = {ScalarType, ScalarType,
6417 ConstantAsMetadata::get(
6418 Constant::getNullValue(Type::getInt64Ty(Context))),
6419 MD.getOperand(2)};
6420 return MDNode::get(Context, Elts2);
6421 }
6422 // Create a MDNode <MD, MD, offset 0>
6423 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6424 Type::getInt64Ty(Context)))};
6425 return MDNode::get(Context, Elts);
6426}
6427
6428Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6429 Instruction *&Temp) {
6430 if (Opc != Instruction::BitCast)
6431 return nullptr;
6432
6433 Temp = nullptr;
6434 Type *SrcTy = V->getType();
6435 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6436 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6437 LLVMContext &Context = V->getContext();
6438
6439 // We have no information about target data layout, so we assume that
6440 // the maximum pointer size is 64bit.
6441 Type *MidTy = Type::getInt64Ty(Context);
6442 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6443
6444 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6445 }
6446
6447 return nullptr;
6448}
6449
6450Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6451 if (Opc != Instruction::BitCast)
6452 return nullptr;
6453
6454 Type *SrcTy = C->getType();
6455 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6456 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6457 LLVMContext &Context = C->getContext();
6458
6459 // We have no information about target data layout, so we assume that
6460 // the maximum pointer size is 64bit.
6461 Type *MidTy = Type::getInt64Ty(Context);
6462
6463 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6464 DestTy);
6465 }
6466
6467 return nullptr;
6468}
6469
6470static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6471 if (Flag.getNumOperands() < 3)
6472 return std::nullopt;
6473 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6474 return Name->getString();
6475 return std::nullopt;
6476}
6477
6478/// Check the debug info version number, if it is out-dated, drop the debug
6479/// info. Return true if module is modified.
6480bool llvm::UpgradeDebugInfo(Module &M) {
6481 if (DisableAutoUpgradeDebugInfo)
6482 return false;
6483
6484 llvm::TimeTraceScope timeScope("Upgrade debug info");
6485 // We need to get metadata before the module is verified (i.e., getModuleFlag
6486 // makes assumptions that we haven't verified yet). Carefully extract the flag
6487 // from the metadata.
6488 unsigned Version = 0;
6489 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6490 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6491 if (auto Name = getModuleFlagNameSafely(*Flag))
6492 return *Name == "Debug Info Version";
6493 return false;
6494 });
6495 if (OpIt != ModFlags->op_end()) {
6496 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6497 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6498 Version = CI->getZExtValue();
6499 }
6500 }
6501
6502 if (Version == DEBUG_METADATA_VERSION) {
6503 bool BrokenDebugInfo = false;
6504 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6505 report_fatal_error("Broken module found, compilation aborted!");
6506 if (!BrokenDebugInfo)
6507 // Everything is ok.
6508 return false;
6509 else {
6510 // Diagnose malformed debug info.
6511 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6512 M.getContext().diagnose(Diag);
6513 }
6514 }
6515 bool Modified = StripDebugInfo(M);
6516 if (Modified && Version != DEBUG_METADATA_VERSION) {
6517 // Diagnose a version mismatch.
6518 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6519 M.getContext().diagnose(DiagVersion);
6520 }
6521 return Modified;
6522}
6523
6524static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6525 GlobalValue *GV, const Metadata *V) {
6526 Function *F = cast<Function>(GV);
6527
6528 constexpr StringLiteral DefaultValue = "1";
6529 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6530 unsigned Length = 0;
6531
6532 if (F->hasFnAttribute(Attr)) {
6533 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6534 // parse these elements placing them into Vect3
6535 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6536 for (; Length < 3 && !S.empty(); Length++) {
6537 auto [Part, Rest] = S.split(',');
6538 Vect3[Length] = Part.trim();
6539 S = Rest;
6540 }
6541 }
6542
6543 const unsigned Dim = DimC - 'x';
6544 assert(Dim < 3 && "Unexpected dim char");
6545
6546 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6547
6548 // local variable required for StringRef in Vect3 to point to.
6549 const std::string VStr = llvm::utostr(VInt);
6550 Vect3[Dim] = VStr;
6551 Length = std::max(Length, Dim + 1);
6552
6553 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6554 F->addFnAttr(Attr, NewAttr);
6555}
6556
6557static inline bool isXYZ(StringRef S) {
6558 return S == "x" || S == "y" || S == "z";
6559}
6560
6561bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6562 const Metadata *V) {
6563 if (K == "kernel") {
6564 if (!mdconst::extract<ConstantInt>(V)->isZero())
6565 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6566 return true;
6567 }
6568 if (K == "align") {
6569 // V is a bitfeild specifying two 16-bit values. The alignment value is
6570 // specfied in low 16-bits, The index is specified in the high bits. For the
6571 // index, 0 indicates the return value while higher values correspond to
6572 // each parameter (idx = param + 1).
6573 const uint64_t AlignIdxValuePair =
6574 mdconst::extract<ConstantInt>(V)->getZExtValue();
6575 const unsigned Idx = (AlignIdxValuePair >> 16);
6576 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6577 cast<Function>(GV)->addAttributeAtIndex(
6578 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6579 return true;
6580 }
6581 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6582 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6583 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6584 return true;
6585 }
6586 if (K == "minctasm") {
6587 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6588 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6589 return true;
6590 }
6591 if (K == "maxnreg") {
6592 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6593 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6594 return true;
6595 }
6596 if (K.consume_front("maxntid") && isXYZ(K)) {
6597 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6598 return true;
6599 }
6600 if (K.consume_front("reqntid") && isXYZ(K)) {
6601 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6602 return true;
6603 }
6604 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6605 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6606 return true;
6607 }
6608 if (K == "grid_constant") {
6609 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6610 for (const auto &Op : cast<MDNode>(V)->operands()) {
6611 // For some reason, the index is 1-based in the metadata. Good thing we're
6612 // able to auto-upgrade it!
6613 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6614 cast<Function>(GV)->addParamAttr(Index, Attr);
6615 }
6616 return true;
6617 }
6618
6619 return false;
6620}
6621
6622void llvm::UpgradeNVVMAnnotations(Module &M) {
6623 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6624 if (!NamedMD)
6625 return;
6626
6627 SmallVector<MDNode *, 8> NewNodes;
6628 SmallPtrSet<const MDNode *, 8> SeenNodes;
6629 for (MDNode *MD : NamedMD->operands()) {
6630 if (!SeenNodes.insert(MD).second)
6631 continue;
6632
6633 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6634 if (!GV)
6635 continue;
6636
6637 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6638
6639 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6640 // Each nvvm.annotations metadata entry will be of the following form:
6641 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6642 // start index = 1, to skip the global variable key
6643 // increment = 2, to skip the value for each property-value pairs
6644 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6645 MDString *K = cast<MDString>(MD->getOperand(j));
6646 const MDOperand &V = MD->getOperand(j + 1);
6647 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6648 if (!Upgraded)
6649 NewOperands.append({K, V});
6650 }
6651
6652 if (NewOperands.size() > 1)
6653 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6654 }
6655
6656 NamedMD->clearOperands();
6657 for (MDNode *N : NewNodes)
6658 NamedMD->addOperand(N);
6659}
6660
6661/// This checks for objc retain release marker which should be upgraded. It
6662/// returns true if module is modified.
6663static bool upgradeRetainReleaseMarker(Module &M) {
6664 bool Changed = false;
6665 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6666 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6667 if (ModRetainReleaseMarker) {
6668 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6669 if (Op) {
6670 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6671 if (ID) {
6672 SmallVector<StringRef, 4> ValueComp;
6673 ID->getString().split(ValueComp, "#");
6674 if (ValueComp.size() == 2) {
6675 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6676 ID = MDString::get(M.getContext(), NewValue);
6677 }
6678 M.addModuleFlag(Module::Error, MarkerKey, ID);
6679 M.eraseNamedMetadata(ModRetainReleaseMarker);
6680 Changed = true;
6681 }
6682 }
6683 }
6684 return Changed;
6685}
6686
6687void llvm::UpgradeARCRuntime(Module &M) {
6688 // This lambda converts normal function calls to ARC runtime functions to
6689 // intrinsic calls.
6690 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6691 llvm::Intrinsic::ID IntrinsicFunc) {
6692 Function *Fn = M.getFunction(OldFunc);
6693
6694 if (!Fn)
6695 return;
6696
6697 Function *NewFn =
6698 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6699
6700 for (User *U : make_early_inc_range(Fn->users())) {
6701 CallInst *CI = dyn_cast<CallInst>(U);
6702 if (!CI || CI->getCalledFunction() != Fn)
6703 continue;
6704
6705 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6706 FunctionType *NewFuncTy = NewFn->getFunctionType();
6707 SmallVector<Value *, 2> Args;
6708
6709 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6710 // value to the return type of the old function.
6711 if (NewFuncTy->getReturnType() != CI->getType() &&
6712 !CastInst::castIsValid(Instruction::BitCast, CI,
6713 NewFuncTy->getReturnType()))
6714 continue;
6715
6716 bool InvalidCast = false;
6717
6718 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6719 Value *Arg = CI->getArgOperand(I);
6720
6721 // Bitcast argument to the parameter type of the new function if it's
6722 // not a variadic argument.
6723 if (I < NewFuncTy->getNumParams()) {
6724 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6725 // to the parameter type of the new function.
6726 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6727 NewFuncTy->getParamType(I))) {
6728 InvalidCast = true;
6729 break;
6730 }
6731 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6732 }
6733 Args.push_back(Arg);
6734 }
6735
6736 if (InvalidCast)
6737 continue;
6738
6739 // Create a call instruction that calls the new function.
6740 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6741 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6742 NewCall->takeName(CI);
6743
6744 // Bitcast the return value back to the type of the old call.
6745 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6746
6747 if (!CI->use_empty())
6748 CI->replaceAllUsesWith(NewRetVal);
6749 CI->eraseFromParent();
6750 }
6751
6752 if (Fn->use_empty())
6753 Fn->eraseFromParent();
6754 };
6755
6756 // Unconditionally convert a call to "clang.arc.use" to a call to
6757 // "llvm.objc.clang.arc.use".
6758 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6759
6760 // Upgrade the retain release marker. If there is no need to upgrade
6761 // the marker, that means either the module is already new enough to contain
6762 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6763 if (!upgradeRetainReleaseMarker(M))
6764 return;
6765
6766 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6767 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6768 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6769 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6770 {"objc_autoreleaseReturnValue",
6771 llvm::Intrinsic::objc_autoreleaseReturnValue},
6772 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6773 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6774 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6775 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6776 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6777 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6778 {"objc_release", llvm::Intrinsic::objc_release},
6779 {"objc_retain", llvm::Intrinsic::objc_retain},
6780 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6781 {"objc_retainAutoreleaseReturnValue",
6782 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6783 {"objc_retainAutoreleasedReturnValue",
6784 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6785 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6786 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6787 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6788 {"objc_unsafeClaimAutoreleasedReturnValue",
6789 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6790 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6791 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6792 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6793 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6794 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6795 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6796 {"objc_arc_annotation_topdown_bbstart",
6797 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6798 {"objc_arc_annotation_topdown_bbend",
6799 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6800 {"objc_arc_annotation_bottomup_bbstart",
6801 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6802 {"objc_arc_annotation_bottomup_bbend",
6803 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6804
6805 for (auto &I : RuntimeFuncs)
6806 UpgradeToIntrinsic(I.first, I.second);
6807}
6808
6809// Upgrade the way signing of pointers to init/fini functions is described.
6810//
6811// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6812// constants, if signing was requested. After the upgrade, these arrays contain
6813// plain function pointers and the desired signing schema is described via a
6814// pair of module flags.
6815//
6816// Note that the upgrade is only performed if all elements of *both* arrays
6817// agree on a common signing schema.
6818static bool upgradePtrauthInitFiniArrays(Module &M) {
6819 // As we cannot always decide whether the particular module should have
6820 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6821 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6822 // returns as soon as it spots any non-signed init/fini pointer: either we
6823 // should request non-signed pointers (safe to omit both flags) or there is
6824 // no common schema (and thus we do not modify anything).
6825 //
6826 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6827 // or whether we should request address diversity in addition to the basic
6828 // constant diversity. There is no value representing "decided not to sign"
6829 // for the reasons explained above.
6830 std::optional<bool> UseAddressDisc;
6831
6832 // Do not attempt upgrading if the new module flags already exist.
6833 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6834 for (const MDNode *Flag : ModFlags->operands()) {
6835 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
6836 if (Name && (*Name == "ptrauth-init-fini" ||
6837 *Name == "ptrauth-init-fini-address-discrimination"))
6838 return false;
6839 }
6840 }
6841
6842 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6843 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6844 constexpr unsigned ExpectedAddressMarker = 1;
6845
6846 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
6847 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
6848 return nullptr; // Nothing to upgrade or unknown pattern found.
6849
6850 bool HasAddressDisc;
6851 if (!CPA->hasAddressDiscriminator())
6852 HasAddressDisc = false;
6853 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
6854 HasAddressDisc = true;
6855 else
6856 return nullptr; // Unknown pattern.
6857
6858 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6859 return nullptr; // Disagreement with the decided mode.
6860
6861 UseAddressDisc = HasAddressDisc;
6862 return CPA->getPointer();
6863 };
6864
6865 // Do not apply any changes until we know the upgrade is non-ambiguous.
6866 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6867 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6868
6869 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6870 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
6871 if (!GV || !GV->hasInitializer())
6872 continue; // Skip, but it is okay to upgrade the other variable.
6873
6874 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
6875 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6876 return false;
6877
6878 std::vector<Constant *> NewStructors;
6879 NewStructors.reserve(OldStructorsArray->getNumOperands());
6880
6881 for (Use &U : OldStructorsArray->operands()) {
6882 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
6883 if (!Structor || Structor->getNumOperands() != 3)
6884 return false;
6885
6886 Constant *Prio = Structor->getOperand(0);
6887 Constant *Func = Structor->getOperand(1);
6888 Constant *Arg = Structor->getOperand(2);
6889
6890 Func = UpgradeSinglePointer(Func);
6891 if (!Func)
6892 return false;
6893
6894 NewStructors.push_back(
6895 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
6896 }
6897
6898 Constant *NewInit =
6899 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
6900 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
6901 }
6902
6903 if (GlobalArraysToUpgrade.empty())
6904 return false;
6905 assert(UseAddressDisc.has_value());
6906
6907 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6908 GV->setInitializer(NewInit);
6909
6910 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
6911 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
6912 *UseAddressDisc);
6913
6914 return true;
6915}
6916
6917bool llvm::UpgradeModuleFlags(Module &M) {
6918 bool Changed = false;
6919 Changed |= upgradePtrauthInitFiniArrays(M);
6920
6921 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6922 if (!ModFlags)
6923 return Changed;
6924
6925 bool HasObjCFlag = false, HasClassProperties = false;
6926 bool HasSwiftVersionFlag = false;
6927 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6928 uint32_t SwiftABIVersion;
6929 auto Int8Ty = Type::getInt8Ty(M.getContext());
6930 auto Int32Ty = Type::getInt32Ty(M.getContext());
6931
6932 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6933 MDNode *Op = ModFlags->getOperand(I);
6934 if (Op->getNumOperands() != 3)
6935 continue;
6936 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6937 if (!ID)
6938 continue;
6939 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6940 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6941 Type::getInt32Ty(M.getContext()), B)),
6942 MDString::get(M.getContext(), ID->getString()),
6943 Op->getOperand(2)};
6944 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6945 Changed = true;
6946 };
6947
6948 if (ID->getString() == "Objective-C Image Info Version")
6949 HasObjCFlag = true;
6950 if (ID->getString() == "Objective-C Class Properties")
6951 HasClassProperties = true;
6952 // Upgrade PIC from Error/Max to Min.
6953 if (ID->getString() == "PIC Level") {
6954 if (auto *Behavior =
6955 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6956 uint64_t V = Behavior->getLimitedValue();
6957 if (V == Module::Error || V == Module::Max)
6958 SetBehavior(Module::Min);
6959 }
6960 }
6961 // Upgrade "PIE Level" from Error to Max.
6962 if (ID->getString() == "PIE Level")
6963 if (auto *Behavior =
6964 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
6965 if (Behavior->getLimitedValue() == Module::Error)
6966 SetBehavior(Module::Max);
6967
6968 // Upgrade branch protection and return address signing module flags. The
6969 // module flag behavior for these fields were Error and now they are Min.
6970 if (ID->getString() == "branch-target-enforcement" ||
6971 ID->getString().starts_with("sign-return-address")) {
6972 if (auto *Behavior =
6973 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6974 if (Behavior->getLimitedValue() == Module::Error) {
6975 Type *Int32Ty = Type::getInt32Ty(M.getContext());
6976 Metadata *Ops[3] = {
6977 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
6978 Op->getOperand(1), Op->getOperand(2)};
6979 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6980 Changed = true;
6981 }
6982 }
6983 }
6984
6985 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
6986 // section name so that llvm-lto will not complain about mismatching
6987 // module flags that is functionally the same.
6988 if (ID->getString() == "Objective-C Image Info Section") {
6989 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
6990 SmallVector<StringRef, 4> ValueComp;
6991 Value->getString().split(ValueComp, " ");
6992 if (ValueComp.size() != 1) {
6993 std::string NewValue;
6994 for (auto &S : ValueComp)
6995 NewValue += S.str();
6996 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
6997 MDString::get(M.getContext(), NewValue)};
6998 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6999 Changed = true;
7000 }
7001 }
7002 }
7003
7004 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7005 // If the higher bits are set, it adds new module flag for swift info.
7006 if (ID->getString() == "Objective-C Garbage Collection") {
7007 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7008 if (Md) {
7009 assert(Md->getValue() && "Expected non-empty metadata");
7010 auto Type = Md->getValue()->getType();
7011 if (Type == Int8Ty)
7012 continue;
7013 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7014 if ((Val & 0xff) != Val) {
7015 HasSwiftVersionFlag = true;
7016 SwiftABIVersion = (Val & 0xff00) >> 8;
7017 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7018 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7019 }
7020 Metadata *Ops[3] = {
7021 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7022 Op->getOperand(1),
7023 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7024 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7025 Changed = true;
7026 }
7027 }
7028
7029 if (ID->getString() == "amdgpu_code_object_version") {
7030 Metadata *Ops[3] = {
7031 Op->getOperand(0),
7032 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7033 Op->getOperand(2)};
7034 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7035 Changed = true;
7036 }
7037
7038 // clang/PowerPC used to use "float-abi" to describe the long double format;
7039 // it has been renamed to "long-double-type", with its values changed to the
7040 // corresponding IR floating-point type names.
7041 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7042 StringRef Format;
7043 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7044 Format = S->getString();
7045
7046 // The "float-abi" key is now reserved for the target-independent
7047 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7048 // (including unrecognized ones, which were never valid) to the default.
7049 if (!FloatABI::parseABIType(Format)) {
7050 LongDoubleFormat NewFormat =
7051 StringSwitch<LongDoubleFormat>(Format)
7052 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7053 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7054 .Default(LongDoubleFormat::PPCDoubleDouble);
7055 Metadata *Ops[3] = {
7056 Op->getOperand(0),
7057 MDString::get(M.getContext(), "long-double-type"),
7058 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7059 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7060 Changed = true;
7061 }
7062 }
7063 }
7064
7065 // "Objective-C Class Properties" is recently added for Objective-C. We
7066 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7067 // flag of value 0, so we can correclty downgrade this flag when trying to
7068 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7069 // this module flag.
7070 if (HasObjCFlag && !HasClassProperties) {
7071 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7072 (uint32_t)0);
7073 Changed = true;
7074 }
7075
7076 if (HasSwiftVersionFlag) {
7077 M.addModuleFlag(Module::Error, "Swift ABI Version",
7078 SwiftABIVersion);
7079 M.addModuleFlag(Module::Error, "Swift Major Version",
7080 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7081 M.addModuleFlag(Module::Error, "Swift Minor Version",
7082 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7083 Changed = true;
7084 }
7085
7086 return Changed;
7087}
7088
7089bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7090 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7091 // If this metadata has operands, we expect all of them to be either from
7092 // before or from after the format change handled here, so we can bail out
7093 // fast if the first (if any) operands is of the new format.
7094 auto MatchesVersion = [](const MDNode *Op) {
7095 return Op->getNumOperands() >= 3 &&
7096 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7097 cast<ConstantAsMetadata>(Op->getOperand(2))
7098 ->getType()
7099 ->isIntegerTy(64);
7100 };
7101
7102 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7103 MatchesVersion(CFIConsts->getOperand(0)))
7104 return false;
7105
7106 bool Changed = false;
7107 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7108 MDNode *Op = CFIConsts->getOperand(I);
7109 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7110 assert(Op->getNumOperands() >= 2 &&
7111 "Expected at least 2 operands - name and linkage type");
7112 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7113 StringRef Name = NameMD->getString();
7114 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7115 GlobalValue::dropLLVMManglingEscape(Name));
7116
7117 SmallVector<Metadata *, 4> Elts;
7118 Elts.push_back(Op->getOperand(0));
7119 Elts.push_back(Op->getOperand(1));
7120 Elts.push_back(ConstantAsMetadata::get(
7121 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7122
7123 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7124 Elts.push_back(Op->getOperand(J));
7125
7126 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7127 Changed = true;
7128 }
7129
7130 return Changed;
7131}
7132
7133void llvm::UpgradeSectionAttributes(Module &M) {
7134 auto TrimSpaces = [](StringRef Section) -> std::string {
7135 SmallVector<StringRef, 5> Components;
7136 Section.split(Components, ',');
7137
7138 SmallString<32> Buffer;
7139 raw_svector_ostream OS(Buffer);
7140
7141 for (auto Component : Components)
7142 OS << ',' << Component.trim();
7143
7144 return std::string(OS.str().substr(1));
7145 };
7146
7147 for (auto &GV : M.globals()) {
7148 if (!GV.hasSection())
7149 continue;
7150
7151 StringRef Section = GV.getSection();
7152
7153 if (!Section.starts_with("__DATA, __objc_catlist"))
7154 continue;
7155
7156 // __DATA, __objc_catlist, regular, no_dead_strip
7157 // __DATA,__objc_catlist,regular,no_dead_strip
7158 GV.setSection(TrimSpaces(Section));
7159 }
7160}
7161
7162namespace {
7163// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7164// callsites within a function that did not also have the strictfp attribute.
7165// Since 10.0, if strict FP semantics are needed within a function, the
7166// function must have the strictfp attribute and all calls within the function
7167// must also have the strictfp attribute. This latter restriction is
7168// necessary to prevent unwanted libcall simplification when a function is
7169// being cloned (such as for inlining).
7170//
7171// The "dangling" strictfp attribute usage was only used to prevent constant
7172// folding and other libcall simplification. The nobuiltin attribute on the
7173// callsite has the same effect.
7174struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7175 StrictFPUpgradeVisitor() = default;
7176
7177 void visitCallBase(CallBase &Call) {
7178 if (!Call.isStrictFP())
7179 return;
7180 if (isa<ConstrainedFPIntrinsic>(&Call))
7181 return;
7182 // If we get here, the caller doesn't have the strictfp attribute
7183 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7184 Call.removeFnAttr(Attribute::StrictFP);
7185 Call.addFnAttr(Attribute::NoBuiltin);
7186 }
7187};
7188
7189/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7190struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7191 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7192 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7193
7194 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7195 if (!RMW.isFloatingPointOperation())
7196 return;
7197
7198 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7199 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7200 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7201 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
7202 }
7203};
7204} // namespace
7205
7206void llvm::UpgradeFunctionAttributes(Function &F) {
7207 // If a function definition doesn't have the strictfp attribute,
7208 // convert any callsite strictfp attributes to nobuiltin.
7209 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7210 StrictFPUpgradeVisitor SFPV;
7211 SFPV.visit(F);
7212 }
7213
7214 // Remove all incompatibile attributes from function.
7215 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7216 F.getReturnType(), F.getAttributes().getRetAttrs()));
7217 for (auto &Arg : F.args())
7218 Arg.removeAttrs(
7219 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7220
7221 bool AddingAttrs = false, RemovingAttrs = false;
7222 AttrBuilder AttrsToAdd(F.getContext());
7223 AttributeMask AttrsToRemove;
7224
7225 // Older versions of LLVM treated an "implicit-section-name" attribute
7226 // similarly to directly setting the section on a Function.
7227 if (Attribute A = F.getFnAttribute("implicit-section-name");
7228 A.isValid() && A.isStringAttribute()) {
7229 F.setSection(A.getValueAsString());
7230 AttrsToRemove.addAttribute("implicit-section-name");
7231 RemovingAttrs = true;
7232 }
7233
7234 if (Attribute A = F.getFnAttribute("nooutline");
7235 A.isValid() && A.isStringAttribute()) {
7236 AttrsToRemove.addAttribute("nooutline");
7237 AttrsToAdd.addAttribute(Attribute::NoOutline);
7238 AddingAttrs = RemovingAttrs = true;
7239 }
7240
7241 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7242 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7243 AttrsToRemove.addAttribute("uniform-work-group-size");
7244 RemovingAttrs = true;
7245 if (A.getValueAsString() == "true") {
7246 AttrsToAdd.addAttribute("uniform-work-group-size");
7247 AddingAttrs = true;
7248 }
7249 }
7250
7251 if (!F.empty()) {
7252 // For some reason this is called twice, and the first time is before any
7253 // instructions are loaded into the body.
7254
7255 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7256 A.isValid()) {
7257
7258 if (A.getValueAsBool()) {
7259 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7260 Visitor.visit(F);
7261 }
7262
7263 // We will leave behind dead attribute uses on external declarations, but
7264 // clang never added these to declarations anyway.
7265 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7266 RemovingAttrs = true;
7267 }
7268 }
7269
7270 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7271 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7272
7273 bool HandleDenormalMode = false;
7274
7275 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7276 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7277 if (ParsedMode.isValid()) {
7278 DenormalFPMath = ParsedMode;
7279 AttrsToRemove.addAttribute("denormal-fp-math");
7280 AddingAttrs = RemovingAttrs = true;
7281 HandleDenormalMode = true;
7282 }
7283 }
7284
7285 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7286 Attr.isValid()) {
7287 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7288 if (ParsedMode.isValid()) {
7289 DenormalFPMathF32 = ParsedMode;
7290 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7291 AddingAttrs = RemovingAttrs = true;
7292 HandleDenormalMode = true;
7293 }
7294 }
7295
7296 if (HandleDenormalMode)
7297 AttrsToAdd.addDenormalFPEnvAttr(
7298 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7299
7300 if (RemovingAttrs)
7301 F.removeFnAttrs(AttrsToRemove);
7302
7303 if (AddingAttrs)
7304 F.addFnAttrs(AttrsToAdd);
7305}
7306
7307// Check if the function attribute is not present and set it.
7308static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7309 StringRef Value) {
7310 if (!F.hasFnAttribute(FnAttrName))
7311 F.addFnAttr(FnAttrName, Value);
7312}
7313
7314// Check if the function attribute is not present and set it if needed.
7315// If the attribute is "false" then removes it.
7316// If the attribute is "true" resets it to a valueless attribute.
7317static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7318 if (!F.hasFnAttribute(FnAttrName)) {
7319 if (Set)
7320 F.addFnAttr(FnAttrName);
7321 } else {
7322 auto A = F.getFnAttribute(FnAttrName);
7323 if ("false" == A.getValueAsString())
7324 F.removeFnAttr(FnAttrName);
7325 else if ("true" == A.getValueAsString()) {
7326 F.removeFnAttr(FnAttrName);
7327 F.addFnAttr(FnAttrName);
7328 }
7329 }
7330}
7331
7332void llvm::copyModuleAttrToFunctions(Module &M) {
7333 Triple T(M.getTargetTriple());
7334 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7335 return;
7336
7337 uint64_t BTEValue = 0;
7338 uint64_t BPPLRValue = 0;
7339 uint64_t GCSValue = 0;
7340 uint64_t SRAValue = 0;
7341 uint64_t SRAALLValue = 0;
7342 uint64_t SRABKeyValue = 0;
7343
7344 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7345 if (ModFlags) {
7346 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7347 MDNode *Op = ModFlags->getOperand(I);
7348 if (Op->getNumOperands() != 3)
7349 continue;
7350
7351 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7352 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7353 if (!ID || !CI)
7354 continue;
7355
7356 StringRef IDStr = ID->getString();
7357 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7358 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7359 : IDStr == "guarded-control-stack" ? &GCSValue
7360 : IDStr == "sign-return-address" ? &SRAValue
7361 : IDStr == "sign-return-address-all" ? &SRAALLValue
7362 : IDStr == "sign-return-address-with-bkey"
7363 ? &SRABKeyValue
7364 : nullptr;
7365 if (!ValPtr)
7366 continue;
7367
7368 *ValPtr = CI->getZExtValue();
7369 if (*ValPtr == 2)
7370 return;
7371 }
7372 }
7373
7374 bool BTE = BTEValue == 1;
7375 bool BPPLR = BPPLRValue == 1;
7376 bool GCS = GCSValue == 1;
7377 bool SRA = SRAValue == 1;
7378
7379 StringRef SignTypeValue = "non-leaf";
7380 if (SRA && SRAALLValue == 1)
7381 SignTypeValue = "all";
7382
7383 StringRef SignKeyValue = "a_key";
7384 if (SRA && SRABKeyValue == 1)
7385 SignKeyValue = "b_key";
7386
7387 for (Function &F : M.getFunctionList()) {
7388 if (F.isDeclaration())
7389 continue;
7390
7391 if (SRA) {
7392 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7393 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7394 } else {
7395 if (auto A = F.getFnAttribute("sign-return-address");
7396 A.isValid() && "none" == A.getValueAsString()) {
7397 F.removeFnAttr("sign-return-address");
7398 F.removeFnAttr("sign-return-address-key");
7399 }
7400 }
7401 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7402 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7403 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7404 }
7405
7406 if (BTE)
7407 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7408 if (BPPLR)
7409 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7410 if (GCS)
7411 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7412 if (SRA) {
7413 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7414 if (SRAALLValue == 1)
7415 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7416 if (SRABKeyValue == 1)
7417 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7418 }
7419}
7420
7421/// Return the replacement tags if \p T still uses a removed two-operand form.
7422static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7423 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7424 return nullptr;
7425 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7426 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7427}
7428
7429/// Build the single-operand node that replaces a boolean operand: nonzero
7430/// selects the enable tag, zero the disable tag.
7431static Metadata *makeBooleanLoopNode(LLVMContext &C,
7432 const BooleanLoopTags &Tags,
7433 const MDOperand &Op) {
7434 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7435 return MDTuple::get(C,
7436 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7437}
7438
7439static bool isOldLoopArgument(Metadata *MD) {
7440 auto *T = dyn_cast_or_null<MDTuple>(MD);
7441 if (!T)
7442 return false;
7443 if (T->getNumOperands() < 1)
7444 return false;
7445 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7446 if (!S)
7447 return false;
7448 if (S->getString().starts_with("llvm.vectorizer."))
7449 return true;
7450 return getOldBooleanLoopTags(T) != nullptr;
7451}
7452
7453static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7454 StringRef OldPrefix = "llvm.vectorizer.";
7455 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7456
7457 if (OldTag == "llvm.vectorizer.unroll")
7458 return MDString::get(C, "llvm.loop.interleave.count");
7459
7460 return MDString::get(
7461 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7462 .str());
7463}
7464
7465static Metadata *upgradeLoopArgument(Metadata *MD) {
7466 auto *T = dyn_cast_or_null<MDTuple>(MD);
7467 if (!T)
7468 return MD;
7469 if (T->getNumOperands() < 1)
7470 return MD;
7471 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7472 if (!OldTag)
7473 return MD;
7474
7475 LLVMContext &C = T->getContext();
7476
7477 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7478 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7479 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7480
7481 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7482 return MD;
7483
7484 // This has an old tag. Upgrade it.
7485 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7486
7487 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7488 // vectorize.enable/disable pair, not a two-operand enable node.
7489 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7490 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7491 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7492
7493 SmallVector<Metadata *, 8> Ops;
7494 Ops.reserve(T->getNumOperands());
7495 Ops.push_back(NewTag);
7496 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7497 Ops.push_back(T->getOperand(I));
7498
7499 return MDTuple::get(C, Ops);
7500}
7501
7502MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7503 auto *T = dyn_cast<MDTuple>(&N);
7504 if (!T)
7505 return &N;
7506
7507 if (none_of(T->operands(), isOldLoopArgument))
7508 return &N;
7509
7510 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7511 // any MDNode carrying those tags with more than one operand, so a leftover
7512 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7513 // In-place mutation is safe on distinct MDNodes.
7514 if (T->isDistinct()) {
7515 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7516 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7517 if (OpT && getOldBooleanLoopTags(OpT))
7518 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7519 }
7520 if (none_of(T->operands(), isOldLoopArgument))
7521 return &N;
7522 }
7523
7524 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7525 // attachment; the original distinct loop-ID is kept as the first operand.
7526 SmallVector<Metadata *, 8> Ops;
7527 Ops.reserve(T->getNumOperands());
7528 for (Metadata *MD : T->operands())
7529 Ops.push_back(upgradeLoopArgument(MD));
7530
7531 return MDTuple::get(T->getContext(), Ops);
7532}
7533
7534std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7535 Triple T(TT);
7536 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7537 // the address space of globals to 1. This does not apply to SPIRV Logical.
7538 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7539 !DL.contains("-G") && !DL.starts_with("G")) {
7540 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7541 }
7542
7543 if (T.isLoongArch64() || T.isRISCV64()) {
7544 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7545 auto I = DL.find("-n64-");
7546 if (I != StringRef::npos)
7547 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7548 return DL.str();
7549 }
7550
7551 // AMDGPU data layout upgrades.
7552 std::string Res = DL.str();
7553 if (T.isAMDGPU()) {
7554 // Define address spaces for constants.
7555 if (!DL.contains("-G") && !DL.starts_with("G"))
7556 Res.append(Res.empty() ? "G1" : "-G1");
7557
7558 // AMDGCN data layout upgrades.
7559 if (T.isAMDGCN()) {
7560
7561 // Add missing non-integral declarations.
7562 // This goes before adding new address spaces to prevent incoherent string
7563 // values.
7564 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7565 Res.append("-ni:7:8:9");
7566 // Update ni:7 to ni:7:8:9.
7567 if (DL.ends_with("ni:7"))
7568 Res.append(":8:9");
7569 if (DL.ends_with("ni:7:8"))
7570 Res.append(":9");
7571
7572 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7573 // resources) An empty data layout has already been upgraded to G1 by now.
7574 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7575 Res.append("-p7:160:256:256:32");
7576 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7577 Res.append("-p8:128:128:128:48");
7578 constexpr StringRef OldP8("-p8:128:128-");
7579 if (DL.contains(OldP8))
7580 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7581 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7582 Res.append("-p9:192:256:256:32");
7583 }
7584
7585 // Upgrade the ELF mangling mode.
7586 if (!DL.contains("m:e"))
7587 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7588
7589 return Res;
7590 }
7591
7592 if (T.isSystemZ() && !DL.empty()) {
7593 // Make sure the stack alignment is present.
7594 if (!DL.contains("-S64"))
7595 return "E-S64" + DL.drop_front(1).str();
7596 return DL.str();
7597 }
7598
7599 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7600 // If the datalayout matches the expected format, add pointer size address
7601 // spaces to the datalayout.
7602 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7603 if (!DL.contains(AddrSpaces)) {
7604 SmallVector<StringRef, 4> Groups;
7605 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7606 if (R.match(Res, &Groups))
7607 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7608 }
7609 };
7610
7611 // AArch64 data layout upgrades.
7612 if (T.isAArch64()) {
7613 // Add "-Fn32"
7614 if (!DL.empty() && !DL.contains("-Fn32"))
7615 Res.append("-Fn32");
7616 AddPtr32Ptr64AddrSpaces();
7617 return Res;
7618 }
7619
7620 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7621 T.isWasm()) {
7622 // Mips64 with o32 ABI did not add "-i128:128".
7623 // Add "-i128:128"
7624 std::string I64 = "-i64:64";
7625 std::string I128 = "-i128:128";
7626 if (!StringRef(Res).contains(I128)) {
7627 size_t Pos = Res.find(I64);
7628 if (Pos != size_t(-1))
7629 Res.insert(Pos + I64.size(), I128);
7630 }
7631 }
7632
7633 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7634 size_t Pos = Res.find("-S128");
7635 if (Pos == StringRef::npos)
7636 Pos = Res.size();
7637 Res.insert(Pos, "-f64:32:64");
7638 }
7639
7640 if (!T.isX86())
7641 return Res;
7642
7643 AddPtr32Ptr64AddrSpaces();
7644
7645 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7646 // for i128 operations prior to this being reflected in the data layout, and
7647 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7648 // boundaries, so although this is a breaking change, the upgrade is expected
7649 // to fix more IR than it breaks.
7650 // Intel MCU is an exception and uses 4-byte-alignment.
7651 if (!T.isOSIAMCU()) {
7652 std::string I128 = "-i128:128";
7653 if (StringRef Ref = Res; !Ref.contains(I128)) {
7654 SmallVector<StringRef, 4> Groups;
7655 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7656 if (R.match(Res, &Groups))
7657 Res = (Groups[1] + I128 + Groups[3]).str();
7658 }
7659 }
7660
7661 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7662 // Raising the alignment is safe because Clang did not produce f80 values in
7663 // the MSVC environment before this upgrade was added.
7664 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7665 StringRef Ref = Res;
7666 auto I = Ref.find("-f80:32-");
7667 if (I != StringRef::npos)
7668 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7669 }
7670
7671 return Res;
7672}
7673
7674void llvm::UpgradeAttributes(AttrBuilder &B) {
7675 StringRef FramePointer;
7676 Attribute A = B.getAttribute("no-frame-pointer-elim");
7677 if (A.isValid()) {
7678 // The value can be "true" or "false".
7679 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7680 B.removeAttribute("no-frame-pointer-elim");
7681 }
7682 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7683 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7684 if (FramePointer != "all")
7685 FramePointer = "non-leaf";
7686 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7687 }
7688 if (!FramePointer.empty())
7689 B.addAttribute("frame-pointer", FramePointer);
7690
7691 A = B.getAttribute("null-pointer-is-valid");
7692 if (A.isValid()) {
7693 // The value can be "true" or "false".
7694 bool NullPointerIsValid = A.getValueAsString() == "true";
7695 B.removeAttribute("null-pointer-is-valid");
7696 if (NullPointerIsValid)
7697 B.addAttribute(Attribute::NullPointerIsValid);
7698 }
7699
7700 A = B.getAttribute("uniform-work-group-size");
7701 if (A.isValid()) {
7702 StringRef Val = A.getValueAsString();
7703 if (!Val.empty()) {
7704 bool IsTrue = Val == "true";
7705 B.removeAttribute("uniform-work-group-size");
7706 if (IsTrue)
7707 B.addAttribute("uniform-work-group-size");
7708 }
7709 }
7710}
7711
7712void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7713 // clang.arc.attachedcall bundles are now required to have an operand.
7714 // If they don't, it's okay to drop them entirely: when there is an operand,
7715 // the "attachedcall" is meaningful and required, but without an operand,
7716 // it's just a marker NOP. Dropping it merely prevents an optimization.
7717 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7718 return OBD.getTag() == "clang.arc.attachedcall" &&
7719 OBD.inputs().empty();
7720 });
7721}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:482
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:483
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1513
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2456
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7465
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6557
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1543
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2296
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1254
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5409
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1221
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1234
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6663
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2557
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2783
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1493
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2531
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1446
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7308
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1325
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6561
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5269
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7453
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5369
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6524
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2411
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2668
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1489
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2382
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4930
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2769
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7422
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2708
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1305
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6470
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1403
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7431
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2330
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1409
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1280
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1171
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7439
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5000
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5339
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5090
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2647
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5281
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5440
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2362
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2394
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2596
shouldUpgradeNVPTXTcgen05MMAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1391
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2730
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2516
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3040
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2762
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2792
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5291
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7317
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2678
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2629
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5352
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6818
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3238
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5387
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:769
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:929
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:781
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:785
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:806
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:845
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:799
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:789
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:795
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:849
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:837
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:797
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:817
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:813
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:841
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:976
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:507
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:355
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1590
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1710
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3105
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3388
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3474
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:168
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:211
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:246
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:166
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:448
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:885
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:216
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:870
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2903
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:321
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:519
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:317
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1792
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:700
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:316
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1500
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:310
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:190
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:633
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:615
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:111
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:117
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:138
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:120
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:152
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:149
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1576
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1568
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1564
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1583
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1574
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2090
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:386
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:532
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:977
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:702
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1236
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:331
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:823
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5276
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2356
Changed
Changed
Definition ObjCARCOpts.cpp:2366
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1418
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:758
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1399
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1392
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:578
llvm::Length
@ Length
Definition DWP.cpp:578
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5495
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7133
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3031
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2244
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7502
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7674
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6383
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6622
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6917
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7089
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7332
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7712
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6450
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7534
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2263
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:322
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6428
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6480
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7206
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6402
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6687
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7722
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0