70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
248 return isRegOrInline(RCID, type) || isLiteralImm(type);
251 bool isRegOrImmWithInt16InputMods()
const {
255 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
257 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
260 bool isRegOrImmWithInt32InputMods()
const {
264 bool isRegOrInlineImmWithInt16InputMods()
const {
265 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFP32InputMods()
const {
294 bool isRegOrImmWithFP64InputMods()
const {
298 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
299 return isRegOrInline(
300 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
303 bool isRegOrInlineImmWithFP32InputMods()
const {
304 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
307 bool isRegOrInlineImmWithFP64InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
311 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
313 bool isVRegWithFP32InputMods()
const {
314 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
317 bool isVRegWithFP64InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
321 bool isPackedFP16InputMods()
const {
325 bool isPackedVGPRFP32InputMods()
const {
329 bool isVReg()
const {
330 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
331 isRegClass(AMDGPU::VReg_64RegClassID) ||
332 isRegClass(AMDGPU::VReg_96RegClassID) ||
333 isRegClass(AMDGPU::VReg_128RegClassID) ||
334 isRegClass(AMDGPU::VReg_160RegClassID) ||
335 isRegClass(AMDGPU::VReg_192RegClassID) ||
336 isRegClass(AMDGPU::VReg_256RegClassID) ||
337 isRegClass(AMDGPU::VReg_512RegClassID) ||
338 isRegClass(AMDGPU::VReg_1024RegClassID);
341 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
343 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
345 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
347 bool isAV_LdSt_32_Align2_RegOp()
const {
348 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
349 isRegClass(AMDGPU::AGPR_32RegClassID);
352 bool isVRegWithInputMods()
const;
353 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
356 bool isSDWAOperand(MVT type)
const;
357 bool isSDWAFP16Operand()
const;
358 bool isSDWAFP32Operand()
const;
359 bool isSDWAInt16Operand()
const;
360 bool isSDWAInt32Operand()
const;
362 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
364 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
366 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
368 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
370 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
371 bool isDim()
const {
return isImmTy(ImmTyDim); }
372 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
373 bool isOff()
const {
return isImmTy(ImmTyOff); }
374 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
375 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
376 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
377 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
378 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
379 bool isFlatOffset()
const {
380 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
382 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
383 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
384 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
385 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
386 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
387 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
388 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
389 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
390 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
391 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
392 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
393 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
394 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
395 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
396 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
397 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
398 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
399 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
400 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
401 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
402 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
403 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
404 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
405 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
406 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
407 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
408 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
409 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
410 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
411 bool isDone()
const {
return isImmTy(ImmTyDone); }
412 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
414 bool isRegOrImm()
const {
return isReg() || isImm(); }
416 bool isRegClass(
unsigned RCID)
const;
420 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
421 return isRegOrInline(RCID, type) && !hasModifiers();
424 bool isSCSrcB16()
const {
425 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
428 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
430 bool isSCSrc_b32()
const {
431 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
434 bool isSCSrc_b64()
const {
435 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
438 bool isBoolReg()
const;
440 bool isSCSrcF16()
const {
441 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
444 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
446 bool isSCSrcF32()
const {
447 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
450 bool isSCSrcF64()
const {
451 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
454 bool isSSrc_b32()
const {
455 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
458 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
460 bool isSSrcV2B16()
const {
465 bool isSSrc_b64()
const {
468 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
469 (((
const MCTargetAsmParser *)AsmParser)
470 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
474 bool isSSrc_f32()
const {
475 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
478 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
480 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
482 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
484 bool isSSrcV2F16()
const {
489 bool isSSrcV2FP32()
const {
494 bool isSCSrcV2FP32()
const {
499 bool isSSrcV2INT32()
const {
504 bool isSCSrcV2INT32()
const {
506 return isSCSrc_b32();
509 bool isSSrcOrLds_b32()
const {
510 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
511 isLiteralImm(MVT::i32) || isExpr();
514 bool isVCSrc_b32()
const {
515 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
518 bool isVCSrc_b32_Lo256()
const {
519 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
522 bool isVCSrc_b64_Lo256()
const {
523 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
526 bool isVCSrc_b64()
const {
527 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
530 bool isVCSrcT_b16()
const {
531 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
534 bool isVCSrcTB16_Lo128()
const {
535 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
538 bool isVCSrcFake16B16_Lo128()
const {
539 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
542 bool isVCSrc_b16()
const {
543 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
546 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
548 bool isVCSrc_f32()
const {
549 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
552 bool isVCSrc_f64()
const {
553 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
556 bool isVCSrcTBF16()
const {
557 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
560 bool isVCSrcT_f16()
const {
561 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
564 bool isVCSrcT_bf16()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
568 bool isVCSrcTBF16_Lo128()
const {
569 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
572 bool isVCSrcTF16_Lo128()
const {
573 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
576 bool isVCSrcFake16BF16_Lo128()
const {
577 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
580 bool isVCSrcFake16F16_Lo128()
const {
581 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
584 bool isVCSrc_bf16()
const {
585 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
588 bool isVCSrc_f16()
const {
589 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
592 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
594 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
596 bool isVSrc_b32()
const {
597 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
600 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
602 bool isVSrc_v2b64()
const {
603 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
604 isLiteralImm(MVT::i64);
607 bool isVSrc_v2f64()
const {
608 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
609 isLiteralImm(MVT::f64);
612 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
614 bool isVSrcT_b16_Lo128()
const {
615 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
618 bool isVSrcFake16_b16_Lo128()
const {
619 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
622 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
624 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
626 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
628 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
630 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
632 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
634 bool isVSrc_f32()
const {
635 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
638 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
640 bool isVSrcT_bf16()
const {
641 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
644 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
646 bool isVSrcT_bf16_Lo128()
const {
647 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
650 bool isVSrcT_f16_Lo128()
const {
651 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
654 bool isVSrcFake16_bf16_Lo128()
const {
655 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
658 bool isVSrcFake16_f16_Lo128()
const {
659 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
662 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
664 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
666 bool isVSrc_v2bf16()
const {
667 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
670 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
672 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
674 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
676 bool isVISrcB32()
const {
677 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
680 bool isVISrcB16()
const {
681 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
684 bool isVISrcV2B16()
const {
return isVISrcB16(); }
686 bool isVISrcF32()
const {
687 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
690 bool isVISrcF16()
const {
691 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
694 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
696 bool isVISrc_64_bf16()
const {
697 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
700 bool isVISrc_64_f16()
const {
701 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
704 bool isVISrc_64_b32()
const {
705 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
708 bool isVISrc_64B64()
const {
709 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
712 bool isVISrc_64_f64()
const {
713 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
716 bool isVISrc_64V2FP32()
const {
717 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
720 bool isVISrc_64V2INT32()
const {
721 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
724 bool isVISrc_256_b32()
const {
725 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
728 bool isVISrc_256_f32()
const {
729 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
732 bool isVISrc_256B64()
const {
733 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
736 bool isVISrc_256_f64()
const {
737 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
740 bool isVISrc_512_f64()
const {
741 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
744 bool isVISrc_128B16()
const {
745 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
748 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
750 bool isVISrc_128_b32()
const {
751 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
754 bool isVISrc_128_f32()
const {
755 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
758 bool isVISrc_256V2FP32()
const {
759 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
762 bool isVISrc_256V2INT32()
const {
763 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
766 bool isVISrc_512_b32()
const {
767 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
770 bool isVISrc_512B16()
const {
771 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
774 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
776 bool isVISrc_512_f32()
const {
777 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
780 bool isVISrc_512F16()
const {
781 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
784 bool isVISrc_512V2F16()
const {
785 return isVISrc_512F16() || isVISrc_512_b32();
788 bool isVISrc_1024_b32()
const {
789 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
792 bool isVISrc_1024B16()
const {
793 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
796 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
798 bool isVISrc_1024_f32()
const {
799 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
802 bool isVISrc_1024F16()
const {
803 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
806 bool isVISrc_1024V2F16()
const {
807 return isVISrc_1024F16() || isVISrc_1024_b32();
810 bool isAISrcB32()
const {
811 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
814 bool isAISrcB16()
const {
815 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
818 bool isAISrcV2B16()
const {
return isAISrcB16(); }
820 bool isAISrcF32()
const {
821 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
824 bool isAISrcF16()
const {
825 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
828 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
830 bool isAISrc_64B64()
const {
831 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
834 bool isAISrc_64_f64()
const {
835 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
838 bool isAISrc_128_b32()
const {
839 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
842 bool isAISrc_128B16()
const {
843 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
846 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
848 bool isAISrc_128_f32()
const {
849 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
852 bool isAISrc_128F16()
const {
853 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
856 bool isAISrc_128V2F16()
const {
857 return isAISrc_128F16() || isAISrc_128_b32();
860 bool isVISrc_128_bf16()
const {
861 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
864 bool isVISrc_128_f16()
const {
865 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
868 bool isVISrc_128V2F16()
const {
869 return isVISrc_128_f16() || isVISrc_128_b32();
872 bool isAISrc_256B64()
const {
873 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
876 bool isAISrc_256_f64()
const {
877 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
880 bool isAISrc_512_b32()
const {
881 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
884 bool isAISrc_512B16()
const {
885 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
888 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
890 bool isAISrc_512_f32()
const {
891 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
894 bool isAISrc_512F16()
const {
895 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
898 bool isAISrc_512V2F16()
const {
899 return isAISrc_512F16() || isAISrc_512_b32();
902 bool isAISrc_1024_b32()
const {
903 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
906 bool isAISrc_1024B16()
const {
907 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
910 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
912 bool isAISrc_1024_f32()
const {
913 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
916 bool isAISrc_1024F16()
const {
917 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
920 bool isAISrc_1024V2F16()
const {
921 return isAISrc_1024F16() || isAISrc_1024_b32();
924 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
926 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
928 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
930 bool isMem()
const override {
return false; }
932 bool isExpr()
const {
return Kind == Expression; }
934 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
936 bool isSWaitCnt()
const;
937 bool isDepCtr()
const;
938 bool isSDelayALU()
const;
939 bool isHwreg()
const;
940 bool isSendMsg()
const;
941 bool isWaitEvent()
const;
942 bool isSplitBarrier()
const;
943 bool isSwizzle()
const;
944 bool isSMRDOffset8()
const;
945 bool isSMEMOffset()
const;
946 bool isSMRDLiteralOffset()
const;
948 bool isDPPCtrl()
const;
950 bool isGPRIdxMode()
const;
951 bool isS16Imm()
const;
952 bool isU16Imm()
const;
953 bool isEndpgm()
const;
955 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
956 return [
this,
P]() {
return P(*
this); };
961 return StringRef(Tok.Data, Tok.Length);
969 void setImm(int64_t Val) {
974 ImmTy getImmTy()
const {
979 MCRegister
getReg()
const override {
984 SMLoc getStartLoc()
const override {
return StartLoc; }
986 SMLoc getEndLoc()
const override {
return EndLoc; }
988 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
990 int getMCOpIdx()
const {
return MCOpIdx; }
992 Modifiers getModifiers()
const {
993 assert(isRegKind() || isImmTy(ImmTyNone));
994 return isRegKind() ?
Reg.Mods :
Imm.Mods;
997 void setModifiers(Modifiers Mods) {
998 assert(isRegKind() || isImmTy(ImmTyNone));
1005 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1007 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1009 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1011 bool isForcedLit()
const {
1012 return isImmLiteral() && getModifiers().isForcedLit();
1015 bool isForcedLit64()
const {
1016 return isImmLiteral() && getModifiers().isForcedLit64();
1021 void addImmOperands(MCInst &Inst,
unsigned N,
1022 bool ApplyModifiers =
true)
const;
1024 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1025 bool ApplyModifiers)
const;
1027 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1029 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1031 addRegOperands(Inst,
N);
1033 addImmOperands(Inst,
N);
1036 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1037 Modifiers Mods = getModifiers();
1040 addRegOperands(Inst,
N);
1042 addImmOperands(Inst,
N,
false);
1046 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1047 assert(!hasIntModifiers());
1048 addRegOrImmWithInputModsOperands(Inst,
N);
1051 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1052 assert(!hasFPModifiers());
1053 addRegOrImmWithInputModsOperands(Inst,
N);
1056 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1057 Modifiers Mods = getModifiers();
1060 addRegOperands(Inst,
N);
1063 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1064 assert(!hasIntModifiers());
1065 addRegWithInputModsOperands(Inst,
N);
1068 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1069 assert(!hasFPModifiers());
1070 addRegWithInputModsOperands(Inst,
N);
1073 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1076 case ImmTyNone: OS <<
"None";
break;
1077 case ImmTyGDS: OS <<
"GDS";
break;
1078 case ImmTyLDS: OS <<
"LDS";
break;
1079 case ImmTyOffen: OS <<
"Offen";
break;
1080 case ImmTyIdxen: OS <<
"Idxen";
break;
1081 case ImmTyAddr64: OS <<
"Addr64";
break;
1082 case ImmTyOffset: OS <<
"Offset";
break;
1083 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1084 case ImmTyOffset0: OS <<
"Offset0";
break;
1085 case ImmTyOffset1: OS <<
"Offset1";
break;
1086 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1087 case ImmTyCPol: OS <<
"CPol";
break;
1088 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1089 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1091 case ImmTyTFE: OS <<
"TFE";
break;
1092 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1093 case ImmTyD16: OS <<
"D16";
break;
1094 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1095 case ImmTyClamp: OS <<
"Clamp";
break;
1096 case ImmTyOModSI: OS <<
"OModSI";
break;
1097 case ImmTyDPP8: OS <<
"DPP8";
break;
1098 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1099 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1100 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1101 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1102 case ImmTyDppFI: OS <<
"DppFI";
break;
1103 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1104 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1105 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1106 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1107 case ImmTyDMask: OS <<
"DMask";
break;
1108 case ImmTyDim: OS <<
"Dim";
break;
1109 case ImmTyUNorm: OS <<
"UNorm";
break;
1110 case ImmTyDA: OS <<
"DA";
break;
1111 case ImmTyR128A16: OS <<
"R128A16";
break;
1112 case ImmTyA16: OS <<
"A16";
break;
1113 case ImmTyLWE: OS <<
"LWE";
break;
1114 case ImmTyOff: OS <<
"Off";
break;
1115 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1116 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1117 case ImmTyExpVM: OS <<
"ExpVM";
break;
1118 case ImmTyDone: OS <<
"Done";
break;
1119 case ImmTyRowEn: OS <<
"RowEn";
break;
1120 case ImmTyHwreg: OS <<
"Hwreg";
break;
1121 case ImmTySendMsg: OS <<
"SendMsg";
break;
1122 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1123 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1124 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1125 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1126 case ImmTyOpSel: OS <<
"OpSel";
break;
1127 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1128 case ImmTyNegLo: OS <<
"NegLo";
break;
1129 case ImmTyNegHi: OS <<
"NegHi";
break;
1130 case ImmTySwizzle: OS <<
"Swizzle";
break;
1131 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1132 case ImmTyHigh: OS <<
"High";
break;
1133 case ImmTyBLGP: OS <<
"BLGP";
break;
1134 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1135 case ImmTyABID: OS <<
"ABID";
break;
1136 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1137 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1138 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1139 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1140 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1141 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1142 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1143 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1144 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1145 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1146 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1147 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1148 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1149 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1150 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1151 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1156 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1160 <<
" mods: " <<
Reg.Mods <<
'>';
1164 if (getImmTy() != ImmTyNone) {
1166 printImmTy(OS, getImmTy());
1168 OS <<
" mods: " <<
Imm.Mods <<
'>';
1181 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1182 int64_t Val, SMLoc Loc,
1183 ImmTy
Type = ImmTyNone,
1184 bool IsFPImm =
false) {
1185 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1187 Op->Imm.IsFPImm = IsFPImm;
1189 Op->Imm.Mods = Modifiers();
1195 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1196 StringRef Str, SMLoc Loc,
1197 bool HasExplicitEncodingSize =
true) {
1198 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1199 Res->Tok.Data = Str.data();
1200 Res->Tok.Length = Str.size();
1201 Res->StartLoc = Loc;
1206 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1207 MCRegister
Reg, SMLoc S, SMLoc
E) {
1208 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1209 Op->Reg.RegNo =
Reg;
1210 Op->Reg.Mods = Modifiers();
1216 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1217 const class MCExpr *Expr, SMLoc S) {
1218 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1227 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1236#define GET_REGISTER_MATCHER
1237#include "AMDGPUGenAsmMatcher.inc"
1238#undef GET_REGISTER_MATCHER
1239#undef GET_SUBTARGET_FEATURE_NAME
1244class KernelScopeInfo {
1245 int SgprIndexUnusedMin = -1;
1246 int VgprIndexUnusedMin = -1;
1247 int AgprIndexUnusedMin = -1;
1251 void usesSgprAt(
int i) {
1252 if (i >= SgprIndexUnusedMin) {
1253 SgprIndexUnusedMin = ++i;
1256 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1262 void usesVgprAt(
int i) {
1263 if (i >= VgprIndexUnusedMin) {
1264 VgprIndexUnusedMin = ++i;
1267 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1269 VgprIndexUnusedMin);
1275 void usesAgprAt(
int i) {
1280 if (i >= AgprIndexUnusedMin) {
1281 AgprIndexUnusedMin = ++i;
1284 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1289 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1291 VgprIndexUnusedMin);
1298 KernelScopeInfo() =
default;
1302 MSTI = Ctx->getSubtargetInfo();
1304 usesSgprAt(SgprIndexUnusedMin = -1);
1305 usesVgprAt(VgprIndexUnusedMin = -1);
1307 usesAgprAt(AgprIndexUnusedMin = -1);
1311 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1312 unsigned RegWidth) {
1315 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1318 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1321 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1330 MCAsmParser &Parser;
1332 unsigned ForcedEncodingSize = 0;
1333 bool ForcedDPP =
false;
1334 bool ForcedSDWA =
false;
1335 KernelScopeInfo KernelScope;
1336 const unsigned HwMode;
1338 const AMDGPU::IsaVersion ISA;
1343#define GET_ASSEMBLER_HEADER
1344#include "AMDGPUGenAsmMatcher.inc"
1349 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1351 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1355 std::optional<AMDGPU::InfoSectionData> InfoData;
1362 bool TargetDirectiveEmitted =
false;
1371 SmallVector<unsigned> OpcodeStream;
1373 OpcodeStreamSymbols;
1374 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1377 void checkKernelPrologues();
1380 void createConstantSymbol(StringRef Id, int64_t Val);
1382 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1383 bool OutOfRangeError(SMRange
Range);
1399 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1400 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1401 std::optional<bool> EnableWavefrontSize32,
1402 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1403 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1404 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1405 bool ParseDirectiveAMDGCNTarget();
1406 bool ParseDirectiveAMDHSACodeObjectVersion();
1407 bool ParseDirectiveAMDHSAKernel();
1408 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1409 bool ParseDirectiveAMDKernelCodeT();
1411 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1412 bool ParseDirectiveAMDGPUHsaKernel();
1414 bool ParseDirectiveISAVersion();
1415 bool ParseDirectiveHSAMetadata();
1416 bool ParseDirectivePALMetadataBegin();
1417 bool ParseDirectivePALMetadata();
1418 bool ParseDirectiveAMDGPULDS();
1419 bool ParseDirectiveAMDGPUInfo();
1423 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1424 const char *AssemblerDirectiveEnd,
1425 std::string &CollectString);
1427 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1428 RegisterKind RegKind, MCRegister Reg1,
1429 RegisterKind RegKind1, SMLoc Loc);
1430 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1431 unsigned &RegNum,
unsigned &RegWidth,
1432 bool RestoreOnFailure =
false);
1433 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1434 unsigned &RegNum,
unsigned &RegWidth,
1435 SmallVectorImpl<AsmToken> &Tokens);
1436 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1438 SmallVectorImpl<AsmToken> &Tokens);
1439 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1441 SmallVectorImpl<AsmToken> &Tokens);
1442 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1444 SmallVectorImpl<AsmToken> &Tokens);
1445 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1446 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1447 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1450 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1451 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1452 void initializeGprCountSymbol(RegisterKind RegKind);
1453 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1459 OperandMode_Default,
1463 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1465 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1466 const MCInstrInfo &MII)
1467 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1468 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1473 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1475 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1476 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1477 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1478 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1480 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1481 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1482 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1484 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1485 initializeGprCountSymbol(IS_VGPR);
1486 initializeGprCountSymbol(IS_SGPR);
1491 createConstantSymbol(Symbol, Code);
1493 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1494 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1495 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1543 bool hasBVHRayTracingInsts()
const {
1544 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1549 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1551 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1553 bool hasInv2PiInlineImm()
const {
1554 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1557 bool has64BitLiterals()
const {
1558 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1561 bool hasFlatOffsets()
const {
1562 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1565 bool hasTrue16Insts()
const {
1566 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1570 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1573 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1575 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1577 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1579 bool hasPartialNSAEncoding()
const {
1580 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1583 bool hasGloballyAddressableScratch()
const {
1584 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1597 AMDGPUTargetStreamer &getTargetStreamer() {
1598 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1599 return static_cast<AMDGPUTargetStreamer &
>(TS);
1605 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1608 const MCRegisterInfo *getMRI()
const {
1612 const MCInstrInfo *getMII()
const {
return &MII; }
1616 const FeatureBitset &getFeatureBits()
const {
1617 return getSTI().getFeatureBits();
1620 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1621 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1622 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1624 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1625 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1626 bool isForcedDPP()
const {
return ForcedDPP; }
1627 bool isForcedSDWA()
const {
return ForcedSDWA; }
1628 ArrayRef<unsigned> getMatchedVariants()
const;
1629 StringRef getMatchedVariantName()
const;
1631 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1632 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1633 bool RestoreOnFailure);
1634 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1635 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1636 SMLoc &EndLoc)
override;
1637 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1638 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1639 unsigned Kind)
override;
1640 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1643 bool MatchingInlineAsm)
override;
1644 bool ParseDirective(AsmToken DirectiveID)
override;
1645 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1646 void onEndOfFile()
override;
1648 OperandMode
Mode = OperandMode_Default);
1649 StringRef parseMnemonicSuffix(StringRef Name);
1650 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1656 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1660 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1661 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1663 ParseStatus parseOperandArrayWithPrefix(
1665 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1666 bool (*ConvertResult)(int64_t &) =
nullptr);
1670 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1671 bool IgnoreNegative =
false);
1672 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1676 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1680 ArrayRef<const char *> Ids,
1684 ArrayRef<const char *> Ids,
1685 AMDGPUOperand::ImmTy
Type);
1688 bool isOperandModifier(
const AsmToken &Token,
1689 const AsmToken &NextToken)
const;
1690 bool isRegOrOperandModifier(
const AsmToken &Token,
1691 const AsmToken &NextToken)
const;
1692 bool isNamedOperandModifier(
const AsmToken &Token,
1693 const AsmToken &NextToken)
const;
1694 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1695 const AsmToken &NextToken)
const;
1696 bool parseSP3NegModifier();
1703 bool AllowImm =
true);
1705 bool AllowImm =
true);
1710 AMDGPUOperand::ImmTy ImmTy);
1715 AMDGPUOperand::ImmTy
Type);
1719 AMDGPUOperand::ImmTy
Type);
1723 AMDGPUOperand::ImmTy
Type);
1727 ParseStatus parseDfmtNfmt(int64_t &
Format);
1728 ParseStatus parseUfmt(int64_t &
Format);
1729 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1731 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1734 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1735 ParseStatus parseNumericFormat(int64_t &
Format);
1739 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1740 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1745 bool parseCnt(int64_t &IntVal);
1748 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1749 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1752 bool parseDelay(int64_t &Delay);
1758 struct OperandInfoTy {
1761 bool IsSymbolic =
false;
1762 bool IsDefined =
false;
1764 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1767 struct StructuredOpField : OperandInfoTy {
1771 bool IsDefined =
false;
1773 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1774 unsigned Width, int64_t
Default)
1775 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1776 virtual ~StructuredOpField() =
default;
1778 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1779 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1783 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1785 return Error(Parser,
"not supported on this GPU");
1787 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1795 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1796 OperandInfoTy &Stream);
1797 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1798 const OperandInfoTy &Stream);
1800 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1801 OperandInfoTy &Width);
1806 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1813 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1815 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1819 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1825 bool validateConstantBusLimitations(
const MCInst &Inst,
1827 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1830 bool tryVOPD(
const MCInst &Inst);
1831 bool tryVOPD3(
const MCInst &Inst);
1832 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1834 bool validateIntClampSupported(
const MCInst &Inst);
1835 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1836 bool validateMIMGGatherDMask(
const MCInst &Inst);
1838 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1839 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGD16(
const MCInst &Inst);
1842 bool validateTensorR128(
const MCInst &Inst);
1843 bool validateMIMGMSAA(
const MCInst &Inst);
1844 bool validateOpSel(
const MCInst &Inst);
1845 bool validateTrue16OpSel(
const MCInst &Inst);
1846 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1848 bool validateVccOperand(MCRegister
Reg)
const;
1853 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1854 bool validateVGPRAlign(
const MCInst &Inst)
const;
1858 bool validateDivScale(
const MCInst &Inst);
1863 const unsigned CPol);
1868 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1869 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1870 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1871 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1874 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1875 ArrayRef<unsigned> Variants);
1876 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1878 bool isId(
const StringRef Id)
const;
1879 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1881 StringRef getId()
const;
1882 bool trySkipId(
const StringRef Id);
1883 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1887 bool parseString(StringRef &Val,
1888 const StringRef ErrMsg =
"expected a string");
1889 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1895 StringRef getTokenStr()
const;
1896 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1898 SMLoc getLoc()
const;
1902 void onBeginOfFile()
override;
1906 void emitTargetDirective();
1907 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1919 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1920 const unsigned MaxVal,
const Twine &ErrMsg,
1922 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1923 const unsigned MinVal,
const unsigned MaxVal,
1924 const StringRef ErrMsg);
1926 bool parseSwizzleOffset(int64_t &
Imm);
1927 bool parseSwizzleMacro(int64_t &
Imm);
1928 bool parseSwizzleQuadPerm(int64_t &
Imm);
1929 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1930 bool parseSwizzleBroadcast(int64_t &
Imm);
1931 bool parseSwizzleSwap(int64_t &
Imm);
1932 bool parseSwizzleReverse(int64_t &
Imm);
1933 bool parseSwizzleFFT(int64_t &
Imm);
1934 bool parseSwizzleRotate(int64_t &
Imm);
1937 int64_t parseGPRIdxMacro();
1940 cvtMubufImpl(Inst,
Operands,
false);
1943 cvtMubufImpl(Inst,
Operands,
true);
1949 OptionalImmIndexMap &OptionalIdx);
1958 OptionalImmIndexMap &OptionalIdx);
1960 OptionalImmIndexMap &OptionalIdx);
1964 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1966 bool parseDimId(
unsigned &Encoding);
1968 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1972 int64_t parseDPPCtrlSel(StringRef Ctrl);
1973 int64_t parseDPPCtrlPerm();
1979 bool IsDPP8 =
false);
1985 AMDGPUOperand::ImmTy
Type);
1993 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1996 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1997 bool SkipSrcVcc =
false);
2106bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2116 if (!isImmTy(ImmTyNone)) {
2121 if (getModifiers().
Lit != LitModifier::None)
2131 if (type == MVT::f64 || type == MVT::i64) {
2133 AsmParser->hasInv2PiInlineImm());
2136 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2155 APFloat::rmNearestTiesToEven, &Lost);
2162 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2164 AsmParser->hasInv2PiInlineImm());
2169 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2170 AsmParser->hasInv2PiInlineImm());
2174 if (type == MVT::f64 || type == MVT::i64) {
2176 AsmParser->hasInv2PiInlineImm());
2185 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2186 AsmParser->hasInv2PiInlineImm());
2190 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2191 AsmParser->hasInv2PiInlineImm());
2194bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2196 if (!isImmTy(ImmTyNone)) {
2201 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2206 if (type == MVT::f64 && hasFPModifiers()) {
2226 if (type == MVT::f64) {
2231 if (type == MVT::i64) {
2244 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2245 : (type == MVT::v2i16) ? MVT::f32
2246 : (type == MVT::v2f32) ? MVT::f32
2249 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2253bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2254 return isRegKind() &&
2255 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2258bool AMDGPUOperand::isVRegWithInputMods()
const {
2259 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2261 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2262 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2265template <
bool IsFake16>
2266bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2267 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2268 : AMDGPU::VGPR_16_Lo128RegClassID);
2271template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2272 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2273 : AMDGPU::VGPR_16RegClassID);
2276bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2277 if (AsmParser->isVI())
2279 if (AsmParser->isGFX9Plus())
2280 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2284bool AMDGPUOperand::isSDWAFP16Operand()
const {
2285 return isSDWAOperand(MVT::f16);
2288bool AMDGPUOperand::isSDWAFP32Operand()
const {
2289 return isSDWAOperand(MVT::f32);
2292bool AMDGPUOperand::isSDWAInt16Operand()
const {
2293 return isSDWAOperand(MVT::i16);
2296bool AMDGPUOperand::isSDWAInt32Operand()
const {
2297 return isSDWAOperand(MVT::i32);
2300bool AMDGPUOperand::isBoolReg()
const {
2301 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2302 (AsmParser->isWave32() && isSCSrc_b32()));
2306 unsigned Size)
const {
2307 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2322void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2323 bool ApplyModifiers)
const {
2333 addLiteralImmOperand(Inst,
Imm.Val,
2334 ApplyModifiers & isImmTy(ImmTyNone) &&
2335 Imm.Mods.hasFPModifiers());
2337 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2342void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2343 bool ApplyModifiers)
const {
2344 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2349 if (ApplyModifiers) {
2351 const unsigned Size =
2353 Val = applyInputFPModifiers(Val,
Size);
2357 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2359 bool CanUse64BitLiterals =
2362 MCContext &Ctx = AsmParser->getContext();
2373 if (
Lit == LitModifier::None &&
2375 AsmParser->hasInv2PiInlineImm())) {
2383 bool HasMandatoryLiteral =
2386 if (
Literal.getLoBits(32) != 0 &&
2387 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2388 !HasMandatoryLiteral) {
2389 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2391 "Can't encode literal as exact 64-bit floating-point operand. "
2392 "Low 32-bits will be set to zero");
2393 Val &= 0xffffffff00000000u;
2399 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2405 Lit = LitModifier::Lit64;
2406 }
else if (
Lit == LitModifier::Lit) {
2420 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2422 Lit = LitModifier::Lit64;
2429 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2430 Literal == 0x3fc45f306725feed) {
2469 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2476 if (
Lit != LitModifier::None) {
2507 if (
Lit == LitModifier::None &&
2517 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2525 if (
Lit == LitModifier::None &&
2533 if (!AsmParser->has64BitLiterals()) {
2534 Val =
static_cast<uint64_t>(Val) << 32;
2541 if (
Lit == LitModifier::Lit ||
2543 Val =
static_cast<uint64_t>(Val) << 32;
2547 if (
Lit == LitModifier::Lit)
2573 if (
Lit != LitModifier::None) {
2581void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2587bool AMDGPUOperand::isInlineValue()
const {
2595void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2606 if (Is == IS_VGPR) {
2611 return AMDGPU::VGPR_32RegClassID;
2613 return AMDGPU::VReg_64RegClassID;
2615 return AMDGPU::VReg_96RegClassID;
2617 return AMDGPU::VReg_128RegClassID;
2619 return AMDGPU::VReg_160RegClassID;
2621 return AMDGPU::VReg_192RegClassID;
2623 return AMDGPU::VReg_224RegClassID;
2625 return AMDGPU::VReg_256RegClassID;
2627 return AMDGPU::VReg_288RegClassID;
2629 return AMDGPU::VReg_320RegClassID;
2631 return AMDGPU::VReg_352RegClassID;
2633 return AMDGPU::VReg_384RegClassID;
2635 return AMDGPU::VReg_512RegClassID;
2637 return AMDGPU::VReg_1024RegClassID;
2639 }
else if (Is == IS_TTMP) {
2644 return AMDGPU::TTMP_32RegClassID;
2646 return AMDGPU::TTMP_64RegClassID;
2648 return AMDGPU::TTMP_128RegClassID;
2650 return AMDGPU::TTMP_256RegClassID;
2652 return AMDGPU::TTMP_512RegClassID;
2654 }
else if (Is == IS_SGPR) {
2659 return AMDGPU::SGPR_32RegClassID;
2661 return AMDGPU::SGPR_64RegClassID;
2663 return AMDGPU::SGPR_96RegClassID;
2665 return AMDGPU::SGPR_128RegClassID;
2667 return AMDGPU::SGPR_160RegClassID;
2669 return AMDGPU::SGPR_192RegClassID;
2671 return AMDGPU::SGPR_224RegClassID;
2673 return AMDGPU::SGPR_256RegClassID;
2675 return AMDGPU::SGPR_288RegClassID;
2677 return AMDGPU::SGPR_320RegClassID;
2679 return AMDGPU::SGPR_352RegClassID;
2681 return AMDGPU::SGPR_384RegClassID;
2683 return AMDGPU::SGPR_512RegClassID;
2685 }
else if (Is == IS_AGPR) {
2690 return AMDGPU::AGPR_32RegClassID;
2692 return AMDGPU::AReg_64RegClassID;
2694 return AMDGPU::AReg_96RegClassID;
2696 return AMDGPU::AReg_128RegClassID;
2698 return AMDGPU::AReg_160RegClassID;
2700 return AMDGPU::AReg_192RegClassID;
2702 return AMDGPU::AReg_224RegClassID;
2704 return AMDGPU::AReg_256RegClassID;
2706 return AMDGPU::AReg_288RegClassID;
2708 return AMDGPU::AReg_320RegClassID;
2710 return AMDGPU::AReg_352RegClassID;
2712 return AMDGPU::AReg_384RegClassID;
2714 return AMDGPU::AReg_512RegClassID;
2716 return AMDGPU::AReg_1024RegClassID;
2724 .
Case(
"exec", AMDGPU::EXEC)
2725 .
Case(
"vcc", AMDGPU::VCC)
2726 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2727 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2728 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2730 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2732 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2734 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2736 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2737 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2738 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2740 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2742 .
Case(
"m0", AMDGPU::M0)
2743 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2745 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2747 .
Case(
"scc", AMDGPU::SRC_SCC)
2748 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2749 .
Case(
"tba", AMDGPU::TBA)
2750 .
Case(
"tma", AMDGPU::TMA)
2751 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2752 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2753 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2754 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2755 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2756 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2757 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2758 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2759 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2760 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2761 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2762 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2763 .
Case(
"pc", AMDGPU::PC_REG)
2764 .
Case(
"null", AMDGPU::SGPR_NULL)
2768bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2769 SMLoc &EndLoc,
bool RestoreOnFailure) {
2770 auto R = parseRegister();
2774 RegNo =
R->getReg();
2775 StartLoc =
R->getStartLoc();
2776 EndLoc =
R->getEndLoc();
2780bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2782 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2785ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2787 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2788 bool PendingErrors = getParser().hasPendingError();
2789 getParser().clearPendingErrors();
2797bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2798 RegisterKind RegKind,
2800 RegisterKind RegKind1, SMLoc Loc) {
2802 if (RegKind == IS_SGPR) {
2803 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2804 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2805 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2811 if (RegKind != RegKind1) {
2812 Error(Loc,
"registers in a list must be of the same kind");
2818 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2823 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2824 Reg = AMDGPU::FLAT_SCR;
2828 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2829 Reg = AMDGPU::XNACK_MASK;
2833 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2838 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2843 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2848 Error(Loc,
"register does not fit in the list");
2854 if (Reg1 !=
Reg + RegWidth / 32) {
2855 Error(Loc,
"registers in a list must have consecutive indices");
2871 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2872 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2876 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2882 if (Str.starts_with(
Reg.Name))
2888 return !Str.getAsInteger(10, Num);
2891bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2892 const AsmToken &NextToken)
const {
2907 StringRef RegSuffix = Str.substr(
RegName.size());
2908 if (!RegSuffix.
empty()) {
2925bool AMDGPUAsmParser::isRegister() {
2926 return isRegister(
getToken(), peekToken());
2929MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2930 unsigned SubReg,
unsigned RegWidth,
2934 unsigned AlignSize = 1;
2935 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2941 if (RegNum % AlignSize != 0) {
2942 Error(Loc,
"invalid register alignment");
2943 return MCRegister();
2946 unsigned RegIdx = RegNum / AlignSize;
2949 Error(Loc,
"invalid or unsupported register size");
2950 return MCRegister();
2954 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2955 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2956 Error(Loc,
"register index is out of range");
2957 return AMDGPU::NoRegister;
2960 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2961 Error(Loc,
"register index is out of range");
2962 return MCRegister();
2978bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2980 int64_t RegLo, RegHi;
2984 SMLoc FirstIdxLoc = getLoc();
2991 SecondIdxLoc = getLoc();
3002 Error(FirstIdxLoc,
"invalid register index");
3007 Error(SecondIdxLoc,
"invalid register index");
3011 if (RegLo > RegHi) {
3012 Error(FirstIdxLoc,
"first register index should not exceed second index");
3016 if (RegHi == RegLo) {
3017 StringRef RegSuffix = getTokenStr();
3018 if (RegSuffix ==
".l") {
3019 SubReg = AMDGPU::lo16;
3021 }
else if (RegSuffix ==
".h") {
3022 SubReg = AMDGPU::hi16;
3027 Num =
static_cast<unsigned>(RegLo);
3028 RegWidth = 32 * ((RegHi - RegLo) + 1);
3033MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3036 SmallVectorImpl<AsmToken> &Tokens) {
3042 RegKind = IS_SPECIAL;
3049MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3052 SmallVectorImpl<AsmToken> &Tokens) {
3054 StringRef
RegName = getTokenStr();
3055 auto Loc = getLoc();
3059 Error(Loc,
"invalid register name");
3060 return MCRegister();
3068 unsigned SubReg = NoSubRegister;
3069 bool IsRange =
false;
3070 if (!RegSuffix.
empty()) {
3072 SubReg = AMDGPU::lo16;
3074 SubReg = AMDGPU::hi16;
3078 Error(Loc,
"invalid register index");
3079 return MCRegister();
3085 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3086 return MCRegister();
3090 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3091 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3092 if (RegKind == IS_SGPR && IsRange
3093 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3094 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3095 Error(Loc,
"register index is out of range");
3096 return MCRegister();
3102MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3103 unsigned &RegNum,
unsigned &RegWidth,
3104 SmallVectorImpl<AsmToken> &Tokens) {
3106 auto ListLoc = getLoc();
3109 "expected a register or a list of registers")) {
3110 return MCRegister();
3115 auto Loc = getLoc();
3116 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3117 return MCRegister();
3118 if (RegWidth != 32) {
3119 Error(Loc,
"expected a single 32-bit register");
3120 return MCRegister();
3124 RegisterKind NextRegKind;
3126 unsigned NextRegNum, NextRegWidth;
3129 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3131 return MCRegister();
3133 if (NextRegWidth != 32) {
3134 Error(Loc,
"expected a single 32-bit register");
3135 return MCRegister();
3137 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3139 return MCRegister();
3143 "expected a comma or a closing square bracket")) {
3144 return MCRegister();
3148 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3153bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3154 MCRegister &
Reg,
unsigned &RegNum,
3156 SmallVectorImpl<AsmToken> &Tokens) {
3157 auto Loc = getLoc();
3161 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3163 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3165 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3170 assert(Parser.hasPendingError());
3174 if (!subtargetHasRegister(*
TRI,
Reg)) {
3175 if (
Reg == AMDGPU::SGPR_NULL) {
3176 Error(Loc,
"'null' operand is not supported on this GPU");
3179 " register not available on this GPU");
3187bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3188 MCRegister &
Reg,
unsigned &RegNum,
3190 bool RestoreOnFailure ) {
3194 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3195 if (RestoreOnFailure) {
3196 while (!Tokens.
empty()) {
3205std::optional<StringRef>
3206AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3209 return StringRef(
".amdgcn.next_free_vgpr");
3211 return StringRef(
".amdgcn.next_free_sgpr");
3213 return std::nullopt;
3217void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3218 auto SymbolName = getGprCountSymbolName(RegKind);
3219 assert(SymbolName &&
"initializing invalid register kind");
3225bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3226 unsigned DwordRegIndex,
3227 unsigned RegWidth) {
3232 auto SymbolName = getGprCountSymbolName(RegKind);
3237 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3241 return !
Error(getLoc(),
3242 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3246 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3248 if (OldCount <= NewMax)
3254std::unique_ptr<AMDGPUOperand>
3255AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3257 SMLoc StartLoc = Tok.getLoc();
3258 SMLoc EndLoc = Tok.getEndLoc();
3259 RegisterKind RegKind;
3261 unsigned RegNum, RegWidth;
3263 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3267 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3270 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3271 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3278 if (isRegister() || isModifier())
3281 if (
Lit == LitModifier::None) {
3282 if (trySkipId(
"lit"))
3283 Lit = LitModifier::Lit;
3284 else if (trySkipId(
"lit64"))
3285 Lit = LitModifier::Lit64;
3287 if (
Lit != LitModifier::None) {
3290 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3299 const auto &NextTok = peekToken();
3302 bool Negate =
false;
3310 AMDGPUOperand::Modifiers Mods;
3318 StringRef Num = getTokenStr();
3321 APFloat RealVal(APFloat::IEEEdouble());
3322 auto roundMode = APFloat::rmNearestTiesToEven;
3323 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3326 RealVal.changeSign();
3329 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3330 S, AMDGPUOperand::ImmTyNone,
true));
3331 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3332 Op.setModifiers(Mods);
3341 if (HasSP3AbsModifier) {
3350 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3353 if (Parser.parseExpression(Expr))
3357 if (Expr->evaluateAsAbsolute(IntVal)) {
3359 return Error(S,
"literal value out of range");
3360 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3361 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3362 Op.setModifiers(Mods);
3364 if (
Lit != LitModifier::None)
3366 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3379 if (
auto R = parseRegister()) {
3389 ParseStatus Res = parseReg(
Operands);
3397bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3398 const AsmToken &NextToken)
const {
3401 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3406bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3407 const AsmToken &NextToken)
const {
3411bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3412 const AsmToken &NextToken)
const {
3413 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3416bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3417 const AsmToken &NextToken)
const {
3418 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3434bool AMDGPUAsmParser::isModifier() {
3437 AsmToken NextToken[2];
3438 peekTokens(NextToken);
3440 return isOperandModifier(Tok, NextToken[0]) ||
3442 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3443 isOpcodeModifierWithVal(Tok, NextToken[0]);
3468bool AMDGPUAsmParser::parseSP3NegModifier() {
3470 AsmToken NextToken[2];
3471 peekTokens(NextToken);
3474 (isRegister(NextToken[0], NextToken[1]) ||
3492 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3494 SP3Neg = parseSP3NegModifier();
3497 Neg = trySkipId(
"neg");
3499 return Error(Loc,
"expected register or immediate");
3503 Abs = trySkipId(
"abs");
3508 if (trySkipId(
"lit")) {
3509 Lit = LitModifier::Lit;
3512 }
else if (trySkipId(
"lit64")) {
3513 Lit = LitModifier::Lit64;
3516 if (!has64BitLiterals())
3517 return Error(Loc,
"lit64 is not supported on this GPU");
3523 return Error(Loc,
"expected register or immediate");
3532 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3536 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3537 Error(Loc,
"expected immediate with lit modifier");
3539 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3545 if (
Lit != LitModifier::None &&
3549 AMDGPUOperand::Modifiers Mods;
3550 Mods.Abs = Abs || SP3Abs;
3551 Mods.Neg = Neg || SP3Neg;
3554 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3555 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3557 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3558 Op.setModifiers(Mods);
3566 bool Sext = trySkipId(
"sext");
3567 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3582 AMDGPUOperand::Modifiers Mods;
3585 if (Mods.hasIntModifiers()) {
3586 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3588 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3589 Op.setModifiers(Mods);
3596 return parseRegOrImmWithFPInputMods(
Operands,
false);
3600 return parseRegOrImmWithIntInputMods(
Operands,
false);
3604 auto Loc = getLoc();
3605 if (trySkipId(
"off")) {
3607 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3614 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3623unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3628 return Match_InvalidOperand;
3630 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3631 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3634 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3636 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3637 return Match_InvalidOperand;
3645 if (tryAnotherVOPDEncoding(Inst))
3646 return Match_InvalidOperand;
3648 return Match_Success;
3652 static const unsigned Variants[] = {
3661ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3662 if (isForcedDPP() && isForcedVOP3()) {
3666 if (getForcedEncodingSize() == 32) {
3671 if (isForcedVOP3()) {
3676 if (isForcedSDWA()) {
3682 if (isForcedDPP()) {
3690StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3691 if (isForcedDPP() && isForcedVOP3())
3694 if (getForcedEncodingSize() == 32)
3710AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3714 case AMDGPU::FLAT_SCR:
3716 case AMDGPU::VCC_LO:
3717 case AMDGPU::VCC_HI:
3724 return MCRegister();
3731bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3732 unsigned OpIdx)
const {
3740 const MCOperand &MO = Inst.
getOperand(OpIdx);
3789unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3795 case AMDGPU::V_LSHLREV_B64_e64:
3796 case AMDGPU::V_LSHLREV_B64_gfx10:
3797 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3798 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3799 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3800 case AMDGPU::V_LSHRREV_B64_e64:
3801 case AMDGPU::V_LSHRREV_B64_gfx10:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3803 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3804 case AMDGPU::V_ASHRREV_I64_e64:
3805 case AMDGPU::V_ASHRREV_I64_gfx10:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3807 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3808 case AMDGPU::V_LSHL_B64_e64:
3809 case AMDGPU::V_LSHR_B64_e64:
3810 case AMDGPU::V_ASHR_I64_e64:
3823 bool AddMandatoryLiterals =
false) {
3826 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3830 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3832 return {getNamedOperandIdx(Opcode, OpName::src0X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3834 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3835 getNamedOperandIdx(Opcode, OpName::src0Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3837 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3842 return {getNamedOperandIdx(Opcode, OpName::src0),
3843 getNamedOperandIdx(Opcode, OpName::src1),
3844 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3847bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3848 const MCOperand &MO = Inst.
getOperand(OpIdx);
3850 return !isInlineConstant(Inst, OpIdx);
3857 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3868 const unsigned Opcode = Inst.
getOpcode();
3869 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3872 if (!LaneSelOp.
isReg())
3875 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3878bool AMDGPUAsmParser::validateConstantBusLimitations(
3880 const unsigned Opcode = Inst.
getOpcode();
3881 const MCInstrDesc &
Desc = MII.
get(Opcode);
3882 MCRegister LastSGPR;
3883 unsigned ConstantBusUseCount = 0;
3884 unsigned NumLiterals = 0;
3885 unsigned LiteralSize;
3901 SmallDenseSet<MCRegister> SGPRsUsed;
3902 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3904 SGPRsUsed.
insert(SGPRUsed);
3905 ++ConstantBusUseCount;
3910 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3912 for (
int OpIdx : OpIndices) {
3916 const MCOperand &MO = Inst.
getOperand(OpIdx);
3917 if (usesConstantBus(Inst, OpIdx)) {
3926 if (SGPRsUsed.
insert(LastSGPR).second) {
3927 ++ConstantBusUseCount;
3947 if (NumLiterals == 0) {
3950 }
else if (LiteralSize !=
Size) {
3956 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3958 "invalid operand (violates constant bus restrictions)");
3965std::optional<unsigned>
3966AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3968 const unsigned Opcode = Inst.
getOpcode();
3974 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3975 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3989 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3993 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3994 int I = getNamedOperandIdx(Opcode, OpName);
3998 int64_t
Imm =
Op.getImm();
4004 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4005 OpName::vsrc2Y, OpName::imm}) {
4006 int I = getNamedOperandIdx(Opcode, OpName);
4016 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4017 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4019 return InvalidCompOprIdx;
4022bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4029 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4030 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4031 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4033 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4037 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4038 if (!InvalidCompOprIdx.has_value())
4041 auto CompOprIdx = *InvalidCompOprIdx;
4044 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4045 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4048 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4049 if (CompOprIdx == VOPD::Component::DST) {
4051 Error(Loc,
"dst registers must be distinct");
4053 Error(Loc,
"one dst register must be even and the other odd");
4055 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4056 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4057 " operands must use different VGPR banks");
4065bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4067 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4068 if (!InvalidCompOprIdx.has_value())
4072 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4073 if (InvalidCompOprIdx.has_value()) {
4078 if (*InvalidCompOprIdx == VOPD::Component::DST)
4091bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4092 const unsigned Opcode = Inst.
getOpcode();
4107 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4108 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4109 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4110 int I = getNamedOperandIdx(Opcode, OpName);
4117 return !tryVOPD3(Inst);
4122bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4127 return tryVOPD(Inst);
4128 return tryVOPD3(Inst);
4131bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4136 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4144bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4152 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4153 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4154 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4159 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4160 hasBVHRayTracingInsts())
4163 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4164 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4169 bool IsPackedD16 =
false;
4172 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4173 IsPackedD16 = D16Idx >= 0;
4175 DataSize = (DataSize + 1) / 2;
4178 if ((VDataSize / 4) == DataSize + TFESize)
4183 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4185 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4187 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4191bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4200 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4202 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4203 AMDGPU::OpName RSrcOpName =
4205 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4206 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4207 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4211 assert(SrsrcIdx > VAddr0Idx);
4214 if (BaseOpcode->
BVH) {
4215 if (IsA16 == BaseOpcode->
A16)
4217 Error(IDLoc,
"image address size does not match a16");
4223 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4224 unsigned ActualAddrSize =
4225 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4227 unsigned ExpectedAddrSize =
4231 if (hasPartialNSAEncoding() &&
4233 int VAddrLastIdx = SrsrcIdx - 1;
4234 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4236 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4239 if (ExpectedAddrSize > 12)
4240 ExpectedAddrSize = 16;
4245 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4249 if (ActualAddrSize == ExpectedAddrSize)
4252 Error(IDLoc,
"image address size does not match dim and a16");
4256bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4263 if (!
Desc.mayLoad() || !
Desc.mayStore())
4266 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4273 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4276bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4283 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4291 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4294bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4308 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4309 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4316bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4323 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4326 if (!BaseOpcode->
MSAA)
4329 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4335 return DimInfo->
MSAA;
4340 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4342 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4352bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4360 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4363 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4371 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4375bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4380 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4383 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4386 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4394 "source operand must be either a VGPR or an inline constant");
4401bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4406 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4409 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4413 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4415 "inline constants are not allowed for this operand");
4422bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4430 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4431 if (BlgpIdx != -1) {
4432 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4433 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4443 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4445 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4450 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4452 "wrong register tuple size for blgp value " + Twine(BLGP));
4460 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4464 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4468 MCRegister Src2Reg = Src2.
getReg();
4470 if (Src2Reg == DstReg)
4475 .getSizeInBits() <= 128)
4478 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4480 "source 2 operand must not partially overlap with dst");
4487bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4491 case V_DIV_SCALE_F32_gfx6_gfx7:
4492 case V_DIV_SCALE_F32_vi:
4493 case V_DIV_SCALE_F32_gfx10:
4494 case V_DIV_SCALE_F64_gfx6_gfx7:
4495 case V_DIV_SCALE_F64_vi:
4496 case V_DIV_SCALE_F64_gfx10:
4503 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4504 AMDGPU::OpName::src2_modifiers}) {
4515bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4522 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4531bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4537 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4544 case AMDGPU::V_SUBREV_F32_e32:
4545 case AMDGPU::V_SUBREV_F32_e64:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4547 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4548 case AMDGPU::V_SUBREV_F32_e32_vi:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4550 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4551 case AMDGPU::V_SUBREV_F32_e64_vi:
4553 case AMDGPU::V_SUBREV_CO_U32_e32:
4554 case AMDGPU::V_SUBREV_CO_U32_e64:
4555 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4556 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4558 case AMDGPU::V_SUBBREV_U32_e32:
4559 case AMDGPU::V_SUBBREV_U32_e64:
4560 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4561 case AMDGPU::V_SUBBREV_U32_e32_vi:
4562 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4563 case AMDGPU::V_SUBBREV_U32_e64_vi:
4565 case AMDGPU::V_SUBREV_U32_e32:
4566 case AMDGPU::V_SUBREV_U32_e64:
4567 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4568 case AMDGPU::V_SUBREV_U32_e32_vi:
4569 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4570 case AMDGPU::V_SUBREV_U32_e64_vi:
4572 case AMDGPU::V_SUBREV_F16_e32:
4573 case AMDGPU::V_SUBREV_F16_e64:
4574 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4575 case AMDGPU::V_SUBREV_F16_e32_vi:
4576 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4577 case AMDGPU::V_SUBREV_F16_e64_vi:
4579 case AMDGPU::V_SUBREV_U16_e32:
4580 case AMDGPU::V_SUBREV_U16_e64:
4581 case AMDGPU::V_SUBREV_U16_e32_vi:
4582 case AMDGPU::V_SUBREV_U16_e64_vi:
4584 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4586 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4589 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4591 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4592 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4595 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4597 case AMDGPU::V_LSHRREV_B32_e32:
4598 case AMDGPU::V_LSHRREV_B32_e64:
4599 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4601 case AMDGPU::V_LSHRREV_B32_e32_vi:
4602 case AMDGPU::V_LSHRREV_B32_e64_vi:
4603 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4604 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4606 case AMDGPU::V_ASHRREV_I32_e32:
4607 case AMDGPU::V_ASHRREV_I32_e64:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4609 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4610 case AMDGPU::V_ASHRREV_I32_e32_vi:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4612 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4613 case AMDGPU::V_ASHRREV_I32_e64_vi:
4615 case AMDGPU::V_LSHLREV_B32_e32:
4616 case AMDGPU::V_LSHLREV_B32_e64:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4618 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4619 case AMDGPU::V_LSHLREV_B32_e32_vi:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4621 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4622 case AMDGPU::V_LSHLREV_B32_e64_vi:
4624 case AMDGPU::V_LSHLREV_B16_e32:
4625 case AMDGPU::V_LSHLREV_B16_e64:
4626 case AMDGPU::V_LSHLREV_B16_e32_vi:
4627 case AMDGPU::V_LSHLREV_B16_e64_vi:
4628 case AMDGPU::V_LSHLREV_B16_gfx10:
4630 case AMDGPU::V_LSHRREV_B16_e32:
4631 case AMDGPU::V_LSHRREV_B16_e64:
4632 case AMDGPU::V_LSHRREV_B16_e32_vi:
4633 case AMDGPU::V_LSHRREV_B16_e64_vi:
4634 case AMDGPU::V_LSHRREV_B16_gfx10:
4636 case AMDGPU::V_ASHRREV_I16_e32:
4637 case AMDGPU::V_ASHRREV_I16_e64:
4638 case AMDGPU::V_ASHRREV_I16_e32_vi:
4639 case AMDGPU::V_ASHRREV_I16_e64_vi:
4640 case AMDGPU::V_ASHRREV_I16_gfx10:
4642 case AMDGPU::V_LSHLREV_B64_e64:
4643 case AMDGPU::V_LSHLREV_B64_gfx10:
4644 case AMDGPU::V_LSHLREV_B64_vi:
4646 case AMDGPU::V_LSHRREV_B64_e64:
4647 case AMDGPU::V_LSHRREV_B64_gfx10:
4648 case AMDGPU::V_LSHRREV_B64_vi:
4650 case AMDGPU::V_ASHRREV_I64_e64:
4651 case AMDGPU::V_ASHRREV_I64_gfx10:
4652 case AMDGPU::V_ASHRREV_I64_vi:
4654 case AMDGPU::V_PK_LSHLREV_B16:
4655 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4656 case AMDGPU::V_PK_LSHLREV_B16_vi:
4658 case AMDGPU::V_PK_LSHRREV_B16:
4659 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4660 case AMDGPU::V_PK_LSHRREV_B16_vi:
4661 case AMDGPU::V_PK_ASHRREV_I16:
4662 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4663 case AMDGPU::V_PK_ASHRREV_I16_vi:
4670bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4672 const unsigned Opcode = Inst.
getOpcode();
4681 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4682 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4686 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4690 "lds_direct is not supported on this GPU");
4696 "lds_direct cannot be used with this instruction");
4700 if (SrcName != OpName::src0) {
4702 "lds_direct may be used as src0 only");
4712 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4713 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4714 if (
Op.isFlatOffset())
4715 return Op.getStartLoc();
4720bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4723 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4728 return validateFlatOffset(Inst,
Operands);
4731 return validateSMEMOffset(Inst,
Operands);
4736 const unsigned OffsetSize = 24;
4737 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4739 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4740 "-bit unsigned offset for buffer ops");
4744 const unsigned OffsetSize = 16;
4745 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4747 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4754bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4760 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4764 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4766 "flat offset modifier is not supported on this GPU");
4773 bool AllowNegative =
4775 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4777 Twine(
"expected a ") +
4778 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4779 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4788 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4789 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4790 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4791 return Op.getStartLoc();
4796bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4805 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4821 ?
"expected a 23-bit unsigned offset for buffer ops"
4822 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4823 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4824 :
"expected a 21-bit signed offset");
4829bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4832 const MCInstrDesc &
Desc = MII.
get(Opcode);
4836 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4837 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4839 const int OpIndices[] = {Src0Idx, Src1Idx};
4841 unsigned NumExprs = 0;
4842 unsigned NumLiterals = 0;
4845 for (
int OpIdx : OpIndices) {
4849 const MCOperand &MO = Inst.
getOperand(OpIdx);
4853 std::optional<int64_t>
Imm;
4856 }
else if (MO.
isExpr()) {
4865 if (!
Imm.has_value()) {
4867 }
else if (!isInlineConstant(Inst, OpIdx)) {
4871 if (NumLiterals == 0 || LiteralValue !=
Value) {
4879 if (NumLiterals + NumExprs <= 1)
4883 "only one unique literal operand is allowed");
4887bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4890 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4899 if (OpSelIdx != -1) {
4903 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4904 if (OpSelHiIdx != -1) {
4913 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4923 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4924 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4925 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4926 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4928 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4929 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4935 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4937 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4947 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4948 if (Src2Idx != -1) {
4949 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4959bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4960 if (!hasTrue16Insts())
4962 const MCRegisterInfo *MRI = getMRI();
4964 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4970 if (OpSelOpValue == 0)
4972 unsigned OpCount = 0;
4973 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4974 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4975 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4982 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4983 if (OpSelOpIsHi != VGPRSuffixIsHi)
4992bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4993 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5005 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5016 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5017 AMDGPU::OpName::src1_modifiers,
5018 AMDGPU::OpName::src2_modifiers};
5020 for (
unsigned i = 0; i < 3; ++i) {
5030bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5033 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5034 if (DppCtrlIdx >= 0) {
5041 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5042 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5043 :
"DP ALU dpp only supports row_newbcast");
5048 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5049 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5052 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5054 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5058 "invalid operand for instruction");
5063 "src1 immediate operand invalid for instruction");
5073bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5074 return (
Reg == AMDGPU::VCC && isWave64()) ||
5075 (
Reg == AMDGPU::VCC_LO && isWave32());
5079bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5082 const MCInstrDesc &
Desc = MII.
get(Opcode);
5083 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5090 std::optional<unsigned> LiteralOpIdx;
5093 for (
int OpIdx : OpIndices) {
5097 const MCOperand &MO = Inst.
getOperand(OpIdx);
5103 std::optional<int64_t>
Imm;
5109 bool IsAnotherLiteral =
false;
5110 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5111 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5112 if (!
Imm.has_value()) {
5114 IsAnotherLiteral =
true;
5115 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5120 HasMandatoryLiteral);
5121 unsigned OpTy =
Desc.operands()[OpIdx].OperandType;
5131 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5132 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5134 "invalid operand for instruction");
5139 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5140 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5142 "invalid operand for instruction");
5146 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5153 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5154 !getFeatureBits()[FeatureVOP3Literal]) {
5156 "literal operands are not supported");
5160 if (LiteralOpIdx && IsAnotherLiteral) {
5162 getOperandLoc(
Operands, *LiteralOpIdx)),
5163 "only one unique literal operand is allowed");
5167 if (IsAnotherLiteral)
5168 LiteralOpIdx = OpIdx;
5177 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5191bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5197 ? AMDGPU::OpName::data0
5198 : AMDGPU::OpName::vdata;
5200 const MCRegisterInfo *MRI = getMRI();
5201 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5205 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5206 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5210 auto FB = getFeatureBits();
5211 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5212 if (DataAreg < 0 || DstAreg < 0)
5214 return DstAreg == DataAreg;
5217 return DstAreg < 1 && DataAreg < 1;
5220bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5221 auto FB = getFeatureBits();
5222 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5226 const MCRegisterInfo *MRI = getMRI();
5229 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5232 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5236 case AMDGPU::DS_LOAD_TR6_B96:
5237 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5242 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5246 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5247 if (VAddrIdx != -1) {
5250 if ((
Sub - AMDGPU::VGPR0) & 1)
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5256 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5261 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5262 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5282 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5283 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5285 return Op.getStartLoc();
5290bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5293 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5296 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5299 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5300 auto FB = getFeatureBits();
5301 bool UsesNeg =
false;
5302 if (FB[AMDGPU::FeatureGFX940Insts]) {
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5305 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5307 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5312 if (IsNeg == UsesNeg)
5315 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5316 :
"invalid modifier: neg is not supported");
5321bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5327 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5330 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5333 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5336 if (
Reg == AMDGPU::SGPR_NULL)
5339 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5343bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5348 return validateGWS(Inst,
Operands);
5353 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5358 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5359 Error(S,
"gds modifier is not supported on this GPU");
5367bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5369 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5373 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5374 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5377 const MCRegisterInfo *MRI = getMRI();
5378 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5380 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5383 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5385 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5392bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5396 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5404 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5407 Error(S,
"scale_offset is not supported on this GPU");
5410 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5413 Error(S,
"nv is not supported on this GPU");
5418 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5421 Error(S,
"scale_offset is not supported for this instruction");
5425 return validateTHAndScopeBits(Inst,
Operands, CPol);
5429 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5430 Error(S,
"cache policy is not supported for SMRD instructions");
5434 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5441 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5445 "scc modifier is not supported for this instruction on this GPU");
5456 :
"instruction must use glc");
5461 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5464 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5466 :
"instruction must not use glc");
5474bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5476 const unsigned CPol) {
5481 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5488 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5493 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5501 return PrintError(
"invalid th value for SMEM instruction");
5508 return PrintError(
"scope and th combination is not valid");
5514 return PrintError(
"invalid th value for atomic instructions");
5517 return PrintError(
"invalid th value for store instructions");
5520 return PrintError(
"invalid th value for load instructions");
5526bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5530 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5532 Error(Loc,
"TFE modifier has no meaning for store instructions");
5540bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5546 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5550 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5553 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5554 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5563 "wrong register tuple size for " +
5568 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5569 !validateFmt(BFmt, AMDGPU::OpName::src1))
5573 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5574 if (AScaleIdx == -1)
5578 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5582 "invalid matrix and scale format combination");
5589bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5592 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5593 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5596 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5599 "sleep forever is unsuported on the target");
5606bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5608 if (!validateLdsDirect(Inst,
Operands))
5610 if (!validateTrue16OpSel(Inst)) {
5612 "op_sel operand conflicts with 16-bit operand suffix");
5615 if (!validateSOPLiteral(Inst,
Operands))
5617 if (!validateVOPLiteral(Inst,
Operands)) {
5620 if (!validateConstantBusLimitations(Inst,
Operands)) {
5623 if (!validateVOPD(Inst,
Operands)) {
5626 if (!validateIntClampSupported(Inst)) {
5628 "integer clamping is not supported on this GPU");
5631 if (!validateOpSel(Inst)) {
5633 "invalid op_sel operand");
5636 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5638 "invalid neg_lo operand");
5641 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5643 "invalid neg_hi operand");
5646 if (!validateDPP(Inst,
Operands)) {
5650 if (!validateMIMGD16(Inst)) {
5652 "d16 modifier is not supported on this GPU");
5655 if (!validateMIMGDim(Inst,
Operands)) {
5656 Error(IDLoc,
"missing dim operand");
5659 if (!validateTensorR128(Inst)) {
5661 "instruction must set modifier r128=0");
5664 if (!validateMIMGMSAA(Inst)) {
5666 "invalid dim; must be MSAA type");
5669 if (!validateMIMGDataSize(Inst, IDLoc)) {
5672 if (!validateMIMGAddrSize(Inst, IDLoc))
5674 if (!validateMIMGAtomicDMask(Inst)) {
5676 "invalid atomic image dmask");
5679 if (!validateMIMGGatherDMask(Inst)) {
5681 "invalid image_gather dmask: only one bit must be set");
5684 if (!validateMovrels(Inst,
Operands)) {
5687 if (!validateOffset(Inst,
Operands)) {
5690 if (!validateMAIAccWrite(Inst,
Operands)) {
5693 if (!validateMAISrc2(Inst,
Operands)) {
5696 if (!validateMFMA(Inst,
Operands)) {
5699 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5703 if (!validateAGPRLdSt(Inst)) {
5706 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5707 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5708 :
"invalid register class: agpr loads and stores not supported on "
5712 if (!validateVGPRAlign(Inst)) {
5713 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5720 if (!validateBLGP(Inst,
Operands)) {
5724 if (!validateDivScale(Inst)) {
5725 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5728 if (!validateWaitCnt(Inst,
Operands)) {
5731 if (!validateTFE(Inst,
Operands)) {
5734 if (!validateWMMA(Inst,
Operands)) {
5737 if (!validateMonitorSleep(Inst,
Operands)) {
5746 unsigned VariantID = 0);
5750 unsigned VariantID);
5752bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5757bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5758 const FeatureBitset &FBS,
5759 ArrayRef<unsigned> Variants) {
5760 for (
auto Variant : Variants) {
5768bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5770 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5773 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5778 getParser().clearPendingErrors();
5782 StringRef VariantName = getMatchedVariantName();
5783 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5784 return Error(IDLoc, Twine(VariantName,
5785 " variant of this instruction is not supported"));
5789 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5790 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5792 FeatureBitset FeaturesWS32 = getFeatureBits();
5793 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5794 .
flip(AMDGPU::FeatureWavefrontSize32);
5795 FeatureBitset AvailableFeaturesWS32 =
5796 ComputeAvailableFeatures(FeaturesWS32);
5798 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5799 return Error(IDLoc,
"instruction requires wavesize=32");
5803 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5804 return Error(IDLoc,
"instruction not supported on this GPU (" +
5805 getSTI().
getCPU() +
")" +
": " + Mnemo);
5810 return Error(IDLoc,
"invalid instruction" + Suggestion);
5816 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5817 if (
Op.isToken() && InvalidOprIdx > 1) {
5818 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5819 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5824bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5828 bool MatchingInlineAsm) {
5831 unsigned Result = Match_Success;
5832 for (
auto Variant : getMatchedVariants()) {
5835 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5839 if (R == Match_Success || R == Match_MissingFeature ||
5840 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5841 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5842 Result != Match_MissingFeature)) {
5846 if (R == Match_Success)
5850 if (Result == Match_Success) {
5851 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5854 emitTargetDirective();
5862 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5869 case Match_MissingFeature:
5873 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5875 case Match_InvalidOperand: {
5876 SMLoc ErrorLoc = IDLoc;
5877 if (ErrorInfo != ~0ULL) {
5878 if (ErrorInfo >=
Operands.size()) {
5879 return Error(IDLoc,
"too few operands for instruction");
5881 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5882 if (ErrorLoc == SMLoc())
5886 return Error(ErrorLoc,
"invalid VOPDY instruction");
5888 return Error(ErrorLoc,
"invalid operand for instruction");
5891 case Match_MnemonicFail:
5897bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5902 if (getParser().parseAbsoluteExpression(Tmp)) {
5905 Ret =
static_cast<uint32_t
>(Tmp);
5909bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5910 if (!getSTI().getTargetTriple().isAMDGCN())
5911 return TokError(
"directive only supported for amdgcn architecture");
5913 std::string TargetIDDirective;
5914 SMLoc TargetStart = getTok().getLoc();
5915 if (getParser().parseEscapedString(TargetIDDirective))
5918 std::optional<AMDGPU::TargetID> MaybeParsed =
5921 return getParser().Error(TargetStart,
5922 "malformed target id '" + TargetIDDirective +
"'");
5925 const Triple &
TT = getSTI().getTargetTriple();
5931 return getParser().Error(
5932 TargetStart,
"target id '" + TargetIDDirective +
5933 "' specifies a processor that is not valid for "
5935 TT.getArchName() +
"'");
5938 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5939 getTargetStreamer().getTargetID();
5942 const Triple &STITriple = getSTI().getTargetTriple();
5943 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5944 return getParser().Error(
5945 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5946 " is incompatible with " +
5947 Twine(CurrentTargetID->toString()));
5951 StringRef DirectiveProcessor =
5954 if (DirectiveISA != ISA) {
5955 return getParser().Error(TargetStart,
5956 ".amdgcn_target directive processor " +
5957 Twine(DirectiveProcessor) +
5958 " does not match the specified processor " +
5959 Twine(getSTI().
getCPU()));
5965 CurrentTargetID->getXnackSetting())) {
5967 ".amdgcn_target directive has conflicting xnack settings");
5971 CurrentTargetID->getSramEccSetting())) {
5973 ".amdgcn_target directive has conflicting sramecc settings");
5979 getTargetStreamer().getTargetID()->setXnackSetting(
5981 getTargetStreamer().getTargetID()->setSramEccSetting(
5987bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5991bool AMDGPUAsmParser::calculateGPRBlocks(
5992 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5993 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5994 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5995 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5996 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6001 const MCExpr *
NumSGPRs = NextFreeSGPR;
6002 int64_t EvaluatedSGPRs;
6004 if (
ISA.Major >= 10)
6009 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6010 !Features.
test(FeatureSGPRInitBug) &&
6011 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6012 return OutOfRangeError(SGPRRange);
6014 const MCExpr *ExtraSGPRs =
6018 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6019 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6020 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6021 return OutOfRangeError(SGPRRange);
6023 if (Features.
test(FeatureSGPRInitBug))
6030 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6031 unsigned Granule) ->
const MCExpr * {
6035 const MCExpr *AlignToGPR =
6037 const MCExpr *DivGPR =
6043 VGPRBlocks = GetNumGPRBlocks(
6052bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6053 if (!getSTI().getTargetTriple().isAMDGCN())
6054 return TokError(
"directive only supported for amdgcn architecture");
6057 return TokError(
"directive only supported for amdhsa OS");
6059 StringRef KernelName;
6060 if (getParser().parseIdentifier(KernelName))
6067 AMDGPU::MCKernelDescriptor KD =
6077 const MCExpr *NextFreeVGPR = ZeroExpr;
6079 const MCExpr *NamedBarCnt = ZeroExpr;
6084 const MCExpr *NextFreeSGPR = ZeroExpr;
6087 unsigned ImpliedUserSGPRCount = 0;
6091 std::optional<unsigned> ExplicitUserSGPRCount;
6092 const MCExpr *ReserveVCC = OneExpr;
6093 const MCExpr *ReserveFlatScr = OneExpr;
6094 std::optional<bool> EnableWavefrontSize32;
6101 SMRange IDRange = getTok().getLocRange();
6102 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6105 if (ID ==
".end_amdhsa_kernel")
6108 if (!Seen.
insert(ID).second)
6109 return TokError(
".amdhsa_ directives cannot be repeated");
6111 SMLoc ValStart = getLoc();
6112 const MCExpr *ExprVal;
6113 if (getParser().parseExpression(ExprVal))
6115 SMLoc ValEnd = getLoc();
6116 SMRange ValRange = SMRange(ValStart, ValEnd);
6120 bool EvaluatableExpr;
6121 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6123 return OutOfRangeError(ValRange);
6127#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6128 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6129 return OutOfRangeError(RANGE); \
6130 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6135#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6137 return Error(IDRange.Start, "directive should have resolvable expression", \
6140 if (ID ==
".amdhsa_group_segment_fixed_size") {
6143 return OutOfRangeError(ValRange);
6145 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6148 return OutOfRangeError(ValRange);
6150 }
else if (ID ==
".amdhsa_kernarg_size") {
6152 return OutOfRangeError(ValRange);
6154 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6156 ExplicitUserSGPRCount = Val;
6157 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6161 "directive is not supported with architected flat scratch",
6164 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6167 ImpliedUserSGPRCount += 4;
6168 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6171 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6174 return OutOfRangeError(ValRange);
6178 ImpliedUserSGPRCount += Val;
6179 PreloadLength = Val;
6181 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6184 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6187 return OutOfRangeError(ValRange);
6191 PreloadOffset = Val;
6192 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6195 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6198 ImpliedUserSGPRCount += 2;
6199 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6202 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6205 ImpliedUserSGPRCount += 2;
6206 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6209 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6212 ImpliedUserSGPRCount += 2;
6213 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6216 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6219 ImpliedUserSGPRCount += 2;
6220 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6223 "directive is not supported with architected flat scratch",
6227 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6230 ImpliedUserSGPRCount += 2;
6231 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6234 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6237 ImpliedUserSGPRCount += 1;
6238 }
else if (ID ==
".amdhsa_wavefront_size32") {
6241 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6242 EnableWavefrontSize32 = Val;
6244 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6246 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6248 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6250 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6253 "directive is not supported with architected flat scratch",
6256 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6258 }
else if (ID ==
".amdhsa_enable_private_segment") {
6262 "directive is not supported without architected flat scratch",
6265 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6267 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6269 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6271 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6273 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6275 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6277 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6279 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6281 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6283 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6285 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6287 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6288 VGPRRange = ValRange;
6289 NextFreeVGPR = ExprVal;
6290 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6291 SGPRRange = ValRange;
6292 NextFreeSGPR = ExprVal;
6293 }
else if (ID ==
".amdhsa_accum_offset") {
6295 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6296 AccumOffset = ExprVal;
6297 }
else if (ID ==
".amdhsa_named_barrier_count") {
6299 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6300 NamedBarCnt = ExprVal;
6301 }
else if (ID ==
".amdhsa_reserve_vcc") {
6303 return OutOfRangeError(ValRange);
6304 ReserveVCC = ExprVal;
6305 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6307 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6310 "directive is not supported with architected flat scratch",
6313 return OutOfRangeError(ValRange);
6314 ReserveFlatScr = ExprVal;
6315 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6317 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6319 return OutOfRangeError(ValRange);
6320 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6321 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6322 if (Val != XnackOn) {
6323 return getParser().Error(
6325 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6327 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6329 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6331 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6333 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6335 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6337 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6339 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6341 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6343 }
else if (ID ==
".amdhsa_dx10_clamp") {
6344 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6345 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6348 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6350 }
else if (ID ==
".amdhsa_ieee_mode") {
6351 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6352 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6355 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6357 }
else if (ID ==
".amdhsa_fp16_overflow") {
6359 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6361 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6363 }
else if (ID ==
".amdhsa_tg_split") {
6365 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6368 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6371 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6373 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6375 }
else if (ID ==
".amdhsa_memory_ordered") {
6377 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6379 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6381 }
else if (ID ==
".amdhsa_forward_progress") {
6383 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6385 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6387 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6389 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6390 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6392 SharedVGPRCount = Val;
6394 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6396 }
else if (ID ==
".amdhsa_inst_pref_size") {
6398 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6399 if (
ISA.Major == 11) {
6401 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6405 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6408 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6411 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6413 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6415 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6417 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6420 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6422 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6424 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6426 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6428 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6430 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6432 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6434 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6436 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6438 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6440 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6442 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6445 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6448#undef PARSE_BITS_ENTRY
6451 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6452 return TokError(
".amdhsa_next_free_vgpr directive is required");
6454 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6455 return TokError(
".amdhsa_next_free_sgpr directive is required");
6457 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6459 return TokError(
"too many user SGPRs enabled, found " +
6460 Twine(UserSGPRCount) +
", but only " +
6466 if (PreloadLength) {
6472 const MCExpr *VGPRBlocks;
6473 const MCExpr *SGPRBlocks;
6474 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6475 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6476 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6477 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6480 int64_t EvaluatedVGPRBlocks;
6481 bool VGPRBlocksEvaluatable =
6482 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6483 if (VGPRBlocksEvaluatable &&
6485 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6486 return OutOfRangeError(VGPRRange);
6490 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6491 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6493 int64_t EvaluatedSGPRBlocks;
6494 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6496 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6497 return OutOfRangeError(SGPRRange);
6500 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6501 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6503 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6504 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6505 "enabled user SGPRs");
6511 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6512 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6517 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6518 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6523 return TokError(
"Kernarg size should be resolvable");
6525 if (PreloadLength && kernarg_size &&
6526 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6527 return TokError(
"Kernarg preload length + offset is larger than the "
6528 "kernarg segment size");
6531 if (!Seen.
contains(
".amdhsa_accum_offset"))
6532 return TokError(
".amdhsa_accum_offset directive is required");
6533 int64_t EvaluatedAccum;
6534 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6535 uint64_t UEvaluatedAccum = EvaluatedAccum;
6536 if (AccumEvaluatable &&
6537 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6538 return TokError(
"accum_offset should be in range [4..256] in "
6541 int64_t EvaluatedNumVGPR;
6542 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6546 return TokError(
"accum_offset exceeds total VGPR allocation");
6552 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6553 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6559 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6560 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6563 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6565 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6566 return TokError(
"shared_vgpr_count directive not valid on "
6567 "wavefront size 32");
6570 if (VGPRBlocksEvaluatable &&
6571 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6573 return TokError(
"shared_vgpr_count*2 + "
6574 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6579 emitTargetDirective();
6580 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6581 NextFreeVGPR, NextFreeSGPR,
6582 ReserveVCC, ReserveFlatScr);
6586bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6588 if (ParseAsAbsoluteExpression(
Version))
6591 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6592 emitTargetDirective();
6596bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6597 AMDGPUMCKernelCodeT &
C) {
6600 if (ID ==
"max_scratch_backing_memory_byte_size") {
6601 Parser.eatToEndOfStatement();
6605 SmallString<40> ErrStr;
6606 raw_svector_ostream Err(ErrStr);
6607 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6608 return TokError(Err.
str());
6612 if (ID ==
"enable_wavefront_size32") {
6615 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6617 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6620 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6624 if (ID ==
"wavefront_size") {
6625 if (
C.wavefront_size == 5) {
6627 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6629 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6630 }
else if (
C.wavefront_size == 6) {
6632 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6639bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6640 AMDGPUMCKernelCodeT KernelCode;
6650 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6653 if (ID ==
".end_amd_kernel_code_t")
6656 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6661 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6666bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6667 StringRef KernelName;
6668 if (!parseId(KernelName,
"expected symbol name"))
6671 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6678bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6679 if (!getSTI().getTargetTriple().isAMDGCN()) {
6680 return Error(getLoc(),
6681 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6685 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6687 std::optional<AMDGPU::TargetID> MaybeParsed =
6690 return Error(getParser().getTok().getLoc(),
6691 "malformed target id '" + TargetIDDirective +
"'");
6694 const Triple &
TT = getSTI().getTargetTriple();
6700 return Error(getParser().getTok().getLoc(),
6701 "target id '" + TargetIDDirective +
6702 "' specifies a processor that is not valid for subarch '" +
6703 TT.getArchName() +
"'");
6706 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6707 getTargetStreamer().getTargetID();
6710 const Triple &STITriple = getSTI().getTargetTriple();
6711 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6712 return Error(getParser().getTok().getLoc(),
6713 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6714 " is incompatible with " +
6715 Twine(CurrentTargetID->toString()));
6719 StringRef DirectiveProcessor =
6722 if (DirectiveISA != ISA) {
6723 return Error(getParser().getTok().getLoc(),
6724 ".amd_amdgpu_isa directive processor " +
6725 Twine(DirectiveProcessor) +
6726 " does not match the specified processor " +
6727 Twine(getSTI().
getCPU()));
6730 getTargetStreamer().EmitISAVersion();
6736bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6739 std::string HSAMetadataString;
6744 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6745 return Error(getLoc(),
"invalid HSA metadata");
6752bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6753 const char *AssemblerDirectiveEnd,
6754 std::string &CollectString) {
6756 raw_string_ostream CollectStream(CollectString);
6758 getLexer().setSkipSpace(
false);
6760 bool FoundEnd =
false;
6763 CollectStream << getTokenStr();
6767 if (trySkipId(AssemblerDirectiveEnd)) {
6772 CollectStream << Parser.parseStringToEndOfStatement()
6773 <<
getContext().getAsmInfo().getSeparatorString();
6775 Parser.eatToEndOfStatement();
6778 getLexer().setSkipSpace(
true);
6781 return TokError(Twine(
"expected directive ") +
6782 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6789bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6795 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6796 if (!PALMetadata->setFromString(
String))
6797 return Error(getLoc(),
"invalid PAL metadata");
6802bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6805 Twine(
" directive is "
6806 "not available on non-amdpal OSes"))
6810 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6811 PALMetadata->setLegacy();
6814 if (ParseAsAbsoluteExpression(
Key)) {
6815 return TokError(Twine(
"invalid value in ") +
6819 return TokError(Twine(
"expected an even number of values in ") +
6822 if (ParseAsAbsoluteExpression(
Value)) {
6823 return TokError(Twine(
"invalid value in ") +
6826 PALMetadata->setRegister(
Key,
Value);
6835bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6836 if (getParser().checkForValidSection())
6840 SMLoc NameLoc = getLoc();
6841 if (getParser().parseIdentifier(Name))
6842 return TokError(
"expected identifier in directive");
6845 if (getParser().parseComma())
6851 SMLoc SizeLoc = getLoc();
6852 if (getParser().parseAbsoluteExpression(
Size))
6855 return Error(SizeLoc,
"size must be non-negative");
6856 if (
Size > LocalMemorySize)
6857 return Error(SizeLoc,
"size is too large");
6861 SMLoc AlignLoc = getLoc();
6862 if (getParser().parseAbsoluteExpression(Alignment))
6865 return Error(AlignLoc,
"alignment must be a power of two");
6870 if (Alignment >= 1u << 31)
6871 return Error(AlignLoc,
"alignment is too large");
6877 Symbol->redefineIfPossible();
6878 if (!
Symbol->isUndefined())
6879 return Error(NameLoc,
"invalid symbol redefinition");
6881 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6885bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6886 if (getParser().checkForValidSection())
6890 if (getParser().parseIdentifier(FuncName))
6891 return TokError(
"expected symbol name after .amdgpu_info");
6894 AMDGPU::InfoSectionData ParsedInfoData;
6895 AMDGPU::FuncInfo FI;
6897 bool HasScalarAttrs =
false;
6904 SMLoc IDLoc = getLoc();
6905 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6908 if (ID ==
".end_amdgpu_info")
6916 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6918 if (Dir ==
"flags") {
6920 if (getParser().parseAbsoluteExpression(Val))
6923 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6925 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6927 HasScalarAttrs =
true;
6928 }
else if (Dir ==
"num_sgpr") {
6930 if (getParser().parseAbsoluteExpression(Val))
6932 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6933 HasScalarAttrs =
true;
6934 }
else if (Dir ==
"num_vgpr") {
6936 if (getParser().parseAbsoluteExpression(Val))
6939 HasScalarAttrs =
true;
6940 }
else if (Dir ==
"num_agpr") {
6942 if (getParser().parseAbsoluteExpression(Val))
6945 HasScalarAttrs =
true;
6946 }
else if (Dir ==
"private_segment_size") {
6948 if (getParser().parseAbsoluteExpression(Val))
6951 HasScalarAttrs =
true;
6952 }
else if (Dir ==
"use") {
6954 if (getParser().parseIdentifier(ResName))
6955 return TokError(
"expected resource symbol for .amdgpu_use");
6956 ParsedInfoData.
Uses.push_back(
6957 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6958 }
else if (Dir ==
"call") {
6960 if (getParser().parseIdentifier(DstName))
6961 return TokError(
"expected callee symbol for .amdgpu_call");
6962 ParsedInfoData.
Calls.push_back(
6963 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6964 }
else if (Dir ==
"indirect_call") {
6966 if (getParser().parseEscapedString(TypeId))
6967 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6968 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6969 }
else if (Dir ==
"typeid") {
6971 if (getParser().parseEscapedString(TypeId))
6972 return TokError(
"expected type ID string for .amdgpu_typeid");
6973 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6975 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6980 ParsedInfoData.
Funcs.push_back(std::move(FI));
6982 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6983 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6984 Data.Funcs.push_back(std::move(Func));
6985 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6986 Data.Uses.push_back(Use);
6987 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6989 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6992 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6993 Data.TypeIds.push_back(std::move(TypeId));
6998void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7005void AMDGPUAsmParser::checkKernelPrologues() {
7006 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7007 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7008 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7009 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7010 if (!AMDHSAKernelSymbols.
contains(Sym))
7012 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7013 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7017 "' does not begin with the required prologue "
7018 "sequence: s_mov_b64 followed by v_nop and "
7019 "global_prefetch_b8");
7023 OpcodeStream.
clear();
7024 OpcodeStreamSymbols.clear();
7025 AMDHSAKernelSymbols.
clear();
7028void AMDGPUAsmParser::onEndOfFile() {
7029 emitTargetDirective();
7030 checkKernelPrologues();
7032 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7035bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7036 StringRef IDVal = DirectiveID.
getString();
7039 if (IDVal ==
".amdhsa_kernel")
7040 return ParseDirectiveAMDHSAKernel();
7042 if (IDVal ==
".amdhsa_code_object_version")
7043 return ParseDirectiveAMDHSACodeObjectVersion();
7047 return ParseDirectiveHSAMetadata();
7049 if (IDVal ==
".amd_kernel_code_t")
7050 return ParseDirectiveAMDKernelCodeT();
7052 if (IDVal ==
".amdgpu_hsa_kernel")
7053 return ParseDirectiveAMDGPUHsaKernel();
7055 if (IDVal ==
".amd_amdgpu_isa")
7056 return ParseDirectiveISAVersion();
7060 Twine(
" directive is "
7061 "not available on non-amdhsa OSes"))
7066 if (IDVal ==
".amdgcn_target")
7067 return ParseDirectiveAMDGCNTarget();
7069 if (IDVal ==
".amdgpu_lds")
7070 return ParseDirectiveAMDGPULDS();
7072 if (IDVal ==
".amdgpu_info")
7073 return ParseDirectiveAMDGPUInfo();
7076 return ParseDirectivePALMetadataBegin();
7079 return ParseDirectivePALMetadata();
7084bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7091 return hasSGPR104_SGPR105();
7094 case SRC_SHARED_BASE_LO:
7095 case SRC_SHARED_BASE:
7096 case SRC_SHARED_LIMIT_LO:
7097 case SRC_SHARED_LIMIT:
7099 case SRC_PRIVATE_BASE_LO:
7100 case SRC_PRIVATE_BASE:
7101 case SRC_PRIVATE_LIMIT_LO:
7102 case SRC_PRIVATE_LIMIT:
7104 case SRC_FLAT_SCRATCH_BASE_LO:
7105 case SRC_FLAT_SCRATCH_BASE_HI:
7106 return hasGloballyAddressableScratch();
7107 case SRC_POPS_EXITING_WAVE_ID:
7120 getTargetStreamer().getTargetID()->isXnackSupported();
7150 return hasSGPR102_SGPR103();
7158 ParseStatus Res = parseVOPD(
Operands);
7163 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7175 SMLoc LBraceLoc = getLoc();
7180 auto Loc = getLoc();
7183 Error(Loc,
"expected a register");
7187 RBraceLoc = getLoc();
7192 "expected a comma or a closing square bracket"))
7196 if (
Operands.size() - Prefix > 1) {
7198 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7199 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7208StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7210 setForcedEncodingSize(0);
7211 setForcedDPP(
false);
7212 setForcedSDWA(
false);
7214 if (
Name.consume_back(
"_e64_dpp")) {
7216 setForcedEncodingSize(64);
7219 if (
Name.consume_back(
"_e64")) {
7220 setForcedEncodingSize(64);
7223 if (
Name.consume_back(
"_e32")) {
7224 setForcedEncodingSize(32);
7227 if (
Name.consume_back(
"_dpp")) {
7231 if (
Name.consume_back(
"_sdwa")) {
7232 setForcedSDWA(
true);
7240 unsigned VariantID);
7246 Name = parseMnemonicSuffix(Name);
7252 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7254 bool IsMIMG = Name.starts_with(
"image_");
7257 OperandMode
Mode = OperandMode_Default;
7259 Mode = OperandMode_NSA;
7263 checkUnsupportedInstruction(Name, NameLoc);
7264 if (!Parser.hasPendingError()) {
7267 :
"not a valid operand.";
7287ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7290 if (!trySkipId(Name))
7293 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7297ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7306ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7308 std::function<
bool(int64_t &)> ConvertResult) {
7312 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7316 if (ConvertResult && !ConvertResult(
Value)) {
7317 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7320 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7324ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7326 bool (*ConvertResult)(int64_t &)) {
7335 const unsigned MaxSize = 4;
7339 for (
int I = 0;; ++
I) {
7341 SMLoc Loc = getLoc();
7345 if (
Op != 0 &&
Op != 1)
7346 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7353 if (
I + 1 == MaxSize)
7354 return Error(getLoc(),
"expected a closing square bracket");
7360 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7364ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7366 AMDGPUOperand::ImmTy ImmTy,
7367 bool IgnoreNegative) {
7371 if (trySkipId(Name)) {
7373 }
else if (trySkipId(
"no", Name)) {
7382 return Error(S,
"r128 modifier is not supported on this GPU");
7383 if (Name ==
"a16" && !
hasA16())
7384 return Error(S,
"a16 modifier is not supported on this GPU");
7386 if (Bit == 0 && Name ==
"gds") {
7389 return Error(S,
"nogds is not allowed");
7392 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7393 ImmTy = AMDGPUOperand::ImmTyR128A16;
7395 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7399unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7400 bool &Disabling)
const {
7401 Disabling =
Id.consume_front(
"no");
7404 return StringSwitch<unsigned>(Id)
7411 return StringSwitch<unsigned>(Id)
7421 SMLoc StringLoc = getLoc();
7423 int64_t CPolVal = 0;
7443 ResScope = parseScope(
Operands, Scope);
7456 if (trySkipId(
"nv")) {
7460 }
else if (trySkipId(
"no",
"nv")) {
7467 if (trySkipId(
"scale_offset")) {
7471 }
else if (trySkipId(
"no",
"scale_offset")) {
7484 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7485 AMDGPUOperand::ImmTyCPol));
7490 SMLoc OpLoc = getLoc();
7491 unsigned Enabled = 0, Seen = 0;
7495 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7502 return Error(S,
"dlc modifier is not supported on this GPU");
7505 return Error(S,
"scc modifier is not supported on this GPU");
7508 return Error(S,
"duplicate cache policy modifier");
7520 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7529 ParseStatus Res = parseStringOrIntWithPrefix(
7530 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7544 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7548 if (
Value ==
"TH_DEFAULT")
7550 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7551 Value ==
"TH_LOAD_NT_WB") {
7552 return Error(StringLoc,
"invalid th value");
7553 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7555 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7557 }
else if (
Value.consume_front(
"TH_STORE_")) {
7560 return Error(StringLoc,
"invalid th value");
7563 if (
Value ==
"BYPASS")
7568 TH |= StringSwitch<int64_t>(
Value)
7578 .Default(0xffffffff);
7580 TH |= StringSwitch<int64_t>(
Value)
7591 .Default(0xffffffff);
7594 if (TH == 0xffffffff)
7595 return Error(StringLoc,
"invalid th value");
7602 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7603 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7604 std::optional<unsigned> InsertAt = std::nullopt) {
7605 auto i = OptionalIdx.find(ImmT);
7606 if (i != OptionalIdx.end()) {
7607 unsigned Idx = i->second;
7608 const AMDGPUOperand &
Op =
7609 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7613 Op.addImmOperands(Inst, 1);
7615 if (InsertAt.has_value())
7622ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7628 StringLoc = getLoc();
7633ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7639 SMLoc StringLoc = getLoc();
7643 Value = getTokenStr();
7647 if (
Value == Ids[IntVal])
7652 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7653 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7658ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7660 AMDGPUOperand::ImmTy
Type) {
7664 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7666 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7675bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7678 SMLoc Loc = getLoc();
7680 auto Res = parseIntWithPrefix(Pref, Val);
7686 if (Val < 0 || Val > MaxVal) {
7687 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7696 AMDGPUOperand::ImmTy ImmTy) {
7697 const char *Pref =
"index_key";
7699 SMLoc Loc = getLoc();
7700 auto Res = parseIntWithPrefix(Pref, ImmVal);
7704 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7705 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7706 (ImmVal < 0 || ImmVal > 1))
7707 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7709 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7710 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7712 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7717 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7721 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7725 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7730 AMDGPUOperand::ImmTy
Type) {
7736 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7737 AMDGPUOperand::ImmTyMatrixAFMT);
7741 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7742 AMDGPUOperand::ImmTyMatrixBFMT);
7747 AMDGPUOperand::ImmTy
Type) {
7753 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7754 AMDGPUOperand::ImmTyMatrixAScale);
7758 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7759 AMDGPUOperand::ImmTyMatrixBScale);
7764 AMDGPUOperand::ImmTy
Type) {
7770 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7771 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7775 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7776 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7781ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7782 using namespace llvm::AMDGPU::MTBUFFormat;
7788 for (
int I = 0;
I < 2; ++
I) {
7789 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7792 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7797 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7803 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7806 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7807 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7813ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7814 using namespace llvm::AMDGPU::MTBUFFormat;
7818 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7821 if (Fmt == UFMT_UNDEF)
7828bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7829 StringRef FormatStr, SMLoc Loc) {
7830 using namespace llvm::AMDGPU::MTBUFFormat;
7834 if (
Format != DFMT_UNDEF) {
7840 if (
Format != NFMT_UNDEF) {
7845 Error(Loc,
"unsupported format");
7849ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7852 using namespace llvm::AMDGPU::MTBUFFormat;
7856 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7861 SMLoc Loc = getLoc();
7862 if (!parseId(Str,
"expected a format string") ||
7863 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7865 if (Dfmt == DFMT_UNDEF)
7866 return Error(Loc,
"duplicate numeric format");
7867 if (Nfmt == NFMT_UNDEF)
7868 return Error(Loc,
"duplicate data format");
7871 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7872 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7876 if (Ufmt == UFMT_UNDEF)
7877 return Error(FormatLoc,
"unsupported format");
7886ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7889 using namespace llvm::AMDGPU::MTBUFFormat;
7892 if (Id == UFMT_UNDEF)
7896 return Error(Loc,
"unified format is not supported on this GPU");
7902ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7903 using namespace llvm::AMDGPU::MTBUFFormat;
7904 SMLoc Loc = getLoc();
7909 return Error(Loc,
"out of range format");
7914ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7915 using namespace llvm::AMDGPU::MTBUFFormat;
7921 StringRef FormatStr;
7922 SMLoc Loc = getLoc();
7923 if (!parseId(FormatStr,
"expected a format string"))
7926 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7928 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7938 return parseNumericFormat(
Format);
7942 using namespace llvm::AMDGPU::MTBUFFormat;
7946 SMLoc Loc = getLoc();
7956 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7975 Res = parseSymbolicOrNumericFormat(
Format);
7980 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7981 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7988 return Error(getLoc(),
"duplicate format");
7994 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7996 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7997 AMDGPUOperand::ImmTyInstOffset);
8004 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8006 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8012 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8015 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8025 OptionalImmIndexMap OptionalIdx;
8027 unsigned OperandIdx[4];
8028 unsigned EnMask = 0;
8031 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8032 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8037 OperandIdx[SrcIdx] = Inst.
size();
8038 Op.addRegOperands(Inst, 1);
8045 OperandIdx[SrcIdx] = Inst.
size();
8051 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8052 Op.addImmOperands(Inst, 1);
8056 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8060 OptionalIdx[
Op.getImmTy()] = i;
8066 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8073 for (
auto i = 0; i < SrcIdx; ++i) {
8075 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8081 AMDGPUOperand::ImmTyExpCompr);
8091 int64_t CntVal,
bool Saturate,
8097 IntVal =
encode(ISA, IntVal, CntVal);
8098 if (CntVal !=
decode(ISA, IntVal)) {
8100 IntVal =
encode(ISA, IntVal, -1);
8108bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8110 SMLoc CntLoc = getLoc();
8111 StringRef CntName = getTokenStr();
8118 SMLoc ValLoc = getLoc();
8125 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8127 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8129 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8132 Error(CntLoc,
"invalid counter name " + CntName);
8137 Error(ValLoc,
"too large value for " + CntName);
8146 Error(getLoc(),
"expected a counter name");
8160 if (!parseCnt(Waitcnt))
8168 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8172bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8173 SMLoc FieldLoc = getLoc();
8174 StringRef FieldName = getTokenStr();
8179 SMLoc ValueLoc = getLoc();
8186 if (FieldName ==
"instid0") {
8188 }
else if (FieldName ==
"instskip") {
8190 }
else if (FieldName ==
"instid1") {
8193 Error(FieldLoc,
"invalid field name " + FieldName);
8212 .Case(
"VALU_DEP_1", 1)
8213 .Case(
"VALU_DEP_2", 2)
8214 .Case(
"VALU_DEP_3", 3)
8215 .Case(
"VALU_DEP_4", 4)
8216 .Case(
"TRANS32_DEP_1", 5)
8217 .Case(
"TRANS32_DEP_2", 6)
8218 .Case(
"TRANS32_DEP_3", 7)
8219 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8220 .Case(
"SALU_CYCLE_1", 9)
8221 .Case(
"SALU_CYCLE_2", 10)
8222 .Case(
"SALU_CYCLE_3", 11)
8230 Delay |=
Value << Shift;
8240 if (!parseDelay(Delay))
8248 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8252bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8254bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8260void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8261 StringRef DepCtrName) {
8264 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8267 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8270 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8273 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8280bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8282 using namespace llvm::AMDGPU::DepCtr;
8284 SMLoc DepCtrLoc = getLoc();
8285 StringRef DepCtrName = getTokenStr();
8295 unsigned PrevOprMask = UsedOprMask;
8296 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8299 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8308 Error(getLoc(),
"expected a counter name");
8313 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8314 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8319 using namespace llvm::AMDGPU::DepCtr;
8322 SMLoc Loc = getLoc();
8325 unsigned UsedOprMask = 0;
8327 if (!parseDepCtr(DepCtr, UsedOprMask))
8335 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8339bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8345ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8347 OperandInfoTy &Width) {
8348 using namespace llvm::AMDGPU::Hwreg;
8354 HwReg.Loc = getLoc();
8357 HwReg.IsSymbolic =
true;
8359 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8367 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8377 Width.Loc = getLoc();
8386 using namespace llvm::AMDGPU::Hwreg;
8389 SMLoc Loc = getLoc();
8391 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8393 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8394 HwregOffset::Default);
8395 struct : StructuredOpField {
8396 using StructuredOpField::StructuredOpField;
8397 bool validate(AMDGPUAsmParser &Parser)
const override {
8399 return Error(Parser,
"only values from 1 to 32 are legal");
8402 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8403 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8406 Res = parseHwregFunc(HwReg,
Offset, Width);
8409 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8411 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8415 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8422 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8424 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8428bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8434bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8435 OperandInfoTy &Stream) {
8436 using namespace llvm::AMDGPU::SendMsg;
8441 Msg.IsSymbolic =
true;
8448 Op.IsDefined =
true;
8454 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8459 Stream.IsDefined =
true;
8460 Stream.Loc = getLoc();
8469bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8470 const OperandInfoTy &
Op,
8471 const OperandInfoTy &Stream) {
8472 using namespace llvm::AMDGPU::SendMsg;
8481 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8486 Error(
Msg.Loc,
"invalid message id");
8492 Error(
Op.Loc,
"message does not support operations");
8494 Error(
Msg.Loc,
"missing message operation");
8500 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8502 Error(
Op.Loc,
"invalid operation id");
8507 Error(Stream.Loc,
"message operation does not support streams");
8511 Error(Stream.Loc,
"invalid message stream id");
8518 using namespace llvm::AMDGPU::SendMsg;
8521 SMLoc Loc = getLoc();
8525 OperandInfoTy
Op(OP_NONE_);
8526 OperandInfoTy Stream(STREAM_ID_NONE_);
8527 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8532 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8534 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8540 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8544bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8547 using namespace llvm::AMDGPU::WaitEvent;
8549 SMLoc Loc = getLoc();
8552 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8554 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8556 StructuredOpField *TargetBitfield =
8557 isGFX11() ? &DontWaitExportReady : &ExportReady;
8559 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8563 if (!validateStructuredOpFields({TargetBitfield}))
8565 ImmVal = TargetBitfield->Val;
8572 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8574 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8575 AMDGPUOperand::ImmTyWaitEvent));
8579bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8592 int Slot = StringSwitch<int>(Str)
8599 return Error(S,
"invalid interpolation slot");
8602 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8613 if (!Str.starts_with(
"attr"))
8614 return Error(S,
"invalid interpolation attribute");
8616 StringRef Chan = Str.take_back(2);
8617 int AttrChan = StringSwitch<int>(Chan)
8624 return Error(S,
"invalid or missing interpolation attribute channel");
8626 Str = Str.drop_back(2).drop_front(4);
8629 if (Str.getAsInteger(10, Attr))
8630 return Error(S,
"invalid or missing interpolation attribute number");
8633 return Error(S,
"out of bounds interpolation attribute number");
8638 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8639 Operands.push_back(AMDGPUOperand::CreateImm(
8640 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8649 using namespace llvm::AMDGPU::Exp;
8659 return Error(S, (Id == ET_INVALID)
8660 ?
"invalid exp target"
8661 :
"exp target is not supported on this GPU");
8664 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8672bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8676bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8681 return getTokenKind() ==
Kind;
8684StringRef AMDGPUAsmParser::getId()
const {
8688bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8696bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8698 StringRef Tok = getTokenStr();
8707bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8709 if (isId(Id) && peekToken().is(Kind)) {
8718 if (isToken(Kind)) {
8726 const StringRef ErrMsg) {
8727 if (!trySkipToken(Kind)) {
8728 Error(getLoc(), ErrMsg);
8734bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8738 if (Parser.parseExpression(Expr))
8741 if (Expr->evaluateAsAbsolute(
Imm))
8744 if (Expected.empty()) {
8745 Error(S,
"expected absolute expression");
8748 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8757 if (Parser.parseExpression(Expr))
8761 if (Expr->evaluateAsAbsolute(IntVal)) {
8762 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8764 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8769bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8771 Val =
getToken().getStringContents();
8775 Error(getLoc(), ErrMsg);
8779bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8781 Val = getTokenStr();
8785 if (!ErrMsg.
empty())
8786 Error(getLoc(), ErrMsg);
8790AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8792AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8795 : getLexer().peekTok(ShouldSkipSpace);
8799 auto TokCount = getLexer().peekTokens(Tokens);
8801 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8806 return getLexer().getKind();
8809SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8811StringRef AMDGPUAsmParser::getTokenStr()
const {
8815void AMDGPUAsmParser::lex() { Parser.Lex(); }
8817const AMDGPUOperand &
8819 int MCOpIdx)
const {
8821 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8822 if (TargetOp.getMCOpIdx() == MCOpIdx)
8829 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8833SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8838 int MCOpIdx)
const {
8839 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8842SMLoc AMDGPUAsmParser::getOperandLoc(
8843 std::function<
bool(
const AMDGPUOperand &)>
Test,
8845 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8846 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8848 return Op.getStartLoc();
8853SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8855 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8870 StringRef
Id = getTokenStr();
8871 SMLoc IdLoc = getLoc();
8877 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8878 if (
I == Fields.
end())
8879 return Error(IdLoc,
"unknown field");
8880 if ((*I)->IsDefined)
8881 return Error(IdLoc,
"duplicate field");
8884 (*I)->Loc = getLoc();
8887 (*I)->IsDefined =
true;
8894bool AMDGPUAsmParser::validateStructuredOpFields(
8896 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8897 return F->validate(*
this);
8907 const unsigned XorMask) {
8914bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8915 const unsigned MaxVal,
8916 const Twine &ErrMsg, SMLoc &Loc) {
8932bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8933 const unsigned MinVal,
8934 const unsigned MaxVal,
8935 const StringRef ErrMsg) {
8937 for (
unsigned i = 0; i < OpNum; ++i) {
8938 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8945bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8946 using namespace llvm::AMDGPU::Swizzle;
8949 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8950 "expected a 2-bit lane id")) {
8960bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8961 using namespace llvm::AMDGPU::Swizzle;
8967 if (!parseSwizzleOperand(GroupSize, 2, 32,
8968 "group size must be in the interval [2,32]", Loc)) {
8972 Error(Loc,
"group size must be a power of two");
8975 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8976 "lane id must be in the interval [0,group size - 1]",
8984bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8985 using namespace llvm::AMDGPU::Swizzle;
8990 if (!parseSwizzleOperand(GroupSize, 2, 32,
8991 "group size must be in the interval [2,32]", Loc)) {
8995 Error(Loc,
"group size must be a power of two");
9003bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9004 using namespace llvm::AMDGPU::Swizzle;
9009 if (!parseSwizzleOperand(GroupSize, 1, 16,
9010 "group size must be in the interval [1,16]", Loc)) {
9014 Error(Loc,
"group size must be a power of two");
9022bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9023 using namespace llvm::AMDGPU::Swizzle;
9030 SMLoc StrLoc = getLoc();
9031 if (!parseString(Ctl)) {
9034 if (Ctl.
size() != BITMASK_WIDTH) {
9035 Error(StrLoc,
"expected a 5-character mask");
9039 unsigned AndMask = 0;
9040 unsigned OrMask = 0;
9041 unsigned XorMask = 0;
9043 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9047 Error(StrLoc,
"invalid mask");
9068bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9069 using namespace llvm::AMDGPU::Swizzle;
9072 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9078 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9079 "FFT swizzle must be in the interval [0," +
9080 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9088bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9089 using namespace llvm::AMDGPU::Swizzle;
9092 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9099 if (!parseSwizzleOperand(
Direction, 0, 1,
9100 "direction must be 0 (left) or 1 (right)", Loc))
9104 if (!parseSwizzleOperand(
9105 RotateSize, 0, ROTATE_MAX_SIZE,
9106 "number of threads to rotate must be in the interval [0," +
9107 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9112 (RotateSize << ROTATE_SIZE_SHIFT);
9116bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9118 SMLoc OffsetLoc = getLoc();
9124 Error(OffsetLoc,
"expected a 16-bit offset");
9130bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9131 using namespace llvm::AMDGPU::Swizzle;
9135 SMLoc ModeLoc = getLoc();
9138 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9139 Ok = parseSwizzleQuadPerm(
Imm);
9140 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9141 Ok = parseSwizzleBitmaskPerm(
Imm);
9142 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9143 Ok = parseSwizzleBroadcast(
Imm);
9144 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9145 Ok = parseSwizzleSwap(
Imm);
9146 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9147 Ok = parseSwizzleReverse(
Imm);
9148 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9149 Ok = parseSwizzleFFT(
Imm);
9150 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9151 Ok = parseSwizzleRotate(
Imm);
9153 Error(ModeLoc,
"expected a swizzle mode");
9156 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9166 if (trySkipId(
"offset")) {
9170 if (trySkipId(
"swizzle")) {
9171 Ok = parseSwizzleMacro(
Imm);
9173 Ok = parseSwizzleOffset(
Imm);
9178 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9185bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9191int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9193 using namespace llvm::AMDGPU::VGPRIndexMode;
9205 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9206 if (trySkipId(IdSymbolic[ModeId])) {
9214 ?
"expected a VGPR index mode or a closing parenthesis"
9215 :
"expected a VGPR index mode");
9220 Error(S,
"duplicate VGPR index mode");
9228 "expected a comma or a closing parenthesis"))
9237 using namespace llvm::AMDGPU::VGPRIndexMode;
9243 Imm = parseGPRIdxMacro();
9247 if (getParser().parseAbsoluteExpression(
Imm))
9250 return Error(S,
"invalid immediate: only 4-bit values are legal");
9254 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9258bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9269 if (isRegister() || isModifier())
9276 assert(Opr.isImm() || Opr.isExpr());
9277 SMLoc Loc = Opr.getStartLoc();
9281 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9282 Error(Loc,
"expected an absolute expression or a label");
9283 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9284 Error(Loc,
"expected a 16-bit signed jump offset");
9304 OptionalImmIndexMap OptionalIdx;
9305 unsigned FirstOperandIdx = 1;
9306 bool IsAtomicReturn =
false;
9312 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9313 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9317 Op.addRegOperands(Inst, 1);
9321 if (IsAtomicReturn && i == FirstOperandIdx)
9322 Op.addRegOperands(Inst, 1);
9327 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9328 Op.addImmOperands(Inst, 1);
9340 OptionalIdx[
Op.getImmTy()] = i;
9344 AMDGPUOperand::ImmTyOffset);
9356bool AMDGPUOperand::isSMRDOffset8()
const {
9360bool AMDGPUOperand::isSMEMOffset()
const {
9362 return isImmLiteral();
9365bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9400bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9401 if (BoundCtrl == 0 || BoundCtrl == 1) {
9409void AMDGPUAsmParser::onBeginOfFile() {
9410 if (!getParser().getStreamer().getTargetStreamer())
9413 if (!getTargetStreamer().getTargetID())
9414 getTargetStreamer().initializeTargetID(getSTI(),
9418void AMDGPUAsmParser::emitTargetDirective() {
9419 if (TargetDirectiveEmitted)
9421 TargetDirectiveEmitted =
true;
9423 if (!getParser().getStreamer().getTargetStreamer() ||
9428 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9437bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9441 StringRef TokenId = getTokenStr();
9442 AGVK VK = StringSwitch<AGVK>(TokenId)
9443 .Case(
"max", AGVK::AGVK_Max)
9444 .Case(
"min", AGVK::AGVK_Min)
9445 .Case(
"or", AGVK::AGVK_Or)
9446 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9447 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9448 .Case(
"alignto", AGVK::AGVK_AlignTo)
9449 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9450 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9451 .Default(AGVK::AGVK_None);
9460 if (Exprs.
empty()) {
9462 "empty " + Twine(TokenId) +
" expression");
9465 if (CommaCount + 1 != Exprs.
size()) {
9467 "mismatch of commas in " + Twine(TokenId) +
" expression");
9471 Expected && Exprs.
size() != Expected) {
9472 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9473 Twine(Expected) +
" operands");
9480 if (getParser().parseExpression(Expr, EndLoc))
9484 if (LastTokenWasComma)
9488 "unexpected token in " + Twine(TokenId) +
" expression");
9494 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9498 StringRef
Name = getTokenStr();
9499 if (Name ==
"mul") {
9500 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9504 if (Name ==
"div") {
9505 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9516 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9521 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9522 AMDGPU::OpName::src2};
9530 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9535 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9537 if (
DstOp.isReg() &&
9542 if ((OpSel & (1 << SrcNum)) != 0)
9548void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9555 OptionalImmIndexMap &OptionalIdx) {
9556 cvtVOP3P(Inst,
Operands, OptionalIdx);
9565 &&
Desc.NumOperands > (OpNum + 1)
9567 &&
Desc.operands()[OpNum + 1].RegClass != -1
9569 &&
Desc.getOperandConstraint(OpNum + 1,
9573void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9575 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9576 AMDGPU::OpName::src2};
9577 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9578 AMDGPU::OpName::src1_modifiers,
9579 AMDGPU::OpName::src2_modifiers};
9580 for (
int J = 0; J < 3; ++J) {
9581 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9587 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9590 if ((OpSel & (1 << J)) != 0)
9593 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9600void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9602 OptionalImmIndexMap OptionalIdx;
9607 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9608 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9612 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9614 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9615 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9616 Op.isInterpAttrChan()) {
9618 }
else if (
Op.isImmModifier()) {
9619 OptionalIdx[
Op.getImmTy()] =
I;
9627 AMDGPUOperand::ImmTyHigh);
9631 AMDGPUOperand::ImmTyClamp);
9635 AMDGPUOperand::ImmTyOModSI);
9640 AMDGPUOperand::ImmTyOpSel);
9641 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9644 cvtOpSelHelper(Inst, OpSel);
9649 OptionalImmIndexMap OptionalIdx;
9654 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9655 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9659 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9661 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9662 }
else if (
Op.isImmModifier()) {
9663 OptionalIdx[
Op.getImmTy()] =
I;
9671 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9674 AMDGPUOperand::ImmTyOpSel);
9677 AMDGPUOperand::ImmTyWaitEXP);
9683 cvtOpSelHelper(Inst, OpSel);
9686void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9688 OptionalImmIndexMap OptionalIdx;
9691 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9695 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9696 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9699 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9704 if (NumOperands == CbszOpIdx) {
9709 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9710 }
else if (
Op.isImmModifier()) {
9711 OptionalIdx[
Op.getImmTy()] =
I;
9713 Op.addRegOrImmOperands(Inst, 1);
9718 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9719 if (CbszIdx != OptionalIdx.end()) {
9720 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9724 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9725 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9726 if (BlgpIdx != OptionalIdx.end()) {
9727 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9738 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9739 if (OpselIdx != OptionalIdx.end()) {
9740 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9744 unsigned OpSelHi = 0;
9745 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9746 if (OpselHiIdx != OptionalIdx.end()) {
9747 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9750 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9751 AMDGPU::OpName::src1_modifiers};
9753 for (
unsigned J = 0; J < 2; ++J) {
9754 unsigned ModVal = 0;
9755 if (OpSel & (1 << J))
9757 if (OpSelHi & (1 << J))
9760 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9766 OptionalImmIndexMap &OptionalIdx) {
9771 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9772 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9776 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9778 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9779 }
else if (
Op.isImmModifier()) {
9780 OptionalIdx[
Op.getImmTy()] =
I;
9782 Op.addRegOrImmOperands(Inst, 1);
9788 AMDGPUOperand::ImmTyScaleSel);
9792 AMDGPUOperand::ImmTyClamp);
9798 AMDGPUOperand::ImmTyByteSel);
9803 AMDGPUOperand::ImmTyOModSI);
9810 auto *it = Inst.
begin();
9812 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9821 OptionalImmIndexMap OptionalIdx;
9822 cvtVOP3(Inst,
Operands, OptionalIdx);
9826 OptionalImmIndexMap &OptIdx) {
9831 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9832 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9833 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9834 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9835 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9836 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9837 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9838 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9839 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9840 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9849 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9850 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9853 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9854 if (BitOp3Idx != -1) {
9861 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9862 if (OpSelIdx != -1) {
9866 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9867 if (OpSelHiIdx != -1) {
9868 int DefaultVal =
IsPacked ? -1 : 0;
9874 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9875 if (MatrixAFMTIdx != -1) {
9877 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9881 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9882 if (MatrixBFMTIdx != -1) {
9884 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9887 int MatrixAScaleIdx =
9888 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9889 if (MatrixAScaleIdx != -1) {
9891 AMDGPUOperand::ImmTyMatrixAScale, 0);
9894 int MatrixBScaleIdx =
9895 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9896 if (MatrixBScaleIdx != -1) {
9898 AMDGPUOperand::ImmTyMatrixBScale, 0);
9901 int MatrixAScaleFmtIdx =
9902 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9903 if (MatrixAScaleFmtIdx != -1) {
9905 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9908 int MatrixBScaleFmtIdx =
9909 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9910 if (MatrixBScaleFmtIdx != -1) {
9912 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9917 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9921 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9923 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9927 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9931 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9932 AMDGPU::OpName::src2};
9933 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9934 AMDGPU::OpName::src1_modifiers,
9935 AMDGPU::OpName::src2_modifiers};
9938 unsigned OpSelHi = 0;
9945 if (OpSelHiIdx != -1)
9954 for (
int J = 0; J < 3; ++J) {
9955 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9959 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9969 uint32_t ModVal = 0;
9971 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9972 if (SrcOp.
isReg() && getMRI()
9979 if ((OpSel & (1 << J)) != 0)
9983 if ((OpSelHi & (1 << J)) != 0)
9986 if ((NegLo & (1 << J)) != 0)
9989 if ((NegHi & (1 << J)) != 0)
9997 OptionalImmIndexMap OptIdx;
10003 unsigned i,
unsigned Opc,
10004 AMDGPU::OpName
OpName) {
10005 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10006 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10008 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10014 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10017 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10018 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10020 OptionalImmIndexMap OptIdx;
10021 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10022 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10023 OptIdx[
Op.getImmTy()] = i;
10028 AMDGPUOperand::ImmTyIndexKey8bit);
10032 AMDGPUOperand::ImmTyIndexKey16bit);
10036 AMDGPUOperand::ImmTyIndexKey32bit);
10053 SMLoc S = getLoc();
10056 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10057 SMLoc OpYLoc = getLoc();
10060 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10063 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10072 auto addOp = [&](uint16_t ParsedOprIdx) {
10073 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10075 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10079 Op.addRegOperands(Inst, 1);
10083 Op.addImmOperands(Inst, 1);
10095 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10099 const auto &CInfo = InstInfo[CompIdx];
10100 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10101 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10102 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10103 if (CInfo.hasSrc2Acc())
10104 addOp(CInfo.getIndexOfDstInParsedOperands());
10108 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10109 if (BitOp3Idx != -1) {
10110 OptionalImmIndexMap OptIdx;
10111 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10113 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10123bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10125bool AMDGPUOperand::isDPPCtrl()
const {
10126 using namespace AMDGPU::DPP;
10128 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10131 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10132 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10133 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10134 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10135 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10136 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10137 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10138 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10139 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10140 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10141 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10142 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10151bool AMDGPUOperand::isBLGP()
const {
10155bool AMDGPUOperand::isS16Imm()
const {
10159bool AMDGPUOperand::isU16Imm()
const {
10167bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10172 SMLoc Loc =
getToken().getEndLoc();
10173 Token = std::string(getTokenStr());
10175 if (getLoc() != Loc)
10180 if (!parseId(Suffix))
10184 StringRef DimId = Token;
10199 SMLoc S = getLoc();
10205 SMLoc Loc = getLoc();
10206 if (!parseDimId(Encoding))
10207 return Error(Loc,
"invalid dim value");
10210 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10219 SMLoc S = getLoc();
10228 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10231 for (
size_t i = 0; i < 8; ++i) {
10235 SMLoc Loc = getLoc();
10236 if (getParser().parseAbsoluteExpression(Sels[i]))
10238 if (0 > Sels[i] || 7 < Sels[i])
10239 return Error(Loc,
"expected a 3-bit value");
10242 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10246 for (
size_t i = 0; i < 8; ++i)
10247 DPP8 |= (Sels[i] << (i * 3));
10250 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10254bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10256 if (Ctrl ==
"row_newbcast")
10259 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10262 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10263 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10266 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10267 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10271int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10274 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10278 for (
int i = 0; i < 4; ++i) {
10283 SMLoc Loc = getLoc();
10284 if (getParser().parseAbsoluteExpression(Temp))
10286 if (Temp < 0 || Temp > 3) {
10287 Error(Loc,
"expected a 2-bit value");
10291 Val += (Temp << i * 2);
10294 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10300int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10301 using namespace AMDGPU::DPP;
10306 SMLoc Loc = getLoc();
10308 if (getParser().parseAbsoluteExpression(Val))
10311 struct DppCtrlCheck {
10317 DppCtrlCheck
Check =
10318 StringSwitch<DppCtrlCheck>(Ctrl)
10319 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10320 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10321 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10322 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10323 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10324 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10325 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10326 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10327 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10328 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10332 if (
Check.Ctrl == -1) {
10333 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10341 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10349 using namespace AMDGPU::DPP;
10352 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10355 SMLoc S = getLoc();
10361 if (Ctrl ==
"row_mirror") {
10362 Val = DppCtrl::ROW_MIRROR;
10363 }
else if (Ctrl ==
"row_half_mirror") {
10364 Val = DppCtrl::ROW_HALF_MIRROR;
10367 if (Ctrl ==
"quad_perm") {
10368 Val = parseDPPCtrlPerm();
10370 Val = parseDPPCtrlSel(Ctrl);
10379 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10385 OptionalImmIndexMap OptionalIdx;
10392 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10394 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10395 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10399 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10400 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10404 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10405 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10406 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10407 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10408 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10409 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10410 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10411 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10412 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10418 if (OldIdx == NumOperands) {
10420 constexpr int DST_IDX = 0;
10422 }
else if (Src2ModIdx == NumOperands) {
10432 if (IsVOP3CvtSrDpp) {
10441 if (TiedTo != -1) {
10446 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10448 if (IsDPP8 &&
Op.isDppFI()) {
10451 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10452 }
else if (
Op.isReg()) {
10453 Op.addRegOperands(Inst, 1);
10454 }
else if (
Op.isImm() &&
10456 Op.addImmOperands(Inst, 1);
10457 }
else if (
Op.isImm()) {
10458 OptionalIdx[
Op.getImmTy()] =
I;
10466 AMDGPUOperand::ImmTyClamp);
10472 AMDGPUOperand::ImmTyByteSel);
10477 AMDGPUOperand::ImmTyOModSI);
10480 cvtVOP3P(Inst,
Operands, OptionalIdx);
10482 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10485 AMDGPUOperand::ImmTyOpSel);
10490 AMDGPUOperand::ImmTyDPP8);
10491 using namespace llvm::AMDGPU::DPP;
10495 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10497 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10499 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10501 AMDGPUOperand::ImmTyDppBoundCtrl);
10505 AMDGPUOperand::ImmTyDppFI);
10511 OptionalImmIndexMap OptionalIdx;
10515 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10516 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10523 if (TiedTo != -1) {
10528 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10530 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {