71 enum KindTy { Token, Immediate, Register, Expression } Kind;
73 SMLoc StartLoc, EndLoc;
74 const AMDGPUAsmParser *AsmParser;
77 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
78 : Kind(Kind_), AsmParser(AsmParser_) {}
80 using Ptr = std::unique_ptr<AMDGPUOperand>;
88 bool hasFPModifiers()
const {
return Abs || Neg; }
89 bool hasIntModifiers()
const {
return Sext; }
90 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
91 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
92 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
94 int64_t getFPModifiersOperand()
const {
101 int64_t getIntModifiersOperand()
const {
107 int64_t getModifiersOperand()
const {
108 assert(!(hasFPModifiers() && hasIntModifiers()) &&
109 "fp and int modifiers should not be used simultaneously");
110 if (hasFPModifiers())
111 return getFPModifiersOperand();
112 if (hasIntModifiers())
113 return getIntModifiersOperand();
117 friend raw_ostream &
operator<<(raw_ostream &OS,
118 AMDGPUOperand::Modifiers Mods);
192 ImmTyMatrixAScaleFmt,
193 ImmTyMatrixBScaleFmt,
226 mutable int MCOpIdx = -1;
229 bool isToken()
const override {
return Kind == Token; }
231 bool isSymbolRefExpr()
const {
235 bool isImm()
const override {
return Kind == Immediate; }
237 bool isInlinableImm(MVT type)
const;
238 bool isLiteralImm(MVT type)
const;
240 bool isRegKind()
const {
return Kind == Register; }
242 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
244 bool isRegOrInline(
unsigned RCID, MVT type)
const {
245 return isRegClass(RCID) || isInlinableImm(type);
249 return isRegOrInline(RCID, type) || isLiteralImm(type);
252 bool isRegOrImmWithInt16InputMods()
const {
256 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
258 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
261 bool isRegOrImmWithInt32InputMods()
const {
265 bool isRegOrInlineImmWithInt16InputMods()
const {
266 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
269 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
270 return isRegOrInline(
271 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
274 bool isRegOrInlineImmWithInt32InputMods()
const {
275 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
278 bool isRegOrImmWithInt64InputMods()
const {
282 bool isRegOrImmWithFP16InputMods()
const {
286 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
288 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
291 bool isRegOrImmWithFP32InputMods()
const {
295 bool isRegOrImmWithFP64InputMods()
const {
299 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
300 return isRegOrInline(
301 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
304 bool isRegOrInlineImmWithFP32InputMods()
const {
305 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
308 bool isRegOrInlineImmWithFP64InputMods()
const {
309 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
312 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
314 bool isVRegWithFP32InputMods()
const {
315 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
318 bool isVRegWithFP64InputMods()
const {
319 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
322 bool isPackedFP16InputMods()
const {
326 bool isPackedVGPRFP32InputMods()
const {
330 bool isVReg()
const {
331 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
332 isRegClass(AMDGPU::VReg_64RegClassID) ||
333 isRegClass(AMDGPU::VReg_96RegClassID) ||
334 isRegClass(AMDGPU::VReg_128RegClassID) ||
335 isRegClass(AMDGPU::VReg_160RegClassID) ||
336 isRegClass(AMDGPU::VReg_192RegClassID) ||
337 isRegClass(AMDGPU::VReg_256RegClassID) ||
338 isRegClass(AMDGPU::VReg_512RegClassID) ||
339 isRegClass(AMDGPU::VReg_1024RegClassID);
342 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
344 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
346 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
348 bool isAV_LdSt_32_Align2_RegOp()
const {
349 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
350 isRegClass(AMDGPU::AGPR_32RegClassID);
353 bool isVRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
355 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
357 bool isSDWAOperand(MVT type)
const;
358 bool isSDWAFP16Operand()
const;
359 bool isSDWAFP32Operand()
const;
360 bool isSDWAInt16Operand()
const;
361 bool isSDWAInt32Operand()
const;
363 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
365 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
367 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
369 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
371 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
372 bool isDim()
const {
return isImmTy(ImmTyDim); }
373 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
374 bool isOff()
const {
return isImmTy(ImmTyOff); }
375 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
376 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
377 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
378 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
379 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
380 bool isFlatOffset()
const {
381 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
383 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
384 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
385 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
386 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
387 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
388 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
389 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
390 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
391 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
392 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
393 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
394 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
395 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
396 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
397 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
398 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
399 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
400 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
401 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
402 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
403 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
404 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
405 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
406 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
407 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
408 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
409 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
410 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
411 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
412 bool isDone()
const {
return isImmTy(ImmTyDone); }
413 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
415 bool isRegOrImm()
const {
return isReg() || isImm(); }
417 bool isRegClass(
unsigned RCID)
const;
421 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
422 return isRegOrInline(RCID, type) && !hasModifiers();
425 bool isSCSrcB16()
const {
426 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
429 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
431 bool isSCSrc_b32()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
435 bool isSCSrc_b64()
const {
436 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
439 bool isBoolReg()
const;
441 bool isSCSrcF16()
const {
442 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
445 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
447 bool isSCSrcF32()
const {
448 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
451 bool isSCSrcF64()
const {
452 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
455 bool isSSrc_b32()
const {
456 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
459 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
461 bool isSSrcV2B16()
const {
466 bool isSSrc_b64()
const {
469 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
470 (((
const MCTargetAsmParser *)AsmParser)
471 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
475 bool isSSrc_f32()
const {
476 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
479 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
481 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
483 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
485 bool isSSrcV2F16()
const {
490 bool isSSrcV2FP32()
const {
495 bool isSCSrcV2FP32()
const {
500 bool isSSrcV2INT32()
const {
505 bool isSCSrcV2INT32()
const {
507 return isSCSrc_b32();
510 bool isSSrcOrLds_b32()
const {
511 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
512 isLiteralImm(MVT::i32) || isExpr();
515 bool isVCSrc_b32()
const {
516 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
519 bool isVCSrc_b32_Lo256()
const {
520 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
523 bool isVCSrc_b64_Lo256()
const {
524 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
527 bool isVCSrc_b64()
const {
528 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
531 bool isVCSrcT_b16()
const {
532 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
535 bool isVCSrcTB16_Lo128()
const {
536 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
539 bool isVCSrcFake16B16_Lo128()
const {
540 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
543 bool isVCSrc_b16()
const {
544 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
547 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
549 bool isVCSrc_f32()
const {
550 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
553 bool isVCSrc_f64()
const {
554 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
557 bool isVCSrcTBF16()
const {
558 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
561 bool isVCSrcT_f16()
const {
562 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
565 bool isVCSrcT_bf16()
const {
566 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
569 bool isVCSrcTBF16_Lo128()
const {
570 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
573 bool isVCSrcTF16_Lo128()
const {
574 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
577 bool isVCSrcFake16BF16_Lo128()
const {
578 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
581 bool isVCSrcFake16F16_Lo128()
const {
582 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
585 bool isVCSrc_bf16()
const {
586 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
589 bool isVCSrc_f16()
const {
590 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
593 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
595 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
597 bool isVSrc_b32()
const {
598 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
601 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
603 bool isVSrc_v2b64()
const {
604 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
605 isLiteralImm(MVT::i64);
608 bool isVSrc_v2f64()
const {
609 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
610 isLiteralImm(MVT::f64);
613 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
615 bool isVSrcT_b16_Lo128()
const {
616 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
619 bool isVSrcFake16_b16_Lo128()
const {
620 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
623 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
625 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
627 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
629 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
631 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
633 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
635 bool isVSrc_f32()
const {
636 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
639 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
641 bool isVSrcT_bf16()
const {
642 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
645 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
647 bool isVSrcT_bf16_Lo128()
const {
648 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
651 bool isVSrcT_f16_Lo128()
const {
652 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
655 bool isVSrcFake16_bf16_Lo128()
const {
656 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
659 bool isVSrcFake16_f16_Lo128()
const {
660 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
663 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
665 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
667 bool isVSrc_v2bf16()
const {
668 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
671 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
673 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
675 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
677 bool isVISrcB32()
const {
678 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
681 bool isVISrcB16()
const {
682 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
685 bool isVISrcV2B16()
const {
return isVISrcB16(); }
687 bool isVISrcF32()
const {
688 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
691 bool isVISrcF16()
const {
692 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
695 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
697 bool isVISrc_64_bf16()
const {
698 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
701 bool isVISrc_64_f16()
const {
702 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
705 bool isVISrc_64_b32()
const {
706 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
709 bool isVISrc_64B64()
const {
710 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
713 bool isVISrc_64_f64()
const {
714 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
717 bool isVISrc_64V2FP32()
const {
718 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
721 bool isVISrc_64V2INT32()
const {
722 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
725 bool isVISrc_256_b32()
const {
726 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
729 bool isVISrc_256_f32()
const {
730 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
733 bool isVISrc_256B64()
const {
734 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
737 bool isVISrc_256_f64()
const {
738 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
741 bool isVISrc_512_f64()
const {
742 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
745 bool isVISrc_128B16()
const {
746 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
749 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
751 bool isVISrc_128_b32()
const {
752 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
755 bool isVISrc_128_f32()
const {
756 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
759 bool isVISrc_256V2FP32()
const {
760 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
763 bool isVISrc_256V2INT32()
const {
764 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
767 bool isVISrc_512_b32()
const {
768 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
771 bool isVISrc_512B16()
const {
772 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
775 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
777 bool isVISrc_512_f32()
const {
778 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
781 bool isVISrc_512F16()
const {
782 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
785 bool isVISrc_512V2F16()
const {
786 return isVISrc_512F16() || isVISrc_512_b32();
789 bool isVISrc_1024_b32()
const {
790 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
793 bool isVISrc_1024B16()
const {
794 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
797 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
799 bool isVISrc_1024_f32()
const {
800 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
803 bool isVISrc_1024F16()
const {
804 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
807 bool isVISrc_1024V2F16()
const {
808 return isVISrc_1024F16() || isVISrc_1024_b32();
811 bool isAISrcB32()
const {
812 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
815 bool isAISrcB16()
const {
816 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
819 bool isAISrcV2B16()
const {
return isAISrcB16(); }
821 bool isAISrcF32()
const {
822 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
825 bool isAISrcF16()
const {
826 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
829 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
831 bool isAISrc_64B64()
const {
832 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
835 bool isAISrc_64_f64()
const {
836 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
839 bool isAISrc_128_b32()
const {
840 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
843 bool isAISrc_128B16()
const {
844 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
847 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
849 bool isAISrc_128_f32()
const {
850 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
853 bool isAISrc_128F16()
const {
854 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
857 bool isAISrc_128V2F16()
const {
858 return isAISrc_128F16() || isAISrc_128_b32();
861 bool isVISrc_128_bf16()
const {
862 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
865 bool isVISrc_128_f16()
const {
866 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
869 bool isVISrc_128V2F16()
const {
870 return isVISrc_128_f16() || isVISrc_128_b32();
873 bool isAISrc_256B64()
const {
874 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
877 bool isAISrc_256_f64()
const {
878 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
881 bool isAISrc_512_b32()
const {
882 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
885 bool isAISrc_512B16()
const {
886 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
889 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
891 bool isAISrc_512_f32()
const {
892 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
895 bool isAISrc_512F16()
const {
896 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
899 bool isAISrc_512V2F16()
const {
900 return isAISrc_512F16() || isAISrc_512_b32();
903 bool isAISrc_1024_b32()
const {
904 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
907 bool isAISrc_1024B16()
const {
908 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
911 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
913 bool isAISrc_1024_f32()
const {
914 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
917 bool isAISrc_1024F16()
const {
918 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
921 bool isAISrc_1024V2F16()
const {
922 return isAISrc_1024F16() || isAISrc_1024_b32();
925 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
927 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
929 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
931 bool isMem()
const override {
return false; }
933 bool isExpr()
const {
return Kind == Expression; }
935 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
937 bool isSWaitCnt()
const;
938 bool isDepCtr()
const;
939 bool isSDelayALU()
const;
940 bool isHwreg()
const;
941 bool isSendMsg()
const;
942 bool isWaitEvent()
const;
943 bool isSplitBarrier()
const;
944 bool isSwizzle()
const;
945 bool isSMRDOffset8()
const;
946 bool isSMEMOffset()
const;
947 bool isSMRDLiteralOffset()
const;
949 bool isDPPCtrl()
const;
951 bool isGPRIdxMode()
const;
952 bool isS16Imm()
const;
953 bool isU16Imm()
const;
954 bool isEndpgm()
const;
956 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
957 return [
this,
P]() {
return P(*
this); };
962 return StringRef(Tok.Data, Tok.Length);
970 void setImm(int64_t Val) {
975 ImmTy getImmTy()
const {
980 MCRegister
getReg()
const override {
985 SMLoc getStartLoc()
const override {
return StartLoc; }
987 SMLoc getEndLoc()
const override {
return EndLoc; }
989 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
991 int getMCOpIdx()
const {
return MCOpIdx; }
993 Modifiers getModifiers()
const {
994 assert(isRegKind() || isImmTy(ImmTyNone));
995 return isRegKind() ?
Reg.Mods :
Imm.Mods;
998 void setModifiers(Modifiers Mods) {
999 assert(isRegKind() || isImmTy(ImmTyNone));
1006 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1008 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1010 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1012 bool isForcedLit()
const {
1013 return isImmLiteral() && getModifiers().isForcedLit();
1016 bool isForcedLit64()
const {
1017 return isImmLiteral() && getModifiers().isForcedLit64();
1022 void addImmOperands(MCInst &Inst,
unsigned N,
1023 bool ApplyModifiers =
true)
const;
1025 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1026 bool ApplyModifiers)
const;
1028 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1030 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1032 addRegOperands(Inst,
N);
1034 addImmOperands(Inst,
N);
1037 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1038 Modifiers Mods = getModifiers();
1041 addRegOperands(Inst,
N);
1043 addImmOperands(Inst,
N,
false);
1047 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1048 assert(!hasIntModifiers());
1049 addRegOrImmWithInputModsOperands(Inst,
N);
1052 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1053 assert(!hasFPModifiers());
1054 addRegOrImmWithInputModsOperands(Inst,
N);
1057 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1058 Modifiers Mods = getModifiers();
1061 addRegOperands(Inst,
N);
1064 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1065 assert(!hasIntModifiers());
1066 addRegWithInputModsOperands(Inst,
N);
1069 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1070 assert(!hasFPModifiers());
1071 addRegWithInputModsOperands(Inst,
N);
1074 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1077 case ImmTyNone: OS <<
"None";
break;
1078 case ImmTyGDS: OS <<
"GDS";
break;
1079 case ImmTyLDS: OS <<
"LDS";
break;
1080 case ImmTyOffen: OS <<
"Offen";
break;
1081 case ImmTyIdxen: OS <<
"Idxen";
break;
1082 case ImmTyAddr64: OS <<
"Addr64";
break;
1083 case ImmTyOffset: OS <<
"Offset";
break;
1084 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1085 case ImmTyOffset0: OS <<
"Offset0";
break;
1086 case ImmTyOffset1: OS <<
"Offset1";
break;
1087 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1088 case ImmTyCPol: OS <<
"CPol";
break;
1089 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1091 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1092 case ImmTyTFE: OS <<
"TFE";
break;
1093 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1094 case ImmTyD16: OS <<
"D16";
break;
1095 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1096 case ImmTyClamp: OS <<
"Clamp";
break;
1097 case ImmTyOModSI: OS <<
"OModSI";
break;
1098 case ImmTyDPP8: OS <<
"DPP8";
break;
1099 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1100 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1101 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1102 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1103 case ImmTyDppFI: OS <<
"DppFI";
break;
1104 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1105 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1106 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1107 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1108 case ImmTyDMask: OS <<
"DMask";
break;
1109 case ImmTyDim: OS <<
"Dim";
break;
1110 case ImmTyUNorm: OS <<
"UNorm";
break;
1111 case ImmTyDA: OS <<
"DA";
break;
1112 case ImmTyR128A16: OS <<
"R128A16";
break;
1113 case ImmTyA16: OS <<
"A16";
break;
1114 case ImmTyLWE: OS <<
"LWE";
break;
1115 case ImmTyOff: OS <<
"Off";
break;
1116 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1117 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1118 case ImmTyExpVM: OS <<
"ExpVM";
break;
1119 case ImmTyDone: OS <<
"Done";
break;
1120 case ImmTyRowEn: OS <<
"RowEn";
break;
1121 case ImmTyHwreg: OS <<
"Hwreg";
break;
1122 case ImmTySendMsg: OS <<
"SendMsg";
break;
1123 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1124 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1125 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1126 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1127 case ImmTyOpSel: OS <<
"OpSel";
break;
1128 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1129 case ImmTyNegLo: OS <<
"NegLo";
break;
1130 case ImmTyNegHi: OS <<
"NegHi";
break;
1131 case ImmTySwizzle: OS <<
"Swizzle";
break;
1132 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1133 case ImmTyHigh: OS <<
"High";
break;
1134 case ImmTyBLGP: OS <<
"BLGP";
break;
1135 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1136 case ImmTyABID: OS <<
"ABID";
break;
1137 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1138 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1139 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1140 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1141 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1142 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1143 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1144 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1145 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1146 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1147 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1148 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1149 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1150 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1151 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1152 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1157 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1161 <<
" mods: " <<
Reg.Mods <<
'>';
1165 if (getImmTy() != ImmTyNone) {
1167 printImmTy(OS, getImmTy());
1169 OS <<
" mods: " <<
Imm.Mods <<
'>';
1182 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1183 int64_t Val, SMLoc Loc,
1184 ImmTy
Type = ImmTyNone,
1185 bool IsFPImm =
false) {
1186 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1188 Op->Imm.IsFPImm = IsFPImm;
1190 Op->Imm.Mods = Modifiers();
1196 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1197 StringRef Str, SMLoc Loc,
1198 bool HasExplicitEncodingSize =
true) {
1199 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1200 Res->Tok.Data = Str.data();
1201 Res->Tok.Length = Str.size();
1202 Res->StartLoc = Loc;
1207 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1208 MCRegister
Reg, SMLoc S, SMLoc
E) {
1209 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1210 Op->Reg.RegNo =
Reg;
1211 Op->Reg.Mods = Modifiers();
1217 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1218 const class MCExpr *Expr, SMLoc S) {
1219 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1228 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1237#define GET_REGISTER_MATCHER
1238#include "AMDGPUGenAsmMatcher.inc"
1239#undef GET_REGISTER_MATCHER
1240#undef GET_SUBTARGET_FEATURE_NAME
1245class KernelScopeInfo {
1246 int SgprIndexUnusedMin = -1;
1247 int VgprIndexUnusedMin = -1;
1248 int AgprIndexUnusedMin = -1;
1252 void usesSgprAt(
int i) {
1253 if (i >= SgprIndexUnusedMin) {
1254 SgprIndexUnusedMin = ++i;
1257 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1263 void usesVgprAt(
int i) {
1264 if (i >= VgprIndexUnusedMin) {
1265 VgprIndexUnusedMin = ++i;
1268 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1270 VgprIndexUnusedMin);
1276 void usesAgprAt(
int i) {
1281 if (i >= AgprIndexUnusedMin) {
1282 AgprIndexUnusedMin = ++i;
1285 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1290 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1292 VgprIndexUnusedMin);
1299 KernelScopeInfo() =
default;
1303 MSTI = Ctx->getSubtargetInfo();
1305 usesSgprAt(SgprIndexUnusedMin = -1);
1306 usesVgprAt(VgprIndexUnusedMin = -1);
1308 usesAgprAt(AgprIndexUnusedMin = -1);
1312 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1313 unsigned RegWidth) {
1316 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1319 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1322 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1331 MCAsmParser &Parser;
1333 unsigned ForcedEncodingSize = 0;
1334 bool ForcedDPP =
false;
1335 bool ForcedSDWA =
false;
1336 KernelScopeInfo KernelScope;
1337 const unsigned HwMode;
1339 const AMDGPU::IsaVersion ISA;
1344#define GET_ASSEMBLER_HEADER
1345#include "AMDGPUGenAsmMatcher.inc"
1350 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1352 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1356 std::optional<AMDGPU::InfoSectionData> InfoData;
1363 bool TargetDirectiveEmitted =
false;
1372 SmallVector<unsigned> OpcodeStream;
1374 OpcodeStreamSymbols;
1375 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1378 void checkKernelPrologues();
1381 void createConstantSymbol(StringRef Id, int64_t Val);
1383 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1384 bool OutOfRangeError(SMRange
Range);
1400 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1401 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1402 std::optional<bool> EnableWavefrontSize32,
1403 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1404 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1405 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1406 bool ParseDirectiveAMDGCNTarget();
1407 bool ParseDirectiveAMDHSACodeObjectVersion();
1408 bool ParseDirectiveAMDHSAKernel();
1409 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1410 bool ParseDirectiveAMDKernelCodeT();
1412 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1413 bool ParseDirectiveAMDGPUHsaKernel();
1415 bool ParseDirectiveISAVersion();
1416 bool ParseDirectiveHSAMetadata();
1417 bool ParseDirectivePALMetadataBegin();
1418 bool ParseDirectivePALMetadata();
1419 bool ParseDirectiveAMDGPULDS();
1420 bool ParseDirectiveAMDGPUInfo();
1424 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1425 const char *AssemblerDirectiveEnd,
1426 std::string &CollectString);
1428 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1429 RegisterKind RegKind, MCRegister Reg1,
1430 RegisterKind RegKind1, SMLoc Loc);
1431 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1432 unsigned &RegNum,
unsigned &RegWidth,
1433 bool RestoreOnFailure =
false);
1434 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1435 unsigned &RegNum,
unsigned &RegWidth,
1436 SmallVectorImpl<AsmToken> &Tokens);
1437 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1439 SmallVectorImpl<AsmToken> &Tokens);
1440 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1442 SmallVectorImpl<AsmToken> &Tokens);
1443 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1445 SmallVectorImpl<AsmToken> &Tokens);
1446 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1447 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1448 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1451 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1452 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1453 void initializeGprCountSymbol(RegisterKind RegKind);
1454 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1460 OperandMode_Default,
1464 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1466 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1467 const MCInstrInfo &MII)
1468 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1469 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1474 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1476 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1477 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1478 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1479 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1481 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1482 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1483 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1485 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1486 initializeGprCountSymbol(IS_VGPR);
1487 initializeGprCountSymbol(IS_SGPR);
1492 createConstantSymbol(Symbol, Code);
1494 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1495 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1496 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1544 bool hasBVHRayTracingInsts()
const {
1545 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1550 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1552 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1554 bool hasInv2PiInlineImm()
const {
1555 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1558 bool has64BitLiterals()
const {
1559 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1562 bool hasFlatOffsets()
const {
1563 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1566 bool hasTrue16Insts()
const {
1567 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1571 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1574 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1576 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1578 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1580 bool hasPartialNSAEncoding()
const {
1581 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1584 bool hasGloballyAddressableScratch()
const {
1585 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1598 AMDGPUTargetStreamer &getTargetStreamer() {
1599 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1600 return static_cast<AMDGPUTargetStreamer &
>(TS);
1606 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1609 const MCRegisterInfo *getMRI()
const {
1613 const MCInstrInfo *getMII()
const {
return &MII; }
1617 const FeatureBitset &getFeatureBits()
const {
1618 return getSTI().getFeatureBits();
1621 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1622 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1623 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1625 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1626 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1627 bool isForcedDPP()
const {
return ForcedDPP; }
1628 bool isForcedSDWA()
const {
return ForcedSDWA; }
1629 ArrayRef<unsigned> getMatchedVariants()
const;
1630 StringRef getMatchedVariantName()
const;
1632 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1633 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1634 bool RestoreOnFailure);
1635 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1636 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1637 SMLoc &EndLoc)
override;
1638 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1639 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1640 unsigned Kind)
override;
1641 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1644 bool MatchingInlineAsm)
override;
1645 bool ParseDirective(AsmToken DirectiveID)
override;
1646 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1647 void onEndOfFile()
override;
1649 OperandMode
Mode = OperandMode_Default);
1650 StringRef parseMnemonicSuffix(StringRef Name);
1651 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1657 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1661 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1662 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1664 ParseStatus parseOperandArrayWithPrefix(
1666 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1667 bool (*ConvertResult)(int64_t &) =
nullptr);
1671 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1672 bool IgnoreNegative =
false);
1673 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1677 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1681 ArrayRef<const char *> Ids,
1685 ArrayRef<const char *> Ids,
1686 AMDGPUOperand::ImmTy
Type);
1689 bool isOperandModifier(
const AsmToken &Token,
1690 const AsmToken &NextToken)
const;
1691 bool isRegOrOperandModifier(
const AsmToken &Token,
1692 const AsmToken &NextToken)
const;
1693 bool isNamedOperandModifier(
const AsmToken &Token,
1694 const AsmToken &NextToken)
const;
1695 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1696 const AsmToken &NextToken)
const;
1697 bool parseSP3NegModifier();
1704 bool AllowImm =
true);
1706 bool AllowImm =
true);
1711 AMDGPUOperand::ImmTy ImmTy);
1716 AMDGPUOperand::ImmTy
Type);
1720 AMDGPUOperand::ImmTy
Type);
1724 AMDGPUOperand::ImmTy
Type);
1728 ParseStatus parseDfmtNfmt(int64_t &
Format);
1729 ParseStatus parseUfmt(int64_t &
Format);
1730 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1732 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1735 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1736 ParseStatus parseNumericFormat(int64_t &
Format);
1740 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1741 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1746 bool parseCnt(int64_t &IntVal);
1749 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1750 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1753 bool parseDelay(int64_t &Delay);
1759 struct OperandInfoTy {
1762 bool IsSymbolic =
false;
1763 bool IsDefined =
false;
1765 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1768 struct StructuredOpField : OperandInfoTy {
1772 bool IsDefined =
false;
1774 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1775 unsigned Width, int64_t
Default)
1776 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1777 virtual ~StructuredOpField() =
default;
1779 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1780 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1784 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1786 return Error(Parser,
"not supported on this GPU");
1788 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1796 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1797 OperandInfoTy &Stream);
1798 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1799 const OperandInfoTy &Stream);
1801 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1802 OperandInfoTy &Width);
1807 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1814 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1816 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1820 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1826 bool validateConstantBusLimitations(
const MCInst &Inst,
1828 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1831 bool tryVOPD(
const MCInst &Inst);
1832 bool tryVOPD3(
const MCInst &Inst);
1833 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1835 bool validateIntClampSupported(
const MCInst &Inst);
1836 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1837 bool validateMIMGGatherDMask(
const MCInst &Inst);
1839 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1841 bool validateMIMGD16(
const MCInst &Inst);
1843 bool validateTensorR128(
const MCInst &Inst);
1844 bool validateMIMGMSAA(
const MCInst &Inst);
1845 bool validateOpSel(
const MCInst &Inst);
1846 bool validateTrue16OpSel(
const MCInst &Inst);
1847 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1849 bool validateVccOperand(MCRegister
Reg)
const;
1854 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1855 bool validateVGPRAlign(
const MCInst &Inst)
const;
1859 bool validateDivScale(
const MCInst &Inst);
1864 const unsigned CPol);
1868 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1869 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1870 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1871 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1874 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1875 ArrayRef<unsigned> Variants);
1876 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1878 bool isId(
const StringRef Id)
const;
1879 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1881 StringRef getId()
const;
1882 bool trySkipId(
const StringRef Id);
1883 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1887 bool parseString(StringRef &Val,
1888 const StringRef ErrMsg =
"expected a string");
1889 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1895 StringRef getTokenStr()
const;
1896 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1898 SMLoc getLoc()
const;
1902 void onBeginOfFile()
override;
1906 void emitTargetDirective();
1907 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1919 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1920 const unsigned MaxVal,
const Twine &ErrMsg,
1922 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1923 const unsigned MinVal,
const unsigned MaxVal,
1924 const StringRef ErrMsg);
1926 bool parseSwizzleOffset(int64_t &
Imm);
1927 bool parseSwizzleMacro(int64_t &
Imm);
1928 bool parseSwizzleQuadPerm(int64_t &
Imm);
1929 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1930 bool parseSwizzleBroadcast(int64_t &
Imm);
1931 bool parseSwizzleSwap(int64_t &
Imm);
1932 bool parseSwizzleReverse(int64_t &
Imm);
1933 bool parseSwizzleFFT(int64_t &
Imm);
1934 bool parseSwizzleRotate(int64_t &
Imm);
1937 int64_t parseGPRIdxMacro();
1940 cvtMubufImpl(Inst,
Operands,
false);
1943 cvtMubufImpl(Inst,
Operands,
true);
1949 OptionalImmIndexMap &OptionalIdx);
1958 OptionalImmIndexMap &OptionalIdx);
1960 OptionalImmIndexMap &OptionalIdx);
1964 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1966 bool parseDimId(
unsigned &Encoding);
1968 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1972 int64_t parseDPPCtrlSel(StringRef Ctrl);
1973 int64_t parseDPPCtrlPerm();
1979 bool IsDPP8 =
false);
1985 AMDGPUOperand::ImmTy
Type);
1993 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1996 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1997 bool SkipSrcVcc =
false);
2106bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2116 if (!isImmTy(ImmTyNone)) {
2121 if (getModifiers().
Lit != LitModifier::None)
2131 if (type == MVT::f64 || type == MVT::i64) {
2133 AsmParser->hasInv2PiInlineImm());
2136 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2155 APFloat::rmNearestTiesToEven, &Lost);
2162 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2164 AsmParser->hasInv2PiInlineImm());
2169 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2170 AsmParser->hasInv2PiInlineImm());
2174 if (type == MVT::f64 || type == MVT::i64) {
2176 AsmParser->hasInv2PiInlineImm());
2185 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2186 AsmParser->hasInv2PiInlineImm());
2190 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2191 AsmParser->hasInv2PiInlineImm());
2194bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2196 if (!isImmTy(ImmTyNone)) {
2201 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2206 if (type == MVT::f64 && hasFPModifiers()) {
2226 if (type == MVT::f64) {
2231 if (type == MVT::i64) {
2244 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2245 : (type == MVT::v2i16) ? MVT::f32
2246 : (type == MVT::v2f32) ? MVT::f32
2249 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2253bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2254 return isRegKind() &&
2255 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2258bool AMDGPUOperand::isVRegWithInputMods()
const {
2259 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2261 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2262 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2265template <
bool IsFake16>
2266bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2267 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2268 : AMDGPU::VGPR_16_Lo128RegClassID);
2271template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2272 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2273 : AMDGPU::VGPR_16RegClassID);
2276bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2277 if (AsmParser->isVI())
2279 if (AsmParser->isGFX9Plus())
2280 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2284bool AMDGPUOperand::isSDWAFP16Operand()
const {
2285 return isSDWAOperand(MVT::f16);
2288bool AMDGPUOperand::isSDWAFP32Operand()
const {
2289 return isSDWAOperand(MVT::f32);
2292bool AMDGPUOperand::isSDWAInt16Operand()
const {
2293 return isSDWAOperand(MVT::i16);
2296bool AMDGPUOperand::isSDWAInt32Operand()
const {
2297 return isSDWAOperand(MVT::i32);
2300bool AMDGPUOperand::isBoolReg()
const {
2301 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2302 (AsmParser->isWave32() && isSCSrc_b32()));
2306 unsigned Size)
const {
2307 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2322void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2323 bool ApplyModifiers)
const {
2333 addLiteralImmOperand(Inst,
Imm.Val,
2334 ApplyModifiers & isImmTy(ImmTyNone) &&
2335 Imm.Mods.hasFPModifiers());
2337 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2342void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2343 bool ApplyModifiers)
const {
2344 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2349 if (ApplyModifiers) {
2351 const unsigned Size =
2353 Val = applyInputFPModifiers(Val,
Size);
2357 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2359 bool CanUse64BitLiterals =
2362 MCContext &Ctx = AsmParser->getContext();
2373 if (
Lit == LitModifier::None &&
2375 AsmParser->hasInv2PiInlineImm())) {
2383 bool HasMandatoryLiteral =
2386 if (
Literal.getLoBits(32) != 0 &&
2387 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2388 !HasMandatoryLiteral) {
2389 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2391 "Can't encode literal as exact 64-bit floating-point operand. "
2392 "Low 32-bits will be set to zero");
2393 Val &= 0xffffffff00000000u;
2399 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2405 Lit = LitModifier::Lit64;
2406 }
else if (
Lit == LitModifier::Lit) {
2420 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2422 Lit = LitModifier::Lit64;
2429 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2430 Literal == 0x3fc45f306725feed) {
2469 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2476 if (
Lit != LitModifier::None) {
2507 if (
Lit == LitModifier::None &&
2517 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2525 if (
Lit == LitModifier::None &&
2533 if (!AsmParser->has64BitLiterals()) {
2534 Val =
static_cast<uint64_t>(Val) << 32;
2541 if (
Lit == LitModifier::Lit ||
2543 Val =
static_cast<uint64_t>(Val) << 32;
2547 if (
Lit == LitModifier::Lit)
2573 if (
Lit != LitModifier::None) {
2581void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2587bool AMDGPUOperand::isInlineValue()
const {
2595void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2606 if (Is == IS_VGPR) {
2611 return AMDGPU::VGPR_32RegClassID;
2613 return AMDGPU::VReg_64RegClassID;
2615 return AMDGPU::VReg_96RegClassID;
2617 return AMDGPU::VReg_128RegClassID;
2619 return AMDGPU::VReg_160RegClassID;
2621 return AMDGPU::VReg_192RegClassID;
2623 return AMDGPU::VReg_224RegClassID;
2625 return AMDGPU::VReg_256RegClassID;
2627 return AMDGPU::VReg_288RegClassID;
2629 return AMDGPU::VReg_320RegClassID;
2631 return AMDGPU::VReg_352RegClassID;
2633 return AMDGPU::VReg_384RegClassID;
2635 return AMDGPU::VReg_512RegClassID;
2637 return AMDGPU::VReg_1024RegClassID;
2639 }
else if (Is == IS_TTMP) {
2644 return AMDGPU::TTMP_32RegClassID;
2646 return AMDGPU::TTMP_64RegClassID;
2648 return AMDGPU::TTMP_128RegClassID;
2650 return AMDGPU::TTMP_256RegClassID;
2652 return AMDGPU::TTMP_512RegClassID;
2654 }
else if (Is == IS_SGPR) {
2659 return AMDGPU::SGPR_32RegClassID;
2661 return AMDGPU::SGPR_64RegClassID;
2663 return AMDGPU::SGPR_96RegClassID;
2665 return AMDGPU::SGPR_128RegClassID;
2667 return AMDGPU::SGPR_160RegClassID;
2669 return AMDGPU::SGPR_192RegClassID;
2671 return AMDGPU::SGPR_224RegClassID;
2673 return AMDGPU::SGPR_256RegClassID;
2675 return AMDGPU::SGPR_288RegClassID;
2677 return AMDGPU::SGPR_320RegClassID;
2679 return AMDGPU::SGPR_352RegClassID;
2681 return AMDGPU::SGPR_384RegClassID;
2683 return AMDGPU::SGPR_512RegClassID;
2685 }
else if (Is == IS_AGPR) {
2690 return AMDGPU::AGPR_32RegClassID;
2692 return AMDGPU::AReg_64RegClassID;
2694 return AMDGPU::AReg_96RegClassID;
2696 return AMDGPU::AReg_128RegClassID;
2698 return AMDGPU::AReg_160RegClassID;
2700 return AMDGPU::AReg_192RegClassID;
2702 return AMDGPU::AReg_224RegClassID;
2704 return AMDGPU::AReg_256RegClassID;
2706 return AMDGPU::AReg_288RegClassID;
2708 return AMDGPU::AReg_320RegClassID;
2710 return AMDGPU::AReg_352RegClassID;
2712 return AMDGPU::AReg_384RegClassID;
2714 return AMDGPU::AReg_512RegClassID;
2716 return AMDGPU::AReg_1024RegClassID;
2724 .
Case(
"exec", AMDGPU::EXEC)
2725 .
Case(
"vcc", AMDGPU::VCC)
2726 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2727 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2728 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2730 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2732 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2734 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2736 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2737 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2738 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2740 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2742 .
Case(
"m0", AMDGPU::M0)
2743 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2745 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2747 .
Case(
"scc", AMDGPU::SRC_SCC)
2748 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2749 .
Case(
"tba", AMDGPU::TBA)
2750 .
Case(
"tma", AMDGPU::TMA)
2751 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2752 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2753 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2754 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2755 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2756 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2757 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2758 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2759 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2760 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2761 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2762 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2763 .
Case(
"pc", AMDGPU::PC_REG)
2764 .
Case(
"null", AMDGPU::SGPR_NULL)
2768bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2769 SMLoc &EndLoc,
bool RestoreOnFailure) {
2770 auto R = parseRegister();
2774 RegNo =
R->getReg();
2775 StartLoc =
R->getStartLoc();
2776 EndLoc =
R->getEndLoc();
2780bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2782 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2785ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2787 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2788 bool PendingErrors = getParser().hasPendingError();
2789 getParser().clearPendingErrors();
2797bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2798 RegisterKind RegKind,
2800 RegisterKind RegKind1, SMLoc Loc) {
2802 if (RegKind == IS_SGPR) {
2803 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2804 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2805 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2811 if (RegKind != RegKind1) {
2812 Error(Loc,
"registers in a list must be of the same kind");
2818 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2823 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2824 Reg = AMDGPU::FLAT_SCR;
2828 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2829 Reg = AMDGPU::XNACK_MASK;
2833 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2838 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2843 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2848 Error(Loc,
"register does not fit in the list");
2854 if (Reg1 !=
Reg + RegWidth / 32) {
2855 Error(Loc,
"registers in a list must have consecutive indices");
2871 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2872 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2876 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2882 if (Str.starts_with(
Reg.Name))
2888 return !Str.getAsInteger(10, Num);
2891bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2892 const AsmToken &NextToken)
const {
2907 StringRef RegSuffix = Str.substr(
RegName.size());
2908 if (!RegSuffix.
empty()) {
2925bool AMDGPUAsmParser::isRegister() {
2926 return isRegister(
getToken(), peekToken());
2929MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2930 unsigned SubReg,
unsigned RegWidth,
2934 unsigned AlignSize = 1;
2935 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2941 if (RegNum % AlignSize != 0) {
2942 Error(Loc,
"invalid register alignment");
2943 return MCRegister();
2946 unsigned RegIdx = RegNum / AlignSize;
2949 Error(Loc,
"invalid or unsupported register size");
2950 return MCRegister();
2954 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2955 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2956 Error(Loc,
"register index is out of range");
2957 return AMDGPU::NoRegister;
2960 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2961 Error(Loc,
"register index is out of range");
2962 return MCRegister();
2978bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2980 int64_t RegLo, RegHi;
2984 SMLoc FirstIdxLoc = getLoc();
2991 SecondIdxLoc = getLoc();
3002 Error(FirstIdxLoc,
"invalid register index");
3007 Error(SecondIdxLoc,
"invalid register index");
3011 if (RegLo > RegHi) {
3012 Error(FirstIdxLoc,
"first register index should not exceed second index");
3016 if (RegHi == RegLo) {
3017 StringRef RegSuffix = getTokenStr();
3018 if (RegSuffix ==
".l") {
3019 SubReg = AMDGPU::lo16;
3021 }
else if (RegSuffix ==
".h") {
3022 SubReg = AMDGPU::hi16;
3027 Num =
static_cast<unsigned>(RegLo);
3028 RegWidth = 32 * ((RegHi - RegLo) + 1);
3033MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3036 SmallVectorImpl<AsmToken> &Tokens) {
3042 RegKind = IS_SPECIAL;
3049MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3052 SmallVectorImpl<AsmToken> &Tokens) {
3054 StringRef
RegName = getTokenStr();
3055 auto Loc = getLoc();
3059 Error(Loc,
"invalid register name");
3060 return MCRegister();
3068 unsigned SubReg = NoSubRegister;
3069 bool IsRange =
false;
3070 if (!RegSuffix.
empty()) {
3072 SubReg = AMDGPU::lo16;
3074 SubReg = AMDGPU::hi16;
3078 Error(Loc,
"invalid register index");
3079 return MCRegister();
3085 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3086 return MCRegister();
3090 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3091 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3092 if (RegKind == IS_SGPR && IsRange
3093 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3094 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3095 Error(Loc,
"register index is out of range");
3096 return MCRegister();
3102MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3103 unsigned &RegNum,
unsigned &RegWidth,
3104 SmallVectorImpl<AsmToken> &Tokens) {
3106 auto ListLoc = getLoc();
3109 "expected a register or a list of registers")) {
3110 return MCRegister();
3115 auto Loc = getLoc();
3116 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3117 return MCRegister();
3118 if (RegWidth != 32) {
3119 Error(Loc,
"expected a single 32-bit register");
3120 return MCRegister();
3124 RegisterKind NextRegKind;
3126 unsigned NextRegNum, NextRegWidth;
3129 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3131 return MCRegister();
3133 if (NextRegWidth != 32) {
3134 Error(Loc,
"expected a single 32-bit register");
3135 return MCRegister();
3137 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3139 return MCRegister();
3143 "expected a comma or a closing square bracket")) {
3144 return MCRegister();
3148 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3153bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3154 MCRegister &
Reg,
unsigned &RegNum,
3156 SmallVectorImpl<AsmToken> &Tokens) {
3157 auto Loc = getLoc();
3161 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3163 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3165 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3170 assert(Parser.hasPendingError());
3174 if (!subtargetHasRegister(*
TRI,
Reg)) {
3175 if (
Reg == AMDGPU::SGPR_NULL) {
3176 Error(Loc,
"'null' operand is not supported on this GPU");
3179 " register not available on this GPU");
3187bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3188 MCRegister &
Reg,
unsigned &RegNum,
3190 bool RestoreOnFailure ) {
3194 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3195 if (RestoreOnFailure) {
3196 while (!Tokens.
empty()) {
3205std::optional<StringRef>
3206AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3209 return StringRef(
".amdgcn.next_free_vgpr");
3211 return StringRef(
".amdgcn.next_free_sgpr");
3213 return std::nullopt;
3217void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3218 auto SymbolName = getGprCountSymbolName(RegKind);
3219 assert(SymbolName &&
"initializing invalid register kind");
3225bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3226 unsigned DwordRegIndex,
3227 unsigned RegWidth) {
3232 auto SymbolName = getGprCountSymbolName(RegKind);
3237 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3241 return !
Error(getLoc(),
3242 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3246 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3248 if (OldCount <= NewMax)
3254std::unique_ptr<AMDGPUOperand>
3255AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3257 SMLoc StartLoc = Tok.getLoc();
3258 SMLoc EndLoc = Tok.getEndLoc();
3259 RegisterKind RegKind;
3261 unsigned RegNum, RegWidth;
3263 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3267 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3270 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3271 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3278 if (isRegister() || isModifier())
3281 if (
Lit == LitModifier::None) {
3282 if (trySkipId(
"lit"))
3283 Lit = LitModifier::Lit;
3284 else if (trySkipId(
"lit64"))
3285 Lit = LitModifier::Lit64;
3287 if (
Lit != LitModifier::None) {
3290 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3299 const auto &NextTok = peekToken();
3302 bool Negate =
false;
3310 AMDGPUOperand::Modifiers Mods;
3318 StringRef Num = getTokenStr();
3321 APFloat RealVal(APFloat::IEEEdouble());
3322 auto roundMode = APFloat::rmNearestTiesToEven;
3323 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3326 RealVal.changeSign();
3329 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3330 S, AMDGPUOperand::ImmTyNone,
true));
3331 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3332 Op.setModifiers(Mods);
3341 if (HasSP3AbsModifier) {
3350 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3353 if (Parser.parseExpression(Expr))
3357 if (Expr->evaluateAsAbsolute(IntVal)) {
3359 return Error(S,
"literal value out of range");
3360 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3361 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3362 Op.setModifiers(Mods);
3364 if (
Lit != LitModifier::None)
3366 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3379 if (
auto R = parseRegister()) {
3389 ParseStatus Res = parseReg(
Operands);
3397bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3398 const AsmToken &NextToken)
const {
3401 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3406bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3407 const AsmToken &NextToken)
const {
3411bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3412 const AsmToken &NextToken)
const {
3413 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3416bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3417 const AsmToken &NextToken)
const {
3418 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3434bool AMDGPUAsmParser::isModifier() {
3437 AsmToken NextToken[2];
3438 peekTokens(NextToken);
3440 return isOperandModifier(Tok, NextToken[0]) ||
3442 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3443 isOpcodeModifierWithVal(Tok, NextToken[0]);
3468bool AMDGPUAsmParser::parseSP3NegModifier() {
3470 AsmToken NextToken[2];
3471 peekTokens(NextToken);
3474 (isRegister(NextToken[0], NextToken[1]) ||
3492 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3494 SP3Neg = parseSP3NegModifier();
3497 Neg = trySkipId(
"neg");
3499 return Error(Loc,
"expected register or immediate");
3503 Abs = trySkipId(
"abs");
3508 if (trySkipId(
"lit")) {
3509 Lit = LitModifier::Lit;
3512 }
else if (trySkipId(
"lit64")) {
3513 Lit = LitModifier::Lit64;
3516 if (!has64BitLiterals())
3517 return Error(Loc,
"lit64 is not supported on this GPU");
3523 return Error(Loc,
"expected register or immediate");
3532 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3536 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3537 Error(Loc,
"expected immediate with lit modifier");
3539 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3545 if (
Lit != LitModifier::None &&
3549 AMDGPUOperand::Modifiers Mods;
3550 Mods.Abs = Abs || SP3Abs;
3551 Mods.Neg = Neg || SP3Neg;
3554 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3555 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3557 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3558 Op.setModifiers(Mods);
3566 bool Sext = trySkipId(
"sext");
3567 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3582 AMDGPUOperand::Modifiers Mods;
3585 if (Mods.hasIntModifiers()) {
3586 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3588 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3589 Op.setModifiers(Mods);
3596 return parseRegOrImmWithFPInputMods(
Operands,
false);
3600 return parseRegOrImmWithIntInputMods(
Operands,
false);
3604 auto Loc = getLoc();
3605 if (trySkipId(
"off")) {
3607 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3614 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3623unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3628 return Match_InvalidOperand;
3630 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3631 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3634 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3636 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3637 return Match_InvalidOperand;
3645 if (tryAnotherVOPDEncoding(Inst))
3646 return Match_InvalidOperand;
3648 return Match_Success;
3652 static const unsigned Variants[] = {
3661ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3662 if (isForcedDPP() && isForcedVOP3()) {
3666 if (getForcedEncodingSize() == 32) {
3671 if (isForcedVOP3()) {
3676 if (isForcedSDWA()) {
3682 if (isForcedDPP()) {
3690StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3691 if (isForcedDPP() && isForcedVOP3())
3694 if (getForcedEncodingSize() == 32)
3710AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3714 case AMDGPU::FLAT_SCR:
3716 case AMDGPU::VCC_LO:
3717 case AMDGPU::VCC_HI:
3724 return MCRegister();
3731bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3732 unsigned OpIdx)
const {
3740 const MCOperand &MO = Inst.
getOperand(OpIdx);
3789unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3795 case AMDGPU::V_LSHLREV_B64_e64:
3796 case AMDGPU::V_LSHLREV_B64_gfx10:
3797 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3798 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3799 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3800 case AMDGPU::V_LSHRREV_B64_e64:
3801 case AMDGPU::V_LSHRREV_B64_gfx10:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3803 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3804 case AMDGPU::V_ASHRREV_I64_e64:
3805 case AMDGPU::V_ASHRREV_I64_gfx10:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3807 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3808 case AMDGPU::V_LSHL_B64_e64:
3809 case AMDGPU::V_LSHR_B64_e64:
3810 case AMDGPU::V_ASHR_I64_e64:
3823 bool AddMandatoryLiterals =
false) {
3826 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3830 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3832 return {getNamedOperandIdx(Opcode, OpName::src0X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3834 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3835 getNamedOperandIdx(Opcode, OpName::src0Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3837 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3842 return {getNamedOperandIdx(Opcode, OpName::src0),
3843 getNamedOperandIdx(Opcode, OpName::src1),
3844 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3847bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3848 const MCOperand &MO = Inst.
getOperand(OpIdx);
3850 return !isInlineConstant(Inst, OpIdx);
3857 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3868 const unsigned Opcode = Inst.
getOpcode();
3869 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3872 if (!LaneSelOp.
isReg())
3875 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3878bool AMDGPUAsmParser::validateConstantBusLimitations(
3880 const unsigned Opcode = Inst.
getOpcode();
3881 const MCInstrDesc &
Desc = MII.
get(Opcode);
3882 MCRegister LastSGPR;
3883 unsigned ConstantBusUseCount = 0;
3884 unsigned NumLiterals = 0;
3885 unsigned LiteralSize;
3901 SmallDenseSet<MCRegister> SGPRsUsed;
3902 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3904 SGPRsUsed.
insert(SGPRUsed);
3905 ++ConstantBusUseCount;
3910 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3912 for (
int OpIdx : OpIndices) {
3916 const MCOperand &MO = Inst.
getOperand(OpIdx);
3917 if (usesConstantBus(Inst, OpIdx)) {
3926 if (SGPRsUsed.
insert(LastSGPR).second) {
3927 ++ConstantBusUseCount;
3947 if (NumLiterals == 0) {
3950 }
else if (LiteralSize !=
Size) {
3956 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3958 "invalid operand (violates constant bus restrictions)");
3965std::optional<unsigned>
3966AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3968 const unsigned Opcode = Inst.
getOpcode();
3974 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3975 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3989 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3993 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3994 int I = getNamedOperandIdx(Opcode, OpName);
3998 int64_t
Imm =
Op.getImm();
4004 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4005 OpName::vsrc2Y, OpName::imm}) {
4006 int I = getNamedOperandIdx(Opcode, OpName);
4016 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4017 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4019 return InvalidCompOprIdx;
4022bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4029 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4030 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4031 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4033 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4037 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4038 if (!InvalidCompOprIdx.has_value())
4041 auto CompOprIdx = *InvalidCompOprIdx;
4044 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4045 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4048 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4049 if (CompOprIdx == VOPD::Component::DST) {
4051 Error(Loc,
"dst registers must be distinct");
4053 Error(Loc,
"one dst register must be even and the other odd");
4055 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4056 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4057 " operands must use different VGPR banks");
4065bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4067 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4068 if (!InvalidCompOprIdx.has_value())
4072 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4073 if (InvalidCompOprIdx.has_value()) {
4078 if (*InvalidCompOprIdx == VOPD::Component::DST)
4091bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4092 const unsigned Opcode = Inst.
getOpcode();
4107 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4108 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4109 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4110 int I = getNamedOperandIdx(Opcode, OpName);
4117 return !tryVOPD3(Inst);
4122bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4127 return tryVOPD(Inst);
4128 return tryVOPD3(Inst);
4131bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4136 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4144bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4152 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4153 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4154 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4159 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4160 hasBVHRayTracingInsts())
4163 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4164 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4169 bool IsPackedD16 =
false;
4172 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4173 IsPackedD16 = D16Idx >= 0;
4175 DataSize = (DataSize + 1) / 2;
4178 if ((VDataSize / 4) == DataSize + TFESize)
4183 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4185 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4187 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4191bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4200 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4202 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4203 AMDGPU::OpName RSrcOpName =
4205 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4206 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4207 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4211 assert(SrsrcIdx > VAddr0Idx);
4214 if (BaseOpcode->
BVH) {
4215 if (IsA16 == BaseOpcode->
A16)
4217 Error(IDLoc,
"image address size does not match a16");
4223 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4224 unsigned ActualAddrSize =
4225 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4227 unsigned ExpectedAddrSize =
4231 if (hasPartialNSAEncoding() &&
4233 int VAddrLastIdx = SrsrcIdx - 1;
4234 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4236 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4239 if (ExpectedAddrSize > 12)
4240 ExpectedAddrSize = 16;
4245 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4249 if (ActualAddrSize == ExpectedAddrSize)
4252 Error(IDLoc,
"image address size does not match dim and a16");
4256bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4263 if (!
Desc.mayLoad() || !
Desc.mayStore())
4266 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4273 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4276bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4283 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4291 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4294bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4308 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4309 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4316bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4323 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4326 if (!BaseOpcode->
MSAA)
4329 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4335 return DimInfo->
MSAA;
4340 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4342 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4352bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4360 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4363 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4371 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4375bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4380 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4383 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4386 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4394 "source operand must be either a VGPR or an inline constant");
4401bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4406 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4409 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4413 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4415 "inline constants are not allowed for this operand");
4422bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4430 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4431 if (BlgpIdx != -1) {
4432 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4433 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4443 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4445 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4450 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4452 "wrong register tuple size for blgp value " + Twine(BLGP));
4460 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4464 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4468 MCRegister Src2Reg = Src2.
getReg();
4470 if (Src2Reg == DstReg)
4475 .getSizeInBits() <= 128)
4478 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4480 "source 2 operand must not partially overlap with dst");
4487bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4491 case V_DIV_SCALE_F32_gfx6_gfx7:
4492 case V_DIV_SCALE_F32_vi:
4493 case V_DIV_SCALE_F32_gfx10:
4494 case V_DIV_SCALE_F64_gfx6_gfx7:
4495 case V_DIV_SCALE_F64_vi:
4496 case V_DIV_SCALE_F64_gfx10:
4503 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4504 AMDGPU::OpName::src2_modifiers}) {
4515bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4522 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4531bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4537 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4544 case AMDGPU::V_SUBREV_F32_e32:
4545 case AMDGPU::V_SUBREV_F32_e64:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4547 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4548 case AMDGPU::V_SUBREV_F32_e32_vi:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4550 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4551 case AMDGPU::V_SUBREV_F32_e64_vi:
4553 case AMDGPU::V_SUBREV_CO_U32_e32:
4554 case AMDGPU::V_SUBREV_CO_U32_e64:
4555 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4556 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4558 case AMDGPU::V_SUBBREV_U32_e32:
4559 case AMDGPU::V_SUBBREV_U32_e64:
4560 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4561 case AMDGPU::V_SUBBREV_U32_e32_vi:
4562 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4563 case AMDGPU::V_SUBBREV_U32_e64_vi:
4565 case AMDGPU::V_SUBREV_U32_e32:
4566 case AMDGPU::V_SUBREV_U32_e64:
4567 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4568 case AMDGPU::V_SUBREV_U32_e32_vi:
4569 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4570 case AMDGPU::V_SUBREV_U32_e64_vi:
4572 case AMDGPU::V_SUBREV_F16_e32:
4573 case AMDGPU::V_SUBREV_F16_e64:
4574 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4575 case AMDGPU::V_SUBREV_F16_e32_vi:
4576 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4577 case AMDGPU::V_SUBREV_F16_e64_vi:
4579 case AMDGPU::V_SUBREV_U16_e32:
4580 case AMDGPU::V_SUBREV_U16_e64:
4581 case AMDGPU::V_SUBREV_U16_e32_vi:
4582 case AMDGPU::V_SUBREV_U16_e64_vi:
4584 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4586 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4589 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4591 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4592 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4595 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4597 case AMDGPU::V_LSHRREV_B32_e32:
4598 case AMDGPU::V_LSHRREV_B32_e64:
4599 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4601 case AMDGPU::V_LSHRREV_B32_e32_vi:
4602 case AMDGPU::V_LSHRREV_B32_e64_vi:
4603 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4604 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4606 case AMDGPU::V_ASHRREV_I32_e32:
4607 case AMDGPU::V_ASHRREV_I32_e64:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4609 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4610 case AMDGPU::V_ASHRREV_I32_e32_vi:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4612 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4613 case AMDGPU::V_ASHRREV_I32_e64_vi:
4615 case AMDGPU::V_LSHLREV_B32_e32:
4616 case AMDGPU::V_LSHLREV_B32_e64:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4618 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4619 case AMDGPU::V_LSHLREV_B32_e32_vi:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4621 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4622 case AMDGPU::V_LSHLREV_B32_e64_vi:
4624 case AMDGPU::V_LSHLREV_B16_e32:
4625 case AMDGPU::V_LSHLREV_B16_e64:
4626 case AMDGPU::V_LSHLREV_B16_e32_vi:
4627 case AMDGPU::V_LSHLREV_B16_e64_vi:
4628 case AMDGPU::V_LSHLREV_B16_gfx10:
4630 case AMDGPU::V_LSHRREV_B16_e32:
4631 case AMDGPU::V_LSHRREV_B16_e64:
4632 case AMDGPU::V_LSHRREV_B16_e32_vi:
4633 case AMDGPU::V_LSHRREV_B16_e64_vi:
4634 case AMDGPU::V_LSHRREV_B16_gfx10:
4636 case AMDGPU::V_ASHRREV_I16_e32:
4637 case AMDGPU::V_ASHRREV_I16_e64:
4638 case AMDGPU::V_ASHRREV_I16_e32_vi:
4639 case AMDGPU::V_ASHRREV_I16_e64_vi:
4640 case AMDGPU::V_ASHRREV_I16_gfx10:
4642 case AMDGPU::V_LSHLREV_B64_e64:
4643 case AMDGPU::V_LSHLREV_B64_gfx10:
4644 case AMDGPU::V_LSHLREV_B64_vi:
4646 case AMDGPU::V_LSHRREV_B64_e64:
4647 case AMDGPU::V_LSHRREV_B64_gfx10:
4648 case AMDGPU::V_LSHRREV_B64_vi:
4650 case AMDGPU::V_ASHRREV_I64_e64:
4651 case AMDGPU::V_ASHRREV_I64_gfx10:
4652 case AMDGPU::V_ASHRREV_I64_vi:
4654 case AMDGPU::V_PK_LSHLREV_B16:
4655 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4656 case AMDGPU::V_PK_LSHLREV_B16_vi:
4658 case AMDGPU::V_PK_LSHRREV_B16:
4659 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4660 case AMDGPU::V_PK_LSHRREV_B16_vi:
4661 case AMDGPU::V_PK_ASHRREV_I16:
4662 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4663 case AMDGPU::V_PK_ASHRREV_I16_vi:
4670bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4672 const unsigned Opcode = Inst.
getOpcode();
4681 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4682 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4686 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4690 "lds_direct is not supported on this GPU");
4696 "lds_direct cannot be used with this instruction");
4700 if (SrcName != OpName::src0) {
4702 "lds_direct may be used as src0 only");
4712 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4713 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4714 if (
Op.isFlatOffset())
4715 return Op.getStartLoc();
4720bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4723 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4728 return validateFlatOffset(Inst,
Operands);
4731 return validateSMEMOffset(Inst,
Operands);
4736 const unsigned OffsetSize = 24;
4737 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4739 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4740 "-bit unsigned offset for buffer ops");
4744 const unsigned OffsetSize = 16;
4745 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4747 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4754bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4760 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4764 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4766 "flat offset modifier is not supported on this GPU");
4773 bool AllowNegative =
4775 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4777 Twine(
"expected a ") +
4778 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4779 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4788 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4789 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4790 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4791 return Op.getStartLoc();
4796bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4805 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4821 ?
"expected a 23-bit unsigned offset for buffer ops"
4822 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4823 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4824 :
"expected a 21-bit signed offset");
4829bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4832 const MCInstrDesc &
Desc = MII.
get(Opcode);
4836 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4837 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4839 const int OpIndices[] = {Src0Idx, Src1Idx};
4841 unsigned NumExprs = 0;
4842 unsigned NumLiterals = 0;
4845 for (
int OpIdx : OpIndices) {
4849 const MCOperand &MO = Inst.
getOperand(OpIdx);
4853 std::optional<int64_t>
Imm;
4856 }
else if (MO.
isExpr()) {
4865 if (!
Imm.has_value()) {
4867 }
else if (!isInlineConstant(Inst, OpIdx)) {
4871 if (NumLiterals == 0 || LiteralValue !=
Value) {
4879 if (NumLiterals + NumExprs <= 1)
4883 "only one unique literal operand is allowed");
4887bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4890 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4899 if (OpSelIdx != -1) {
4903 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4904 if (OpSelHiIdx != -1) {
4913 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4923 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4924 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4925 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4926 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4928 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4929 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4935 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4937 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4947 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4948 if (Src2Idx != -1) {
4949 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4959bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4960 if (!hasTrue16Insts())
4962 const MCRegisterInfo *MRI = getMRI();
4964 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4970 if (OpSelOpValue == 0)
4972 unsigned OpCount = 0;
4973 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4974 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4975 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4982 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4983 if (OpSelOpIsHi != VGPRSuffixIsHi)
4992bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4993 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5005 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5016 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5017 AMDGPU::OpName::src1_modifiers,
5018 AMDGPU::OpName::src2_modifiers};
5020 for (
unsigned i = 0; i < 3; ++i) {
5030bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5033 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5034 if (DppCtrlIdx >= 0) {
5041 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5042 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5043 :
"DP ALU dpp only supports row_newbcast");
5048 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5049 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5052 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5054 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5058 "invalid operand for instruction");
5063 "src1 immediate operand invalid for instruction");
5073bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5074 return (
Reg == AMDGPU::VCC && isWave64()) ||
5075 (
Reg == AMDGPU::VCC_LO && isWave32());
5079bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5082 const MCInstrDesc &
Desc = MII.
get(Opcode);
5083 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5090 std::optional<unsigned> LiteralOpIdx;
5093 for (
int OpIdx : OpIndices) {
5097 const MCOperand &MO = Inst.
getOperand(OpIdx);
5103 std::optional<int64_t>
Imm;
5109 bool IsAnotherLiteral =
false;
5110 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5111 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5112 if (!
Imm.has_value()) {
5114 IsAnotherLiteral =
true;
5115 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5120 HasMandatoryLiteral);
5121 unsigned OpTy =
Desc.operands()[OpIdx].OperandType;
5131 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5132 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5134 "invalid operand for instruction");
5139 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5140 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5142 "invalid operand for instruction");
5146 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5153 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5154 !getFeatureBits()[FeatureVOP3Literal]) {
5156 "literal operands are not supported");
5160 if (LiteralOpIdx && IsAnotherLiteral) {
5162 getOperandLoc(
Operands, *LiteralOpIdx)),
5163 "only one unique literal operand is allowed");
5167 if (IsAnotherLiteral)
5168 LiteralOpIdx = OpIdx;
5177 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5191bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5197 ? AMDGPU::OpName::data0
5198 : AMDGPU::OpName::vdata;
5200 const MCRegisterInfo *MRI = getMRI();
5201 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5205 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5206 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5210 auto FB = getFeatureBits();
5211 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5212 if (DataAreg < 0 || DstAreg < 0)
5214 return DstAreg == DataAreg;
5217 return DstAreg < 1 && DataAreg < 1;
5220bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5221 auto FB = getFeatureBits();
5222 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5226 const MCRegisterInfo *MRI = getMRI();
5229 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5232 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5236 case AMDGPU::DS_LOAD_TR6_B96:
5237 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5242 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5246 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5247 if (VAddrIdx != -1) {
5250 if ((
Sub - AMDGPU::VGPR0) & 1)
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5256 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5261 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5262 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5282 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5283 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5285 return Op.getStartLoc();
5290bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5293 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5296 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5299 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5300 auto FB = getFeatureBits();
5301 bool UsesNeg =
false;
5302 if (FB[AMDGPU::FeatureGFX940Insts]) {
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5305 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5307 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5312 if (IsNeg == UsesNeg)
5315 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5316 :
"invalid modifier: neg is not supported");
5321bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5327 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5330 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5333 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5336 if (
Reg == AMDGPU::SGPR_NULL)
5339 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5343bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5348 return validateGWS(Inst,
Operands);
5353 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5358 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5359 Error(S,
"gds modifier is not supported on this GPU");
5367bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5369 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5373 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5374 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5377 const MCRegisterInfo *MRI = getMRI();
5378 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5380 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5383 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5385 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5392bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5396 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5404 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5407 Error(S,
"scale_offset is not supported on this GPU");
5410 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5413 Error(S,
"nv is not supported on this GPU");
5418 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5421 Error(S,
"scale_offset is not supported for this instruction");
5425 return validateTHAndScopeBits(Inst,
Operands, CPol);
5429 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5430 Error(S,
"cache policy is not supported for SMRD instructions");
5434 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5441 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5445 "scc modifier is not supported for this instruction on this GPU");
5456 :
"instruction must use glc");
5461 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5464 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5466 :
"instruction must not use glc");
5474bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5476 const unsigned CPol) {
5481 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5488 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5493 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5501 return PrintError(
"invalid th value for SMEM instruction");
5508 return PrintError(
"scope and th combination is not valid");
5514 return PrintError(
"invalid th value for atomic instructions");
5517 return PrintError(
"invalid th value for store instructions");
5520 return PrintError(
"invalid th value for load instructions");
5526bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5530 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5532 Error(Loc,
"TFE modifier has no meaning for store instructions");
5540bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5546 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5550 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5553 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5554 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5563 "wrong register tuple size for " +
5568 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5569 !validateFmt(BFmt, AMDGPU::OpName::src1))
5573 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5574 if (AScaleIdx == -1)
5578 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5582 "invalid matrix and scale format combination");
5589bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5591 if (!validateLdsDirect(Inst,
Operands))
5593 if (!validateTrue16OpSel(Inst)) {
5595 "op_sel operand conflicts with 16-bit operand suffix");
5598 if (!validateSOPLiteral(Inst,
Operands))
5600 if (!validateVOPLiteral(Inst,
Operands)) {
5603 if (!validateConstantBusLimitations(Inst,
Operands)) {
5606 if (!validateVOPD(Inst,
Operands)) {
5609 if (!validateIntClampSupported(Inst)) {
5611 "integer clamping is not supported on this GPU");
5614 if (!validateOpSel(Inst)) {
5616 "invalid op_sel operand");
5619 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5621 "invalid neg_lo operand");
5624 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5626 "invalid neg_hi operand");
5629 if (!validateDPP(Inst,
Operands)) {
5633 if (!validateMIMGD16(Inst)) {
5635 "d16 modifier is not supported on this GPU");
5638 if (!validateMIMGDim(Inst,
Operands)) {
5639 Error(IDLoc,
"missing dim operand");
5642 if (!validateTensorR128(Inst)) {
5644 "instruction must set modifier r128=0");
5647 if (!validateMIMGMSAA(Inst)) {
5649 "invalid dim; must be MSAA type");
5652 if (!validateMIMGDataSize(Inst, IDLoc)) {
5655 if (!validateMIMGAddrSize(Inst, IDLoc))
5657 if (!validateMIMGAtomicDMask(Inst)) {
5659 "invalid atomic image dmask");
5662 if (!validateMIMGGatherDMask(Inst)) {
5664 "invalid image_gather dmask: only one bit must be set");
5667 if (!validateMovrels(Inst,
Operands)) {
5670 if (!validateOffset(Inst,
Operands)) {
5673 if (!validateMAIAccWrite(Inst,
Operands)) {
5676 if (!validateMAISrc2(Inst,
Operands)) {
5679 if (!validateMFMA(Inst,
Operands)) {
5682 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5686 if (!validateAGPRLdSt(Inst)) {
5689 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5690 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5691 :
"invalid register class: agpr loads and stores not supported on "
5695 if (!validateVGPRAlign(Inst)) {
5696 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5703 if (!validateBLGP(Inst,
Operands)) {
5707 if (!validateDivScale(Inst)) {
5708 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5711 if (!validateWaitCnt(Inst,
Operands)) {
5714 if (!validateTFE(Inst,
Operands)) {
5717 if (!validateWMMA(Inst,
Operands)) {
5726 unsigned VariantID = 0);
5730 unsigned VariantID);
5732bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5737bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5738 const FeatureBitset &FBS,
5739 ArrayRef<unsigned> Variants) {
5740 for (
auto Variant : Variants) {
5748bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5750 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5753 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5758 getParser().clearPendingErrors();
5762 StringRef VariantName = getMatchedVariantName();
5763 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5764 return Error(IDLoc, Twine(VariantName,
5765 " variant of this instruction is not supported"));
5769 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5770 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5772 FeatureBitset FeaturesWS32 = getFeatureBits();
5773 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5774 .
flip(AMDGPU::FeatureWavefrontSize32);
5775 FeatureBitset AvailableFeaturesWS32 =
5776 ComputeAvailableFeatures(FeaturesWS32);
5778 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5779 return Error(IDLoc,
"instruction requires wavesize=32");
5783 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5784 return Error(IDLoc,
"instruction not supported on this GPU (" +
5785 getSTI().
getCPU() +
")" +
": " + Mnemo);
5790 return Error(IDLoc,
"invalid instruction" + Suggestion);
5796 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5797 if (
Op.isToken() && InvalidOprIdx > 1) {
5798 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5799 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5804bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5808 bool MatchingInlineAsm) {
5811 unsigned Result = Match_Success;
5812 for (
auto Variant : getMatchedVariants()) {
5815 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5819 if (R == Match_Success || R == Match_MissingFeature ||
5820 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5821 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5822 Result != Match_MissingFeature)) {
5826 if (R == Match_Success)
5830 if (Result == Match_Success) {
5831 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5834 emitTargetDirective();
5842 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5849 case Match_MissingFeature:
5853 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5855 case Match_InvalidOperand: {
5856 SMLoc ErrorLoc = IDLoc;
5857 if (ErrorInfo != ~0ULL) {
5858 if (ErrorInfo >=
Operands.size()) {
5859 return Error(IDLoc,
"too few operands for instruction");
5861 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5862 if (ErrorLoc == SMLoc())
5866 return Error(ErrorLoc,
"invalid VOPDY instruction");
5868 return Error(ErrorLoc,
"invalid operand for instruction");
5871 case Match_MnemonicFail:
5877bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5882 if (getParser().parseAbsoluteExpression(Tmp)) {
5885 Ret =
static_cast<uint32_t
>(Tmp);
5889bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5890 if (!getSTI().getTargetTriple().isAMDGCN())
5891 return TokError(
"directive only supported for amdgcn architecture");
5893 std::string TargetIDDirective;
5894 SMLoc TargetStart = getTok().getLoc();
5895 if (getParser().parseEscapedString(TargetIDDirective))
5898 std::optional<AMDGPU::TargetID> MaybeParsed =
5901 return getParser().Error(TargetStart,
5902 "malformed target id '" + TargetIDDirective +
"'");
5905 const Triple &
TT = getSTI().getTargetTriple();
5911 return getParser().Error(
5912 TargetStart,
"target id '" + TargetIDDirective +
5913 "' specifies a processor that is not valid for "
5915 TT.getArchName() +
"'");
5918 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5919 getTargetStreamer().getTargetID();
5922 const Triple &STITriple = getSTI().getTargetTriple();
5923 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5924 return getParser().Error(
5925 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5926 " is incompatible with " +
5927 Twine(CurrentTargetID->toString()));
5931 StringRef DirectiveProcessor =
5934 if (DirectiveISA != ISA) {
5935 return getParser().Error(TargetStart,
5936 ".amdgcn_target directive processor " +
5937 Twine(DirectiveProcessor) +
5938 " does not match the specified processor " +
5939 Twine(getSTI().
getCPU()));
5945 CurrentTargetID->getXnackSetting())) {
5947 ".amdgcn_target directive has conflicting xnack settings");
5951 CurrentTargetID->getSramEccSetting())) {
5953 ".amdgcn_target directive has conflicting sramecc settings");
5959 getTargetStreamer().getTargetID()->setXnackSetting(
5961 getTargetStreamer().getTargetID()->setSramEccSetting(
5967bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5971bool AMDGPUAsmParser::calculateGPRBlocks(
5972 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5973 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5974 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5975 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5976 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
5981 const MCExpr *
NumSGPRs = NextFreeSGPR;
5982 int64_t EvaluatedSGPRs;
5984 if (
ISA.Major >= 10)
5989 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
5990 !Features.
test(FeatureSGPRInitBug) &&
5991 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
5992 return OutOfRangeError(SGPRRange);
5994 const MCExpr *ExtraSGPRs =
5998 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
5999 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6000 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6001 return OutOfRangeError(SGPRRange);
6003 if (Features.
test(FeatureSGPRInitBug))
6010 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6011 unsigned Granule) ->
const MCExpr * {
6015 const MCExpr *AlignToGPR =
6017 const MCExpr *DivGPR =
6023 VGPRBlocks = GetNumGPRBlocks(
6032bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6033 if (!getSTI().getTargetTriple().isAMDGCN())
6034 return TokError(
"directive only supported for amdgcn architecture");
6037 return TokError(
"directive only supported for amdhsa OS");
6039 StringRef KernelName;
6040 if (getParser().parseIdentifier(KernelName))
6047 AMDGPU::MCKernelDescriptor KD =
6057 const MCExpr *NextFreeVGPR = ZeroExpr;
6059 const MCExpr *NamedBarCnt = ZeroExpr;
6064 const MCExpr *NextFreeSGPR = ZeroExpr;
6067 unsigned ImpliedUserSGPRCount = 0;
6071 std::optional<unsigned> ExplicitUserSGPRCount;
6072 const MCExpr *ReserveVCC = OneExpr;
6073 const MCExpr *ReserveFlatScr = OneExpr;
6074 std::optional<bool> EnableWavefrontSize32;
6081 SMRange IDRange = getTok().getLocRange();
6082 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6085 if (ID ==
".end_amdhsa_kernel")
6088 if (!Seen.
insert(ID).second)
6089 return TokError(
".amdhsa_ directives cannot be repeated");
6091 SMLoc ValStart = getLoc();
6092 const MCExpr *ExprVal;
6093 if (getParser().parseExpression(ExprVal))
6095 SMLoc ValEnd = getLoc();
6096 SMRange ValRange = SMRange(ValStart, ValEnd);
6100 bool EvaluatableExpr;
6101 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6103 return OutOfRangeError(ValRange);
6107#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6108 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6109 return OutOfRangeError(RANGE); \
6110 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6115#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6117 return Error(IDRange.Start, "directive should have resolvable expression", \
6120 if (ID ==
".amdhsa_group_segment_fixed_size") {
6123 return OutOfRangeError(ValRange);
6125 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6128 return OutOfRangeError(ValRange);
6130 }
else if (ID ==
".amdhsa_kernarg_size") {
6132 return OutOfRangeError(ValRange);
6134 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6136 ExplicitUserSGPRCount = Val;
6137 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6141 "directive is not supported with architected flat scratch",
6144 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6147 ImpliedUserSGPRCount += 4;
6148 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6151 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6154 return OutOfRangeError(ValRange);
6158 ImpliedUserSGPRCount += Val;
6159 PreloadLength = Val;
6161 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6164 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6167 return OutOfRangeError(ValRange);
6171 PreloadOffset = Val;
6172 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6175 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6178 ImpliedUserSGPRCount += 2;
6179 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6182 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6185 ImpliedUserSGPRCount += 2;
6186 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6189 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6192 ImpliedUserSGPRCount += 2;
6193 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6196 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6199 ImpliedUserSGPRCount += 2;
6200 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6203 "directive is not supported with architected flat scratch",
6207 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6210 ImpliedUserSGPRCount += 2;
6211 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6214 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6217 ImpliedUserSGPRCount += 1;
6218 }
else if (ID ==
".amdhsa_wavefront_size32") {
6221 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6222 EnableWavefrontSize32 = Val;
6224 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6226 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6228 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6230 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6233 "directive is not supported with architected flat scratch",
6236 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6238 }
else if (ID ==
".amdhsa_enable_private_segment") {
6242 "directive is not supported without architected flat scratch",
6245 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6247 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6249 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6251 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6253 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6255 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6257 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6259 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6261 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6263 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6265 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6267 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6268 VGPRRange = ValRange;
6269 NextFreeVGPR = ExprVal;
6270 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6271 SGPRRange = ValRange;
6272 NextFreeSGPR = ExprVal;
6273 }
else if (ID ==
".amdhsa_accum_offset") {
6275 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6276 AccumOffset = ExprVal;
6277 }
else if (ID ==
".amdhsa_named_barrier_count") {
6279 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6280 NamedBarCnt = ExprVal;
6281 }
else if (ID ==
".amdhsa_reserve_vcc") {
6283 return OutOfRangeError(ValRange);
6284 ReserveVCC = ExprVal;
6285 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6287 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6290 "directive is not supported with architected flat scratch",
6293 return OutOfRangeError(ValRange);
6294 ReserveFlatScr = ExprVal;
6295 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6297 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6299 return OutOfRangeError(ValRange);
6300 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6301 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6302 if (Val != XnackOn) {
6303 return getParser().Error(
6305 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6307 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6309 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6311 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6313 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6315 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6317 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6319 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6321 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6323 }
else if (ID ==
".amdhsa_dx10_clamp") {
6324 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6325 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6328 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6330 }
else if (ID ==
".amdhsa_ieee_mode") {
6331 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6332 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6335 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6337 }
else if (ID ==
".amdhsa_fp16_overflow") {
6339 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6341 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6343 }
else if (ID ==
".amdhsa_tg_split") {
6345 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6348 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6351 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6353 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6355 }
else if (ID ==
".amdhsa_memory_ordered") {
6357 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6359 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6361 }
else if (ID ==
".amdhsa_forward_progress") {
6363 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6365 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6367 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6369 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6370 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6372 SharedVGPRCount = Val;
6374 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6376 }
else if (ID ==
".amdhsa_inst_pref_size") {
6378 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6379 if (
ISA.Major == 11) {
6381 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6385 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6388 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6391 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6393 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6395 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6397 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6400 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6402 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6404 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6406 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6408 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6410 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6412 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6414 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6416 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6418 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6420 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6422 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6425 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6428#undef PARSE_BITS_ENTRY
6431 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6432 return TokError(
".amdhsa_next_free_vgpr directive is required");
6434 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6435 return TokError(
".amdhsa_next_free_sgpr directive is required");
6437 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6439 return TokError(
"too many user SGPRs enabled, found " +
6440 Twine(UserSGPRCount) +
", but only " +
6446 if (PreloadLength) {
6452 const MCExpr *VGPRBlocks;
6453 const MCExpr *SGPRBlocks;
6454 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6455 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6456 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6457 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6460 int64_t EvaluatedVGPRBlocks;
6461 bool VGPRBlocksEvaluatable =
6462 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6463 if (VGPRBlocksEvaluatable &&
6465 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6466 return OutOfRangeError(VGPRRange);
6470 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6471 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6473 int64_t EvaluatedSGPRBlocks;
6474 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6476 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6477 return OutOfRangeError(SGPRRange);
6480 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6481 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6483 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6484 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6485 "enabled user SGPRs");
6491 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6492 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6497 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6498 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6503 return TokError(
"Kernarg size should be resolvable");
6505 if (PreloadLength && kernarg_size &&
6506 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6507 return TokError(
"Kernarg preload length + offset is larger than the "
6508 "kernarg segment size");
6511 if (!Seen.
contains(
".amdhsa_accum_offset"))
6512 return TokError(
".amdhsa_accum_offset directive is required");
6513 int64_t EvaluatedAccum;
6514 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6515 uint64_t UEvaluatedAccum = EvaluatedAccum;
6516 if (AccumEvaluatable &&
6517 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6518 return TokError(
"accum_offset should be in range [4..256] in "
6521 int64_t EvaluatedNumVGPR;
6522 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6526 return TokError(
"accum_offset exceeds total VGPR allocation");
6532 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6533 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6539 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6540 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6543 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6545 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6546 return TokError(
"shared_vgpr_count directive not valid on "
6547 "wavefront size 32");
6550 if (VGPRBlocksEvaluatable &&
6551 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6553 return TokError(
"shared_vgpr_count*2 + "
6554 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6559 emitTargetDirective();
6560 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6561 NextFreeVGPR, NextFreeSGPR,
6562 ReserveVCC, ReserveFlatScr);
6566bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6568 if (ParseAsAbsoluteExpression(
Version))
6571 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6572 emitTargetDirective();
6576bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6577 AMDGPUMCKernelCodeT &
C) {
6580 if (ID ==
"max_scratch_backing_memory_byte_size") {
6581 Parser.eatToEndOfStatement();
6585 SmallString<40> ErrStr;
6586 raw_svector_ostream Err(ErrStr);
6587 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6588 return TokError(Err.
str());
6592 if (ID ==
"enable_wavefront_size32") {
6595 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6597 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6600 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6604 if (ID ==
"wavefront_size") {
6605 if (
C.wavefront_size == 5) {
6607 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6609 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6610 }
else if (
C.wavefront_size == 6) {
6612 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6619bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6620 AMDGPUMCKernelCodeT KernelCode;
6630 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6633 if (ID ==
".end_amd_kernel_code_t")
6636 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6641 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6646bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6647 StringRef KernelName;
6648 if (!parseId(KernelName,
"expected symbol name"))
6651 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6658bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6659 if (!getSTI().getTargetTriple().isAMDGCN()) {
6660 return Error(getLoc(),
6661 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6665 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6667 std::optional<AMDGPU::TargetID> MaybeParsed =
6670 return Error(getParser().getTok().getLoc(),
6671 "malformed target id '" + TargetIDDirective +
"'");
6674 const Triple &
TT = getSTI().getTargetTriple();
6680 return Error(getParser().getTok().getLoc(),
6681 "target id '" + TargetIDDirective +
6682 "' specifies a processor that is not valid for subarch '" +
6683 TT.getArchName() +
"'");
6686 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6687 getTargetStreamer().getTargetID();
6690 const Triple &STITriple = getSTI().getTargetTriple();
6691 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6692 return Error(getParser().getTok().getLoc(),
6693 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6694 " is incompatible with " +
6695 Twine(CurrentTargetID->toString()));
6699 StringRef DirectiveProcessor =
6702 if (DirectiveISA != ISA) {
6703 return Error(getParser().getTok().getLoc(),
6704 ".amd_amdgpu_isa directive processor " +
6705 Twine(DirectiveProcessor) +
6706 " does not match the specified processor " +
6707 Twine(getSTI().
getCPU()));
6710 getTargetStreamer().EmitISAVersion();
6716bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6719 std::string HSAMetadataString;
6724 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6725 return Error(getLoc(),
"invalid HSA metadata");
6732bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6733 const char *AssemblerDirectiveEnd,
6734 std::string &CollectString) {
6736 raw_string_ostream CollectStream(CollectString);
6738 getLexer().setSkipSpace(
false);
6740 bool FoundEnd =
false;
6743 CollectStream << getTokenStr();
6747 if (trySkipId(AssemblerDirectiveEnd)) {
6752 CollectStream << Parser.parseStringToEndOfStatement()
6753 <<
getContext().getAsmInfo().getSeparatorString();
6755 Parser.eatToEndOfStatement();
6758 getLexer().setSkipSpace(
true);
6761 return TokError(Twine(
"expected directive ") +
6762 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6769bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6775 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6776 if (!PALMetadata->setFromString(
String))
6777 return Error(getLoc(),
"invalid PAL metadata");
6782bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6785 Twine(
" directive is "
6786 "not available on non-amdpal OSes"))
6790 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6791 PALMetadata->setLegacy();
6794 if (ParseAsAbsoluteExpression(
Key)) {
6795 return TokError(Twine(
"invalid value in ") +
6799 return TokError(Twine(
"expected an even number of values in ") +
6802 if (ParseAsAbsoluteExpression(
Value)) {
6803 return TokError(Twine(
"invalid value in ") +
6806 PALMetadata->setRegister(
Key,
Value);
6815bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6816 if (getParser().checkForValidSection())
6820 SMLoc NameLoc = getLoc();
6821 if (getParser().parseIdentifier(Name))
6822 return TokError(
"expected identifier in directive");
6825 if (getParser().parseComma())
6831 SMLoc SizeLoc = getLoc();
6832 if (getParser().parseAbsoluteExpression(
Size))
6835 return Error(SizeLoc,
"size must be non-negative");
6836 if (
Size > LocalMemorySize)
6837 return Error(SizeLoc,
"size is too large");
6841 SMLoc AlignLoc = getLoc();
6842 if (getParser().parseAbsoluteExpression(Alignment))
6845 return Error(AlignLoc,
"alignment must be a power of two");
6850 if (Alignment >= 1u << 31)
6851 return Error(AlignLoc,
"alignment is too large");
6857 Symbol->redefineIfPossible();
6858 if (!
Symbol->isUndefined())
6859 return Error(NameLoc,
"invalid symbol redefinition");
6861 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6865bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6866 if (getParser().checkForValidSection())
6870 if (getParser().parseIdentifier(FuncName))
6871 return TokError(
"expected symbol name after .amdgpu_info");
6874 AMDGPU::InfoSectionData ParsedInfoData;
6875 AMDGPU::FuncInfo FI;
6877 bool HasScalarAttrs =
false;
6884 SMLoc IDLoc = getLoc();
6885 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6888 if (ID ==
".end_amdgpu_info")
6896 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6898 if (Dir ==
"flags") {
6900 if (getParser().parseAbsoluteExpression(Val))
6903 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6905 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6907 HasScalarAttrs =
true;
6908 }
else if (Dir ==
"num_sgpr") {
6910 if (getParser().parseAbsoluteExpression(Val))
6912 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6913 HasScalarAttrs =
true;
6914 }
else if (Dir ==
"num_vgpr") {
6916 if (getParser().parseAbsoluteExpression(Val))
6919 HasScalarAttrs =
true;
6920 }
else if (Dir ==
"num_agpr") {
6922 if (getParser().parseAbsoluteExpression(Val))
6925 HasScalarAttrs =
true;
6926 }
else if (Dir ==
"private_segment_size") {
6928 if (getParser().parseAbsoluteExpression(Val))
6931 HasScalarAttrs =
true;
6932 }
else if (Dir ==
"use") {
6934 if (getParser().parseIdentifier(ResName))
6935 return TokError(
"expected resource symbol for .amdgpu_use");
6936 ParsedInfoData.
Uses.push_back(
6937 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6938 }
else if (Dir ==
"call") {
6940 if (getParser().parseIdentifier(DstName))
6941 return TokError(
"expected callee symbol for .amdgpu_call");
6942 ParsedInfoData.
Calls.push_back(
6943 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6944 }
else if (Dir ==
"indirect_call") {
6946 if (getParser().parseEscapedString(TypeId))
6947 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6948 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6949 }
else if (Dir ==
"typeid") {
6951 if (getParser().parseEscapedString(TypeId))
6952 return TokError(
"expected type ID string for .amdgpu_typeid");
6953 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6955 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6960 ParsedInfoData.
Funcs.push_back(std::move(FI));
6962 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6963 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6964 Data.Funcs.push_back(std::move(Func));
6965 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6966 Data.Uses.push_back(Use);
6967 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6969 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6972 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6973 Data.TypeIds.push_back(std::move(TypeId));
6978void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
6985void AMDGPUAsmParser::checkKernelPrologues() {
6986 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
6987 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
6988 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
6989 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
6990 if (!AMDHSAKernelSymbols.
contains(Sym))
6992 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
6993 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
6997 "' does not begin with the required prologue "
6998 "sequence: s_mov_b64 followed by v_nop and "
6999 "global_prefetch_b8");
7003 OpcodeStream.
clear();
7004 OpcodeStreamSymbols.clear();
7005 AMDHSAKernelSymbols.
clear();
7008void AMDGPUAsmParser::onEndOfFile() {
7009 emitTargetDirective();
7010 checkKernelPrologues();
7012 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7015bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7016 StringRef IDVal = DirectiveID.
getString();
7019 if (IDVal ==
".amdhsa_kernel")
7020 return ParseDirectiveAMDHSAKernel();
7022 if (IDVal ==
".amdhsa_code_object_version")
7023 return ParseDirectiveAMDHSACodeObjectVersion();
7027 return ParseDirectiveHSAMetadata();
7029 if (IDVal ==
".amd_kernel_code_t")
7030 return ParseDirectiveAMDKernelCodeT();
7032 if (IDVal ==
".amdgpu_hsa_kernel")
7033 return ParseDirectiveAMDGPUHsaKernel();
7035 if (IDVal ==
".amd_amdgpu_isa")
7036 return ParseDirectiveISAVersion();
7040 Twine(
" directive is "
7041 "not available on non-amdhsa OSes"))
7046 if (IDVal ==
".amdgcn_target")
7047 return ParseDirectiveAMDGCNTarget();
7049 if (IDVal ==
".amdgpu_lds")
7050 return ParseDirectiveAMDGPULDS();
7052 if (IDVal ==
".amdgpu_info")
7053 return ParseDirectiveAMDGPUInfo();
7056 return ParseDirectivePALMetadataBegin();
7059 return ParseDirectivePALMetadata();
7064bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7071 return hasSGPR104_SGPR105();
7074 case SRC_SHARED_BASE_LO:
7075 case SRC_SHARED_BASE:
7076 case SRC_SHARED_LIMIT_LO:
7077 case SRC_SHARED_LIMIT:
7079 case SRC_PRIVATE_BASE_LO:
7080 case SRC_PRIVATE_BASE:
7081 case SRC_PRIVATE_LIMIT_LO:
7082 case SRC_PRIVATE_LIMIT:
7084 case SRC_FLAT_SCRATCH_BASE_LO:
7085 case SRC_FLAT_SCRATCH_BASE_HI:
7086 return hasGloballyAddressableScratch();
7087 case SRC_POPS_EXITING_WAVE_ID:
7100 getTargetStreamer().getTargetID()->isXnackSupported();
7130 return hasSGPR102_SGPR103();
7138 ParseStatus Res = parseVOPD(
Operands);
7143 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7155 SMLoc LBraceLoc = getLoc();
7160 auto Loc = getLoc();
7163 Error(Loc,
"expected a register");
7167 RBraceLoc = getLoc();
7172 "expected a comma or a closing square bracket"))
7176 if (
Operands.size() - Prefix > 1) {
7178 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7179 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7188StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7190 setForcedEncodingSize(0);
7191 setForcedDPP(
false);
7192 setForcedSDWA(
false);
7194 if (
Name.consume_back(
"_e64_dpp")) {
7196 setForcedEncodingSize(64);
7199 if (
Name.consume_back(
"_e64")) {
7200 setForcedEncodingSize(64);
7203 if (
Name.consume_back(
"_e32")) {
7204 setForcedEncodingSize(32);
7207 if (
Name.consume_back(
"_dpp")) {
7211 if (
Name.consume_back(
"_sdwa")) {
7212 setForcedSDWA(
true);
7220 unsigned VariantID);
7226 Name = parseMnemonicSuffix(Name);
7232 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7234 bool IsMIMG = Name.starts_with(
"image_");
7237 OperandMode
Mode = OperandMode_Default;
7239 Mode = OperandMode_NSA;
7243 checkUnsupportedInstruction(Name, NameLoc);
7244 if (!Parser.hasPendingError()) {
7247 :
"not a valid operand.";
7267ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7270 if (!trySkipId(Name))
7273 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7277ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7286ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7288 std::function<
bool(int64_t &)> ConvertResult) {
7292 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7296 if (ConvertResult && !ConvertResult(
Value)) {
7297 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7300 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7304ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7306 bool (*ConvertResult)(int64_t &)) {
7315 const unsigned MaxSize = 4;
7319 for (
int I = 0;; ++
I) {
7321 SMLoc Loc = getLoc();
7325 if (
Op != 0 &&
Op != 1)
7326 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7333 if (
I + 1 == MaxSize)
7334 return Error(getLoc(),
"expected a closing square bracket");
7340 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7344ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7346 AMDGPUOperand::ImmTy ImmTy,
7347 bool IgnoreNegative) {
7351 if (trySkipId(Name)) {
7353 }
else if (trySkipId(
"no", Name)) {
7362 return Error(S,
"r128 modifier is not supported on this GPU");
7363 if (Name ==
"a16" && !
hasA16())
7364 return Error(S,
"a16 modifier is not supported on this GPU");
7366 if (Bit == 0 && Name ==
"gds") {
7369 return Error(S,
"nogds is not allowed");
7372 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7373 ImmTy = AMDGPUOperand::ImmTyR128A16;
7375 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7379unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7380 bool &Disabling)
const {
7381 Disabling =
Id.consume_front(
"no");
7384 return StringSwitch<unsigned>(Id)
7391 return StringSwitch<unsigned>(Id)
7401 SMLoc StringLoc = getLoc();
7403 int64_t CPolVal = 0;
7423 ResScope = parseScope(
Operands, Scope);
7436 if (trySkipId(
"nv")) {
7440 }
else if (trySkipId(
"no",
"nv")) {
7447 if (trySkipId(
"scale_offset")) {
7451 }
else if (trySkipId(
"no",
"scale_offset")) {
7464 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7465 AMDGPUOperand::ImmTyCPol));
7470 SMLoc OpLoc = getLoc();
7471 unsigned Enabled = 0, Seen = 0;
7475 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7482 return Error(S,
"dlc modifier is not supported on this GPU");
7485 return Error(S,
"scc modifier is not supported on this GPU");
7488 return Error(S,
"duplicate cache policy modifier");
7500 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7509 ParseStatus Res = parseStringOrIntWithPrefix(
7510 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7524 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7528 if (
Value ==
"TH_DEFAULT")
7530 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7531 Value ==
"TH_LOAD_NT_WB") {
7532 return Error(StringLoc,
"invalid th value");
7533 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7535 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7537 }
else if (
Value.consume_front(
"TH_STORE_")) {
7540 return Error(StringLoc,
"invalid th value");
7543 if (
Value ==
"BYPASS")
7548 TH |= StringSwitch<int64_t>(
Value)
7558 .Default(0xffffffff);
7560 TH |= StringSwitch<int64_t>(
Value)
7571 .Default(0xffffffff);
7574 if (TH == 0xffffffff)
7575 return Error(StringLoc,
"invalid th value");
7582 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7583 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7584 std::optional<unsigned> InsertAt = std::nullopt) {
7585 auto i = OptionalIdx.find(ImmT);
7586 if (i != OptionalIdx.end()) {
7587 unsigned Idx = i->second;
7588 const AMDGPUOperand &
Op =
7589 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7593 Op.addImmOperands(Inst, 1);
7595 if (InsertAt.has_value())
7602ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7608 StringLoc = getLoc();
7613ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7619 SMLoc StringLoc = getLoc();
7623 Value = getTokenStr();
7627 if (
Value == Ids[IntVal])
7632 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7633 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7638ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7640 AMDGPUOperand::ImmTy
Type) {
7644 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7646 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7655bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7658 SMLoc Loc = getLoc();
7660 auto Res = parseIntWithPrefix(Pref, Val);
7666 if (Val < 0 || Val > MaxVal) {
7667 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7676 AMDGPUOperand::ImmTy ImmTy) {
7677 const char *Pref =
"index_key";
7679 SMLoc Loc = getLoc();
7680 auto Res = parseIntWithPrefix(Pref, ImmVal);
7684 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7685 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7686 (ImmVal < 0 || ImmVal > 1))
7687 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7689 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7690 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7692 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7697 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7701 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7705 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7710 AMDGPUOperand::ImmTy
Type) {
7716 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7717 AMDGPUOperand::ImmTyMatrixAFMT);
7721 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7722 AMDGPUOperand::ImmTyMatrixBFMT);
7727 AMDGPUOperand::ImmTy
Type) {
7733 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7734 AMDGPUOperand::ImmTyMatrixAScale);
7738 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7739 AMDGPUOperand::ImmTyMatrixBScale);
7744 AMDGPUOperand::ImmTy
Type) {
7750 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7751 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7755 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7756 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7761ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7762 using namespace llvm::AMDGPU::MTBUFFormat;
7768 for (
int I = 0;
I < 2; ++
I) {
7769 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7772 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7777 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7783 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7786 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7787 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7793ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7794 using namespace llvm::AMDGPU::MTBUFFormat;
7798 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7801 if (Fmt == UFMT_UNDEF)
7808bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7809 StringRef FormatStr, SMLoc Loc) {
7810 using namespace llvm::AMDGPU::MTBUFFormat;
7814 if (
Format != DFMT_UNDEF) {
7820 if (
Format != NFMT_UNDEF) {
7825 Error(Loc,
"unsupported format");
7829ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7832 using namespace llvm::AMDGPU::MTBUFFormat;
7836 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7841 SMLoc Loc = getLoc();
7842 if (!parseId(Str,
"expected a format string") ||
7843 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7845 if (Dfmt == DFMT_UNDEF)
7846 return Error(Loc,
"duplicate numeric format");
7847 if (Nfmt == NFMT_UNDEF)
7848 return Error(Loc,
"duplicate data format");
7851 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7852 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7856 if (Ufmt == UFMT_UNDEF)
7857 return Error(FormatLoc,
"unsupported format");
7866ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7869 using namespace llvm::AMDGPU::MTBUFFormat;
7872 if (Id == UFMT_UNDEF)
7876 return Error(Loc,
"unified format is not supported on this GPU");
7882ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7883 using namespace llvm::AMDGPU::MTBUFFormat;
7884 SMLoc Loc = getLoc();
7889 return Error(Loc,
"out of range format");
7894ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7895 using namespace llvm::AMDGPU::MTBUFFormat;
7901 StringRef FormatStr;
7902 SMLoc Loc = getLoc();
7903 if (!parseId(FormatStr,
"expected a format string"))
7906 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7908 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7918 return parseNumericFormat(
Format);
7922 using namespace llvm::AMDGPU::MTBUFFormat;
7926 SMLoc Loc = getLoc();
7936 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7955 Res = parseSymbolicOrNumericFormat(
Format);
7960 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7961 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7968 return Error(getLoc(),
"duplicate format");
7974 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7976 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7977 AMDGPUOperand::ImmTyInstOffset);
7984 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
7986 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
7992 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
7995 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8005 OptionalImmIndexMap OptionalIdx;
8007 unsigned OperandIdx[4];
8008 unsigned EnMask = 0;
8011 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8012 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8017 OperandIdx[SrcIdx] = Inst.
size();
8018 Op.addRegOperands(Inst, 1);
8025 OperandIdx[SrcIdx] = Inst.
size();
8031 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8032 Op.addImmOperands(Inst, 1);
8036 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8040 OptionalIdx[
Op.getImmTy()] = i;
8046 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8053 for (
auto i = 0; i < SrcIdx; ++i) {
8055 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8061 AMDGPUOperand::ImmTyExpCompr);
8071 int64_t CntVal,
bool Saturate,
8077 IntVal =
encode(ISA, IntVal, CntVal);
8078 if (CntVal !=
decode(ISA, IntVal)) {
8080 IntVal =
encode(ISA, IntVal, -1);
8088bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8090 SMLoc CntLoc = getLoc();
8091 StringRef CntName = getTokenStr();
8098 SMLoc ValLoc = getLoc();
8105 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8107 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8109 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8112 Error(CntLoc,
"invalid counter name " + CntName);
8117 Error(ValLoc,
"too large value for " + CntName);
8126 Error(getLoc(),
"expected a counter name");
8140 if (!parseCnt(Waitcnt))
8148 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8152bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8153 SMLoc FieldLoc = getLoc();
8154 StringRef FieldName = getTokenStr();
8159 SMLoc ValueLoc = getLoc();
8166 if (FieldName ==
"instid0") {
8168 }
else if (FieldName ==
"instskip") {
8170 }
else if (FieldName ==
"instid1") {
8173 Error(FieldLoc,
"invalid field name " + FieldName);
8192 .Case(
"VALU_DEP_1", 1)
8193 .Case(
"VALU_DEP_2", 2)
8194 .Case(
"VALU_DEP_3", 3)
8195 .Case(
"VALU_DEP_4", 4)
8196 .Case(
"TRANS32_DEP_1", 5)
8197 .Case(
"TRANS32_DEP_2", 6)
8198 .Case(
"TRANS32_DEP_3", 7)
8199 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8200 .Case(
"SALU_CYCLE_1", 9)
8201 .Case(
"SALU_CYCLE_2", 10)
8202 .Case(
"SALU_CYCLE_3", 11)
8210 Delay |=
Value << Shift;
8220 if (!parseDelay(Delay))
8228 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8232bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8234bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8240void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8241 StringRef DepCtrName) {
8244 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8247 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8250 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8253 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8260bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8262 using namespace llvm::AMDGPU::DepCtr;
8264 SMLoc DepCtrLoc = getLoc();
8265 StringRef DepCtrName = getTokenStr();
8275 unsigned PrevOprMask = UsedOprMask;
8276 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8279 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8288 Error(getLoc(),
"expected a counter name");
8293 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8294 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8299 using namespace llvm::AMDGPU::DepCtr;
8302 SMLoc Loc = getLoc();
8305 unsigned UsedOprMask = 0;
8307 if (!parseDepCtr(DepCtr, UsedOprMask))
8315 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8319bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8325ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8327 OperandInfoTy &Width) {
8328 using namespace llvm::AMDGPU::Hwreg;
8334 HwReg.Loc = getLoc();
8337 HwReg.IsSymbolic =
true;
8339 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8347 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8357 Width.Loc = getLoc();
8366 using namespace llvm::AMDGPU::Hwreg;
8369 SMLoc Loc = getLoc();
8371 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8373 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8374 HwregOffset::Default);
8375 struct : StructuredOpField {
8376 using StructuredOpField::StructuredOpField;
8377 bool validate(AMDGPUAsmParser &Parser)
const override {
8379 return Error(Parser,
"only values from 1 to 32 are legal");
8382 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8383 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8386 Res = parseHwregFunc(HwReg,
Offset, Width);
8389 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8391 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8395 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8402 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8404 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8408bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8414bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8415 OperandInfoTy &Stream) {
8416 using namespace llvm::AMDGPU::SendMsg;
8421 Msg.IsSymbolic =
true;
8428 Op.IsDefined =
true;
8434 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8439 Stream.IsDefined =
true;
8440 Stream.Loc = getLoc();
8449bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8450 const OperandInfoTy &
Op,
8451 const OperandInfoTy &Stream) {
8452 using namespace llvm::AMDGPU::SendMsg;
8461 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8466 Error(
Msg.Loc,
"invalid message id");
8472 Error(
Op.Loc,
"message does not support operations");
8474 Error(
Msg.Loc,
"missing message operation");
8480 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8482 Error(
Op.Loc,
"invalid operation id");
8487 Error(Stream.Loc,
"message operation does not support streams");
8491 Error(Stream.Loc,
"invalid message stream id");
8498 using namespace llvm::AMDGPU::SendMsg;
8501 SMLoc Loc = getLoc();
8505 OperandInfoTy
Op(OP_NONE_);
8506 OperandInfoTy Stream(STREAM_ID_NONE_);
8507 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8512 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8514 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8520 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8524bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8527 using namespace llvm::AMDGPU::WaitEvent;
8529 SMLoc Loc = getLoc();
8532 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8534 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8536 StructuredOpField *TargetBitfield =
8537 isGFX11() ? &DontWaitExportReady : &ExportReady;
8539 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8543 if (!validateStructuredOpFields({TargetBitfield}))
8545 ImmVal = TargetBitfield->Val;
8552 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8554 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8555 AMDGPUOperand::ImmTyWaitEvent));
8559bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8572 int Slot = StringSwitch<int>(Str)
8579 return Error(S,
"invalid interpolation slot");
8582 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8593 if (!Str.starts_with(
"attr"))
8594 return Error(S,
"invalid interpolation attribute");
8596 StringRef Chan = Str.take_back(2);
8597 int AttrChan = StringSwitch<int>(Chan)
8604 return Error(S,
"invalid or missing interpolation attribute channel");
8606 Str = Str.drop_back(2).drop_front(4);
8609 if (Str.getAsInteger(10, Attr))
8610 return Error(S,
"invalid or missing interpolation attribute number");
8613 return Error(S,
"out of bounds interpolation attribute number");
8618 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8619 Operands.push_back(AMDGPUOperand::CreateImm(
8620 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8629 using namespace llvm::AMDGPU::Exp;
8639 return Error(S, (Id == ET_INVALID)
8640 ?
"invalid exp target"
8641 :
"exp target is not supported on this GPU");
8644 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8652bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8656bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8661 return getTokenKind() ==
Kind;
8664StringRef AMDGPUAsmParser::getId()
const {
8668bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8676bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8678 StringRef Tok = getTokenStr();
8687bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8689 if (isId(Id) && peekToken().is(Kind)) {
8698 if (isToken(Kind)) {
8706 const StringRef ErrMsg) {
8707 if (!trySkipToken(Kind)) {
8708 Error(getLoc(), ErrMsg);
8714bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8718 if (Parser.parseExpression(Expr))
8721 if (Expr->evaluateAsAbsolute(
Imm))
8724 if (Expected.empty()) {
8725 Error(S,
"expected absolute expression");
8728 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8737 if (Parser.parseExpression(Expr))
8741 if (Expr->evaluateAsAbsolute(IntVal)) {
8742 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8744 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8749bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8751 Val =
getToken().getStringContents();
8755 Error(getLoc(), ErrMsg);
8759bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8761 Val = getTokenStr();
8765 if (!ErrMsg.
empty())
8766 Error(getLoc(), ErrMsg);
8770AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8772AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8775 : getLexer().peekTok(ShouldSkipSpace);
8779 auto TokCount = getLexer().peekTokens(Tokens);
8781 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8786 return getLexer().getKind();
8789SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8791StringRef AMDGPUAsmParser::getTokenStr()
const {
8795void AMDGPUAsmParser::lex() { Parser.Lex(); }
8797const AMDGPUOperand &
8799 int MCOpIdx)
const {
8801 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8802 if (TargetOp.getMCOpIdx() == MCOpIdx)
8809 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8813SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8818 int MCOpIdx)
const {
8819 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8822SMLoc AMDGPUAsmParser::getOperandLoc(
8823 std::function<
bool(
const AMDGPUOperand &)>
Test,
8825 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8826 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8828 return Op.getStartLoc();
8833SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8835 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8850 StringRef
Id = getTokenStr();
8851 SMLoc IdLoc = getLoc();
8857 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8858 if (
I == Fields.
end())
8859 return Error(IdLoc,
"unknown field");
8860 if ((*I)->IsDefined)
8861 return Error(IdLoc,
"duplicate field");
8864 (*I)->Loc = getLoc();
8867 (*I)->IsDefined =
true;
8874bool AMDGPUAsmParser::validateStructuredOpFields(
8876 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8877 return F->validate(*
this);
8887 const unsigned XorMask) {
8894bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8895 const unsigned MaxVal,
8896 const Twine &ErrMsg, SMLoc &Loc) {
8912bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8913 const unsigned MinVal,
8914 const unsigned MaxVal,
8915 const StringRef ErrMsg) {
8917 for (
unsigned i = 0; i < OpNum; ++i) {
8918 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8925bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8926 using namespace llvm::AMDGPU::Swizzle;
8929 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8930 "expected a 2-bit lane id")) {
8940bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8941 using namespace llvm::AMDGPU::Swizzle;
8947 if (!parseSwizzleOperand(GroupSize, 2, 32,
8948 "group size must be in the interval [2,32]", Loc)) {
8952 Error(Loc,
"group size must be a power of two");
8955 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8956 "lane id must be in the interval [0,group size - 1]",
8964bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8965 using namespace llvm::AMDGPU::Swizzle;
8970 if (!parseSwizzleOperand(GroupSize, 2, 32,
8971 "group size must be in the interval [2,32]", Loc)) {
8975 Error(Loc,
"group size must be a power of two");
8983bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
8984 using namespace llvm::AMDGPU::Swizzle;
8989 if (!parseSwizzleOperand(GroupSize, 1, 16,
8990 "group size must be in the interval [1,16]", Loc)) {
8994 Error(Loc,
"group size must be a power of two");
9002bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9003 using namespace llvm::AMDGPU::Swizzle;
9010 SMLoc StrLoc = getLoc();
9011 if (!parseString(Ctl)) {
9014 if (Ctl.
size() != BITMASK_WIDTH) {
9015 Error(StrLoc,
"expected a 5-character mask");
9019 unsigned AndMask = 0;
9020 unsigned OrMask = 0;
9021 unsigned XorMask = 0;
9023 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9027 Error(StrLoc,
"invalid mask");
9048bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9049 using namespace llvm::AMDGPU::Swizzle;
9052 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9058 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9059 "FFT swizzle must be in the interval [0," +
9060 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9068bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9069 using namespace llvm::AMDGPU::Swizzle;
9072 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9079 if (!parseSwizzleOperand(
Direction, 0, 1,
9080 "direction must be 0 (left) or 1 (right)", Loc))
9084 if (!parseSwizzleOperand(
9085 RotateSize, 0, ROTATE_MAX_SIZE,
9086 "number of threads to rotate must be in the interval [0," +
9087 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9092 (RotateSize << ROTATE_SIZE_SHIFT);
9096bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9098 SMLoc OffsetLoc = getLoc();
9104 Error(OffsetLoc,
"expected a 16-bit offset");
9110bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9111 using namespace llvm::AMDGPU::Swizzle;
9115 SMLoc ModeLoc = getLoc();
9118 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9119 Ok = parseSwizzleQuadPerm(
Imm);
9120 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9121 Ok = parseSwizzleBitmaskPerm(
Imm);
9122 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9123 Ok = parseSwizzleBroadcast(
Imm);
9124 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9125 Ok = parseSwizzleSwap(
Imm);
9126 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9127 Ok = parseSwizzleReverse(
Imm);
9128 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9129 Ok = parseSwizzleFFT(
Imm);
9130 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9131 Ok = parseSwizzleRotate(
Imm);
9133 Error(ModeLoc,
"expected a swizzle mode");
9136 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9146 if (trySkipId(
"offset")) {
9150 if (trySkipId(
"swizzle")) {
9151 Ok = parseSwizzleMacro(
Imm);
9153 Ok = parseSwizzleOffset(
Imm);
9158 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9165bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9171int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9173 using namespace llvm::AMDGPU::VGPRIndexMode;
9185 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9186 if (trySkipId(IdSymbolic[ModeId])) {
9194 ?
"expected a VGPR index mode or a closing parenthesis"
9195 :
"expected a VGPR index mode");
9200 Error(S,
"duplicate VGPR index mode");
9208 "expected a comma or a closing parenthesis"))
9217 using namespace llvm::AMDGPU::VGPRIndexMode;
9223 Imm = parseGPRIdxMacro();
9227 if (getParser().parseAbsoluteExpression(
Imm))
9230 return Error(S,
"invalid immediate: only 4-bit values are legal");
9234 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9238bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9249 if (isRegister() || isModifier())
9256 assert(Opr.isImm() || Opr.isExpr());
9257 SMLoc Loc = Opr.getStartLoc();
9261 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9262 Error(Loc,
"expected an absolute expression or a label");
9263 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9264 Error(Loc,
"expected a 16-bit signed jump offset");
9284 OptionalImmIndexMap OptionalIdx;
9285 unsigned FirstOperandIdx = 1;
9286 bool IsAtomicReturn =
false;
9292 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9293 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9297 Op.addRegOperands(Inst, 1);
9301 if (IsAtomicReturn && i == FirstOperandIdx)
9302 Op.addRegOperands(Inst, 1);
9307 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9308 Op.addImmOperands(Inst, 1);
9320 OptionalIdx[
Op.getImmTy()] = i;
9324 AMDGPUOperand::ImmTyOffset);
9336bool AMDGPUOperand::isSMRDOffset8()
const {
9340bool AMDGPUOperand::isSMEMOffset()
const {
9342 return isImmLiteral();
9345bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9380bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9381 if (BoundCtrl == 0 || BoundCtrl == 1) {
9389void AMDGPUAsmParser::onBeginOfFile() {
9390 if (!getParser().getStreamer().getTargetStreamer())
9393 if (!getTargetStreamer().getTargetID())
9394 getTargetStreamer().initializeTargetID(getSTI(),
9398void AMDGPUAsmParser::emitTargetDirective() {
9399 if (TargetDirectiveEmitted)
9401 TargetDirectiveEmitted =
true;
9403 if (!getParser().getStreamer().getTargetStreamer() ||
9408 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9417bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9421 StringRef TokenId = getTokenStr();
9422 AGVK VK = StringSwitch<AGVK>(TokenId)
9423 .Case(
"max", AGVK::AGVK_Max)
9424 .Case(
"min", AGVK::AGVK_Min)
9425 .Case(
"or", AGVK::AGVK_Or)
9426 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9427 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9428 .Case(
"alignto", AGVK::AGVK_AlignTo)
9429 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9430 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9431 .Default(AGVK::AGVK_None);
9440 if (Exprs.
empty()) {
9442 "empty " + Twine(TokenId) +
" expression");
9445 if (CommaCount + 1 != Exprs.
size()) {
9447 "mismatch of commas in " + Twine(TokenId) +
" expression");
9451 Expected && Exprs.
size() != Expected) {
9452 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9453 Twine(Expected) +
" operands");
9460 if (getParser().parseExpression(Expr, EndLoc))
9464 if (LastTokenWasComma)
9468 "unexpected token in " + Twine(TokenId) +
" expression");
9474 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9478 StringRef
Name = getTokenStr();
9479 if (Name ==
"mul") {
9480 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9484 if (Name ==
"div") {
9485 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9496 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9501 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9502 AMDGPU::OpName::src2};
9510 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9515 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9517 if (
DstOp.isReg() &&
9522 if ((OpSel & (1 << SrcNum)) != 0)
9528void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9535 OptionalImmIndexMap &OptionalIdx) {
9536 cvtVOP3P(Inst,
Operands, OptionalIdx);
9545 &&
Desc.NumOperands > (OpNum + 1)
9547 &&
Desc.operands()[OpNum + 1].RegClass != -1
9549 &&
Desc.getOperandConstraint(OpNum + 1,
9553void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9555 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9556 AMDGPU::OpName::src2};
9557 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9558 AMDGPU::OpName::src1_modifiers,
9559 AMDGPU::OpName::src2_modifiers};
9560 for (
int J = 0; J < 3; ++J) {
9561 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9567 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9570 if ((OpSel & (1 << J)) != 0)
9573 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9580void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9582 OptionalImmIndexMap OptionalIdx;
9587 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9588 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9592 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9594 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9595 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9596 Op.isInterpAttrChan()) {
9598 }
else if (
Op.isImmModifier()) {
9599 OptionalIdx[
Op.getImmTy()] =
I;
9607 AMDGPUOperand::ImmTyHigh);
9611 AMDGPUOperand::ImmTyClamp);
9615 AMDGPUOperand::ImmTyOModSI);
9620 AMDGPUOperand::ImmTyOpSel);
9621 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9624 cvtOpSelHelper(Inst, OpSel);
9629 OptionalImmIndexMap OptionalIdx;
9634 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9635 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9639 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9641 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9642 }
else if (
Op.isImmModifier()) {
9643 OptionalIdx[
Op.getImmTy()] =
I;
9651 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9654 AMDGPUOperand::ImmTyOpSel);
9657 AMDGPUOperand::ImmTyWaitEXP);
9663 cvtOpSelHelper(Inst, OpSel);
9666void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9668 OptionalImmIndexMap OptionalIdx;
9671 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9675 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9676 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9679 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9684 if (NumOperands == CbszOpIdx) {
9689 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9690 }
else if (
Op.isImmModifier()) {
9691 OptionalIdx[
Op.getImmTy()] =
I;
9693 Op.addRegOrImmOperands(Inst, 1);
9698 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9699 if (CbszIdx != OptionalIdx.end()) {
9700 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9704 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9705 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9706 if (BlgpIdx != OptionalIdx.end()) {
9707 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9718 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9719 if (OpselIdx != OptionalIdx.end()) {
9720 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9724 unsigned OpSelHi = 0;
9725 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9726 if (OpselHiIdx != OptionalIdx.end()) {
9727 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9730 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9731 AMDGPU::OpName::src1_modifiers};
9733 for (
unsigned J = 0; J < 2; ++J) {
9734 unsigned ModVal = 0;
9735 if (OpSel & (1 << J))
9737 if (OpSelHi & (1 << J))
9740 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9746 OptionalImmIndexMap &OptionalIdx) {
9751 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9752 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9756 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9758 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9759 }
else if (
Op.isImmModifier()) {
9760 OptionalIdx[
Op.getImmTy()] =
I;
9762 Op.addRegOrImmOperands(Inst, 1);
9768 AMDGPUOperand::ImmTyScaleSel);
9772 AMDGPUOperand::ImmTyClamp);
9778 AMDGPUOperand::ImmTyByteSel);
9783 AMDGPUOperand::ImmTyOModSI);
9790 auto *it = Inst.
begin();
9792 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9801 OptionalImmIndexMap OptionalIdx;
9802 cvtVOP3(Inst,
Operands, OptionalIdx);
9806 OptionalImmIndexMap &OptIdx) {
9811 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9812 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9813 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9814 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9815 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9816 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9817 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9818 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9819 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9820 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9829 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9830 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9833 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9834 if (BitOp3Idx != -1) {
9841 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9842 if (OpSelIdx != -1) {
9846 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9847 if (OpSelHiIdx != -1) {
9848 int DefaultVal =
IsPacked ? -1 : 0;
9854 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9855 if (MatrixAFMTIdx != -1) {
9857 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9861 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9862 if (MatrixBFMTIdx != -1) {
9864 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9867 int MatrixAScaleIdx =
9868 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9869 if (MatrixAScaleIdx != -1) {
9871 AMDGPUOperand::ImmTyMatrixAScale, 0);
9874 int MatrixBScaleIdx =
9875 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9876 if (MatrixBScaleIdx != -1) {
9878 AMDGPUOperand::ImmTyMatrixBScale, 0);
9881 int MatrixAScaleFmtIdx =
9882 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9883 if (MatrixAScaleFmtIdx != -1) {
9885 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9888 int MatrixBScaleFmtIdx =
9889 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9890 if (MatrixBScaleFmtIdx != -1) {
9892 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9897 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9901 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9903 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9907 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9911 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9912 AMDGPU::OpName::src2};
9913 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9914 AMDGPU::OpName::src1_modifiers,
9915 AMDGPU::OpName::src2_modifiers};
9918 unsigned OpSelHi = 0;
9925 if (OpSelHiIdx != -1)
9934 for (
int J = 0; J < 3; ++J) {
9935 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9939 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9949 uint32_t ModVal = 0;
9951 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9952 if (SrcOp.
isReg() && getMRI()
9959 if ((OpSel & (1 << J)) != 0)
9963 if ((OpSelHi & (1 << J)) != 0)
9966 if ((NegLo & (1 << J)) != 0)
9969 if ((NegHi & (1 << J)) != 0)
9977 OptionalImmIndexMap OptIdx;
9983 unsigned i,
unsigned Opc,
9985 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
9986 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
9988 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
9994 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9997 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9998 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10000 OptionalImmIndexMap OptIdx;
10001 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10002 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10003 OptIdx[
Op.getImmTy()] = i;
10008 AMDGPUOperand::ImmTyIndexKey8bit);
10012 AMDGPUOperand::ImmTyIndexKey16bit);
10016 AMDGPUOperand::ImmTyIndexKey32bit);
10033 SMLoc S = getLoc();
10036 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10037 SMLoc OpYLoc = getLoc();
10040 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10043 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10052 auto addOp = [&](uint16_t ParsedOprIdx) {
10053 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10055 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10059 Op.addRegOperands(Inst, 1);
10063 Op.addImmOperands(Inst, 1);
10075 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10079 const auto &CInfo = InstInfo[CompIdx];
10080 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10081 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10082 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10083 if (CInfo.hasSrc2Acc())
10084 addOp(CInfo.getIndexOfDstInParsedOperands());
10088 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10089 if (BitOp3Idx != -1) {
10090 OptionalImmIndexMap OptIdx;
10091 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10093 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10103bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10105bool AMDGPUOperand::isDPPCtrl()
const {
10106 using namespace AMDGPU::DPP;
10108 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10111 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10112 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10113 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10114 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10115 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10116 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10117 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10118 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10119 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10120 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10121 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10122 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10131bool AMDGPUOperand::isBLGP()
const {
10135bool AMDGPUOperand::isS16Imm()
const {
10139bool AMDGPUOperand::isU16Imm()
const {
10147bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10152 SMLoc Loc =
getToken().getEndLoc();
10153 Token = std::string(getTokenStr());
10155 if (getLoc() != Loc)
10160 if (!parseId(Suffix))
10164 StringRef DimId = Token;
10179 SMLoc S = getLoc();
10185 SMLoc Loc = getLoc();
10186 if (!parseDimId(Encoding))
10187 return Error(Loc,
"invalid dim value");
10190 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10199 SMLoc S = getLoc();
10208 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10211 for (
size_t i = 0; i < 8; ++i) {
10215 SMLoc Loc = getLoc();
10216 if (getParser().parseAbsoluteExpression(Sels[i]))
10218 if (0 > Sels[i] || 7 < Sels[i])
10219 return Error(Loc,
"expected a 3-bit value");
10222 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10226 for (
size_t i = 0; i < 8; ++i)
10227 DPP8 |= (Sels[i] << (i * 3));
10230 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10234bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10236 if (Ctrl ==
"row_newbcast")
10239 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10242 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10243 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10246 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10247 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10251int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10254 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10258 for (
int i = 0; i < 4; ++i) {
10263 SMLoc Loc = getLoc();
10264 if (getParser().parseAbsoluteExpression(Temp))
10266 if (Temp < 0 || Temp > 3) {
10267 Error(Loc,
"expected a 2-bit value");
10271 Val += (Temp << i * 2);
10274 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10280int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10281 using namespace AMDGPU::DPP;
10286 SMLoc Loc = getLoc();
10288 if (getParser().parseAbsoluteExpression(Val))
10291 struct DppCtrlCheck {
10297 DppCtrlCheck
Check =
10298 StringSwitch<DppCtrlCheck>(Ctrl)
10299 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10300 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10301 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10302 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10303 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10304 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10305 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10306 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10307 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10308 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10312 if (
Check.Ctrl == -1) {
10313 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10321 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10329 using namespace AMDGPU::DPP;
10332 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10335 SMLoc S = getLoc();
10341 if (Ctrl ==
"row_mirror") {
10342 Val = DppCtrl::ROW_MIRROR;
10343 }
else if (Ctrl ==
"row_half_mirror") {
10344 Val = DppCtrl::ROW_HALF_MIRROR;
10347 if (Ctrl ==
"quad_perm") {
10348 Val = parseDPPCtrlPerm();
10350 Val = parseDPPCtrlSel(Ctrl);
10359 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10365 OptionalImmIndexMap OptionalIdx;
10372 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10374 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10375 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10379 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10380 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10384 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10385 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10386 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10387 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10388 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10389 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10390 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10391 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10392 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10398 if (OldIdx == NumOperands) {
10400 constexpr int DST_IDX = 0;
10402 }
else if (Src2ModIdx == NumOperands) {
10412 if (IsVOP3CvtSrDpp) {
10421 if (TiedTo != -1) {
10426 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10428 if (IsDPP8 &&
Op.isDppFI()) {
10431 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10432 }
else if (
Op.isReg()) {
10433 Op.addRegOperands(Inst, 1);
10434 }
else if (
Op.isImm() &&
10436 Op.addImmOperands(Inst, 1);
10437 }
else if (
Op.isImm()) {
10438 OptionalIdx[
Op.getImmTy()] =
I;
10446 AMDGPUOperand::ImmTyClamp);
10452 AMDGPUOperand::ImmTyByteSel);
10457 AMDGPUOperand::ImmTyOModSI);
10460 cvtVOP3P(Inst,
Operands, OptionalIdx);
10462 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10465 AMDGPUOperand::ImmTyOpSel);
10470 AMDGPUOperand::ImmTyDPP8);
10471 using namespace llvm::AMDGPU::DPP;
10475 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10477 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10479 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10481 AMDGPUOperand::ImmTyDppBoundCtrl);
10485 AMDGPUOperand::ImmTyDppFI);
10491 OptionalImmIndexMap OptionalIdx;
10495 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10496 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10503 if (TiedTo != -1) {
10508 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10510 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10518 Op.addImmOperands(Inst, 1);
10520 Op.addRegWithFPInputModsOperands(Inst, 2);
10521 }
else if (
Op.isDppFI()) {
10523 }
else if (
Op.isReg()) {
10524 Op.addRegOperands(Inst, 1);
10530 Op.addRegWithFPInputModsOperands(Inst, 2);
10531 }
else if (
Op.isReg()) {
10532 Op.addRegOperands(Inst, 1);
10533 }
else if (
Op.isDPPCtrl()) {