70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
248 return isRegOrInline(RCID, type) || isLiteralImm(type);
251 bool isRegOrImmWithInt16InputMods()
const {
255 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
257 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
260 bool isRegOrImmWithInt32InputMods()
const {
264 bool isRegOrInlineImmWithInt16InputMods()
const {
265 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFP32InputMods()
const {
294 bool isRegOrImmWithFP64InputMods()
const {
298 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
299 return isRegOrInline(
300 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
303 bool isRegOrInlineImmWithFP32InputMods()
const {
304 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
307 bool isRegOrInlineImmWithFP64InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
311 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
313 bool isVRegWithFP32InputMods()
const {
314 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
317 bool isVRegWithFP64InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
321 bool isPackedFP16InputMods()
const {
325 bool isPackedVGPRFP32InputMods()
const {
329 bool isVReg()
const {
330 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
331 isRegClass(AMDGPU::VReg_64RegClassID) ||
332 isRegClass(AMDGPU::VReg_96RegClassID) ||
333 isRegClass(AMDGPU::VReg_128RegClassID) ||
334 isRegClass(AMDGPU::VReg_160RegClassID) ||
335 isRegClass(AMDGPU::VReg_192RegClassID) ||
336 isRegClass(AMDGPU::VReg_256RegClassID) ||
337 isRegClass(AMDGPU::VReg_512RegClassID) ||
338 isRegClass(AMDGPU::VReg_1024RegClassID);
341 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
343 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
345 bool isRsrcReg32()
const {
return isRegClass(AMDGPU::RsrcReg32RegClassID); }
347 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
349 bool isAV_LdSt_32_Align2_RegOp()
const {
350 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
351 isRegClass(AMDGPU::AGPR_32RegClassID);
354 bool isVRegWithInputMods()
const;
355 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
356 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
358 bool isSDWAOperand(MVT type)
const;
359 bool isSDWAFP16Operand()
const;
360 bool isSDWAFP32Operand()
const;
361 bool isSDWAInt16Operand()
const;
362 bool isSDWAInt32Operand()
const;
364 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
366 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
368 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
370 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
372 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
373 bool isDim()
const {
return isImmTy(ImmTyDim); }
374 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
375 bool isOff()
const {
return isImmTy(ImmTyOff); }
376 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
377 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
378 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
379 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
380 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
381 bool isFlatOffset()
const {
382 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
384 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
385 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
386 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
387 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
388 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
389 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
390 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
391 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
392 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
393 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
394 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
395 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
396 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
397 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
398 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
399 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
400 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
401 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
402 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
403 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
404 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
405 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
406 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
407 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
408 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
409 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
410 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
411 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
412 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
413 bool isDone()
const {
return isImmTy(ImmTyDone); }
414 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
416 bool isRegOrImm()
const {
return isReg() || isImm(); }
418 bool isRegClass(
unsigned RCID)
const;
422 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
423 return isRegOrInline(RCID, type) && !hasModifiers();
426 bool isSCSrcB16()
const {
427 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
430 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
432 bool isSCSrc_b32()
const {
433 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
436 bool isSCSrc_b64()
const {
437 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
440 bool isBoolReg()
const;
442 bool isSCSrcF16()
const {
443 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
446 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
448 bool isSCSrcF32()
const {
449 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
452 bool isSCSrcF64()
const {
453 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
456 bool isSSrc_b32()
const {
457 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
460 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
462 bool isSSrcV2B16()
const {
467 bool isSSrc_b64()
const {
470 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
471 (((
const MCTargetAsmParser *)AsmParser)
472 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
476 bool isSSrc_f32()
const {
477 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
480 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
482 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
484 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
486 bool isSSrcV2F16()
const {
491 bool isSSrcV2FP32()
const {
496 bool isSCSrcV2FP32()
const {
501 bool isSSrcV2INT32()
const {
506 bool isSCSrcV2INT32()
const {
508 return isSCSrc_b32();
511 bool isSSrcOrLds_b32()
const {
512 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
513 isLiteralImm(MVT::i32) || isExpr();
516 bool isVCSrc_b32()
const {
517 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
520 bool isVCSrc_b32_Lo256()
const {
521 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
524 bool isVCSrc_b64_Lo256()
const {
525 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
528 bool isVCSrc_b64()
const {
529 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
532 bool isVCSrcT_b16()
const {
533 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
536 bool isVCSrcTB16_Lo128()
const {
537 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
540 bool isVCSrcFake16B16_Lo128()
const {
541 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
544 bool isVCSrc_b16()
const {
545 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
548 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
550 bool isVCSrc_f32()
const {
551 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
554 bool isVCSrc_f64()
const {
555 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
558 bool isVCSrcTBF16()
const {
559 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
562 bool isVCSrcT_f16()
const {
563 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
566 bool isVCSrcT_bf16()
const {
567 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
570 bool isVCSrcTBF16_Lo128()
const {
571 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
574 bool isVCSrcTF16_Lo128()
const {
575 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
578 bool isVCSrcFake16BF16_Lo128()
const {
579 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
582 bool isVCSrcFake16F16_Lo128()
const {
583 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
586 bool isVCSrc_bf16()
const {
587 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
590 bool isVCSrc_f16()
const {
591 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
594 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
596 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
598 bool isVSrc_b32()
const {
599 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
602 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
604 bool isVSrc_v2b64()
const {
605 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
606 isLiteralImm(MVT::i64);
609 bool isVSrc_v2f64()
const {
610 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
611 isLiteralImm(MVT::f64);
614 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
616 bool isVSrcT_b16_Lo128()
const {
617 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
620 bool isVSrcFake16_b16_Lo128()
const {
621 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
624 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
626 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
628 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
630 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
632 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
634 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
636 bool isVSrc_f32()
const {
637 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
640 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
642 bool isVSrcT_bf16()
const {
643 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
646 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
648 bool isVSrcT_bf16_Lo128()
const {
649 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
652 bool isVSrcT_f16_Lo128()
const {
653 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
656 bool isVSrcFake16_bf16_Lo128()
const {
657 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
660 bool isVSrcFake16_f16_Lo128()
const {
661 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
664 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
666 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
668 bool isVSrc_v2bf16()
const {
669 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
672 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
674 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
676 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
678 bool isVISrcB32()
const {
679 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
682 bool isVISrcB16()
const {
683 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
686 bool isVISrcV2B16()
const {
return isVISrcB16(); }
688 bool isVISrcF32()
const {
689 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
692 bool isVISrcF16()
const {
693 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
696 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
698 bool isVISrc_64_bf16()
const {
699 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
702 bool isVISrc_64_f16()
const {
703 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
706 bool isVISrc_64_b32()
const {
707 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
710 bool isVISrc_64B64()
const {
711 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
714 bool isVISrc_64_f64()
const {
715 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
718 bool isVISrc_64V2FP32()
const {
719 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
722 bool isVISrc_64V2INT32()
const {
723 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
726 bool isVISrc_256_b32()
const {
727 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
730 bool isVISrc_256_f32()
const {
731 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
734 bool isVISrc_256B64()
const {
735 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
738 bool isVISrc_256_f64()
const {
739 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
742 bool isVISrc_512_f64()
const {
743 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
746 bool isVISrc_128B16()
const {
747 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
750 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
752 bool isVISrc_128_b32()
const {
753 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
756 bool isVISrc_128_f32()
const {
757 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
760 bool isVISrc_256V2FP32()
const {
761 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
764 bool isVISrc_256V2INT32()
const {
765 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
768 bool isVISrc_512_b32()
const {
769 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
772 bool isVISrc_512B16()
const {
773 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
776 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
778 bool isVISrc_512_f32()
const {
779 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
782 bool isVISrc_512F16()
const {
783 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
786 bool isVISrc_512V2F16()
const {
787 return isVISrc_512F16() || isVISrc_512_b32();
790 bool isVISrc_1024_b32()
const {
791 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
794 bool isVISrc_1024B16()
const {
795 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
798 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
800 bool isVISrc_1024_f32()
const {
801 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
804 bool isVISrc_1024F16()
const {
805 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
808 bool isVISrc_1024V2F16()
const {
809 return isVISrc_1024F16() || isVISrc_1024_b32();
812 bool isAISrcB32()
const {
813 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
816 bool isAISrcB16()
const {
817 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
820 bool isAISrcV2B16()
const {
return isAISrcB16(); }
822 bool isAISrcF32()
const {
823 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
826 bool isAISrcF16()
const {
827 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
830 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
832 bool isAISrc_64B64()
const {
833 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
836 bool isAISrc_64_f64()
const {
837 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
840 bool isAISrc_128_b32()
const {
841 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
844 bool isAISrc_128B16()
const {
845 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
848 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
850 bool isAISrc_128_f32()
const {
851 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
854 bool isAISrc_128F16()
const {
855 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
858 bool isAISrc_128V2F16()
const {
859 return isAISrc_128F16() || isAISrc_128_b32();
862 bool isVISrc_128_bf16()
const {
863 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
866 bool isVISrc_128_f16()
const {
867 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
870 bool isVISrc_128V2F16()
const {
871 return isVISrc_128_f16() || isVISrc_128_b32();
874 bool isAISrc_256B64()
const {
875 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
878 bool isAISrc_256_f64()
const {
879 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
882 bool isAISrc_512_b32()
const {
883 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
886 bool isAISrc_512B16()
const {
887 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
890 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
892 bool isAISrc_512_f32()
const {
893 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
896 bool isAISrc_512F16()
const {
897 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
900 bool isAISrc_512V2F16()
const {
901 return isAISrc_512F16() || isAISrc_512_b32();
904 bool isAISrc_1024_b32()
const {
905 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
908 bool isAISrc_1024B16()
const {
909 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
912 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
914 bool isAISrc_1024_f32()
const {
915 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
918 bool isAISrc_1024F16()
const {
919 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
922 bool isAISrc_1024V2F16()
const {
923 return isAISrc_1024F16() || isAISrc_1024_b32();
926 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
928 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
930 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
932 bool isMem()
const override {
return false; }
934 bool isExpr()
const {
return Kind == Expression; }
936 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
938 bool isSWaitCnt()
const;
939 bool isDepCtr()
const;
940 bool isSDelayALU()
const;
941 bool isHwreg()
const;
942 bool isSendMsg()
const;
943 bool isWaitEvent()
const;
944 bool isSplitBarrier()
const;
945 bool isSwizzle()
const;
946 bool isSMRDOffset8()
const;
947 bool isSMEMOffset()
const;
948 bool isSMRDLiteralOffset()
const;
950 bool isDPPCtrl()
const;
952 bool isGPRIdxMode()
const;
953 bool isS16Imm()
const;
954 bool isU16Imm()
const;
955 bool isEndpgm()
const;
957 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
958 return [
this,
P]() {
return P(*
this); };
963 return StringRef(Tok.Data, Tok.Length);
971 void setImm(int64_t Val) {
976 ImmTy getImmTy()
const {
981 MCRegister
getReg()
const override {
986 SMLoc getStartLoc()
const override {
return StartLoc; }
988 SMLoc getEndLoc()
const override {
return EndLoc; }
990 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
992 int getMCOpIdx()
const {
return MCOpIdx; }
994 Modifiers getModifiers()
const {
995 assert(isRegKind() || isImmTy(ImmTyNone));
996 return isRegKind() ?
Reg.Mods :
Imm.Mods;
999 void setModifiers(Modifiers Mods) {
1000 assert(isRegKind() || isImmTy(ImmTyNone));
1007 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1009 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1011 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1013 bool isForcedLit()
const {
1014 return isImmLiteral() && getModifiers().isForcedLit();
1017 bool isForcedLit64()
const {
1018 return isImmLiteral() && getModifiers().isForcedLit64();
1023 void addImmOperands(MCInst &Inst,
unsigned N,
1024 bool ApplyModifiers =
true)
const;
1026 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1027 bool ApplyModifiers)
const;
1029 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1031 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1033 addRegOperands(Inst,
N);
1035 addImmOperands(Inst,
N);
1038 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1039 Modifiers Mods = getModifiers();
1042 addRegOperands(Inst,
N);
1044 addImmOperands(Inst,
N,
false);
1048 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1049 assert(!hasIntModifiers());
1050 addRegOrImmWithInputModsOperands(Inst,
N);
1053 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1054 assert(!hasFPModifiers());
1055 addRegOrImmWithInputModsOperands(Inst,
N);
1058 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1059 Modifiers Mods = getModifiers();
1062 addRegOperands(Inst,
N);
1065 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1066 assert(!hasIntModifiers());
1067 addRegWithInputModsOperands(Inst,
N);
1070 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1071 assert(!hasFPModifiers());
1072 addRegWithInputModsOperands(Inst,
N);
1075 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1078 case ImmTyNone: OS <<
"None";
break;
1079 case ImmTyGDS: OS <<
"GDS";
break;
1080 case ImmTyLDS: OS <<
"LDS";
break;
1081 case ImmTyOffen: OS <<
"Offen";
break;
1082 case ImmTyIdxen: OS <<
"Idxen";
break;
1083 case ImmTyAddr64: OS <<
"Addr64";
break;
1084 case ImmTyOffset: OS <<
"Offset";
break;
1085 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1086 case ImmTyOffset0: OS <<
"Offset0";
break;
1087 case ImmTyOffset1: OS <<
"Offset1";
break;
1088 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1089 case ImmTyCPol: OS <<
"CPol";
break;
1090 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1091 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1092 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1093 case ImmTyTFE: OS <<
"TFE";
break;
1094 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1095 case ImmTyD16: OS <<
"D16";
break;
1096 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1097 case ImmTyClamp: OS <<
"Clamp";
break;
1098 case ImmTyOModSI: OS <<
"OModSI";
break;
1099 case ImmTyDPP8: OS <<
"DPP8";
break;
1100 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1101 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1102 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1103 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1104 case ImmTyDppFI: OS <<
"DppFI";
break;
1105 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1106 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1107 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1108 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1109 case ImmTyDMask: OS <<
"DMask";
break;
1110 case ImmTyDim: OS <<
"Dim";
break;
1111 case ImmTyUNorm: OS <<
"UNorm";
break;
1112 case ImmTyDA: OS <<
"DA";
break;
1113 case ImmTyR128A16: OS <<
"R128A16";
break;
1114 case ImmTyA16: OS <<
"A16";
break;
1115 case ImmTyLWE: OS <<
"LWE";
break;
1116 case ImmTyOff: OS <<
"Off";
break;
1117 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1118 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1119 case ImmTyExpVM: OS <<
"ExpVM";
break;
1120 case ImmTyDone: OS <<
"Done";
break;
1121 case ImmTyRowEn: OS <<
"RowEn";
break;
1122 case ImmTyHwreg: OS <<
"Hwreg";
break;
1123 case ImmTySendMsg: OS <<
"SendMsg";
break;
1124 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1125 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1126 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1127 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1128 case ImmTyOpSel: OS <<
"OpSel";
break;
1129 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1130 case ImmTyNegLo: OS <<
"NegLo";
break;
1131 case ImmTyNegHi: OS <<
"NegHi";
break;
1132 case ImmTySwizzle: OS <<
"Swizzle";
break;
1133 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1134 case ImmTyHigh: OS <<
"High";
break;
1135 case ImmTyBLGP: OS <<
"BLGP";
break;
1136 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1137 case ImmTyABID: OS <<
"ABID";
break;
1138 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1139 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1140 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1141 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1142 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1143 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1144 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1145 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1146 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1147 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1148 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1149 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1150 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1151 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1152 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1153 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1158 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1162 <<
" mods: " <<
Reg.Mods <<
'>';
1166 if (getImmTy() != ImmTyNone) {
1168 printImmTy(OS, getImmTy());
1170 OS <<
" mods: " <<
Imm.Mods <<
'>';
1183 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1184 int64_t Val, SMLoc Loc,
1185 ImmTy
Type = ImmTyNone,
1186 bool IsFPImm =
false) {
1187 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1189 Op->Imm.IsFPImm = IsFPImm;
1191 Op->Imm.Mods = Modifiers();
1197 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1198 StringRef Str, SMLoc Loc,
1199 bool HasExplicitEncodingSize =
true) {
1200 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1201 Res->Tok.Data = Str.data();
1202 Res->Tok.Length = Str.size();
1203 Res->StartLoc = Loc;
1208 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1209 MCRegister
Reg, SMLoc S, SMLoc
E) {
1210 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1211 Op->Reg.RegNo =
Reg;
1212 Op->Reg.Mods = Modifiers();
1218 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1219 const class MCExpr *Expr, SMLoc S) {
1220 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1229 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1238#define GET_REGISTER_MATCHER
1239#include "AMDGPUGenAsmMatcher.inc"
1240#undef GET_REGISTER_MATCHER
1241#undef GET_SUBTARGET_FEATURE_NAME
1246class KernelScopeInfo {
1247 int SgprIndexUnusedMin = -1;
1248 int VgprIndexUnusedMin = -1;
1249 int AgprIndexUnusedMin = -1;
1253 void usesSgprAt(
int i) {
1254 if (i >= SgprIndexUnusedMin) {
1255 SgprIndexUnusedMin = ++i;
1258 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1264 void usesVgprAt(
int i) {
1265 if (i >= VgprIndexUnusedMin) {
1266 VgprIndexUnusedMin = ++i;
1269 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1271 VgprIndexUnusedMin);
1277 void usesAgprAt(
int i) {
1282 if (i >= AgprIndexUnusedMin) {
1283 AgprIndexUnusedMin = ++i;
1286 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1291 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1293 VgprIndexUnusedMin);
1300 KernelScopeInfo() =
default;
1304 MSTI = Ctx->getSubtargetInfo();
1306 usesSgprAt(SgprIndexUnusedMin = -1);
1307 usesVgprAt(VgprIndexUnusedMin = -1);
1309 usesAgprAt(AgprIndexUnusedMin = -1);
1313 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1314 unsigned RegWidth) {
1317 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1320 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1323 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1332 MCAsmParser &Parser;
1334 unsigned ForcedEncodingSize = 0;
1335 bool ForcedDPP =
false;
1336 bool ForcedSDWA =
false;
1337 KernelScopeInfo KernelScope;
1338 const unsigned HwMode;
1340 const AMDGPU::IsaVersion ISA;
1345#define GET_ASSEMBLER_HEADER
1346#include "AMDGPUGenAsmMatcher.inc"
1351 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1353 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1357 std::optional<AMDGPU::InfoSectionData> InfoData;
1364 bool TargetDirectiveEmitted =
false;
1373 SmallVector<unsigned> OpcodeStream;
1375 OpcodeStreamSymbols;
1376 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1379 void checkKernelPrologues();
1382 void createConstantSymbol(StringRef Id, int64_t Val);
1384 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1385 bool OutOfRangeError(SMRange
Range);
1401 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1402 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1403 std::optional<bool> EnableWavefrontSize32,
1404 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1405 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1406 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1407 bool ParseDirectiveAMDGCNTarget();
1408 bool ParseDirectiveAMDHSACodeObjectVersion();
1409 bool ParseDirectiveAMDHSAKernel();
1410 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1411 bool ParseDirectiveAMDKernelCodeT();
1413 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1414 bool ParseDirectiveAMDGPUHsaKernel();
1416 bool ParseDirectiveISAVersion();
1417 bool ParseDirectiveHSAMetadata();
1418 bool ParseDirectivePALMetadataBegin();
1419 bool ParseDirectivePALMetadata();
1420 bool ParseDirectiveAMDGPULDS();
1421 bool ParseDirectiveAMDGPUInfo();
1425 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1426 const char *AssemblerDirectiveEnd,
1427 std::string &CollectString);
1429 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1430 RegisterKind RegKind, MCRegister Reg1,
1431 RegisterKind RegKind1, SMLoc Loc);
1432 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1433 unsigned &RegNum,
unsigned &RegWidth,
1434 bool RestoreOnFailure =
false);
1435 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1436 unsigned &RegNum,
unsigned &RegWidth,
1437 SmallVectorImpl<AsmToken> &Tokens);
1438 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1440 SmallVectorImpl<AsmToken> &Tokens);
1441 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1443 SmallVectorImpl<AsmToken> &Tokens);
1444 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1446 SmallVectorImpl<AsmToken> &Tokens);
1447 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1448 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1449 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1452 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1453 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1454 void initializeGprCountSymbol(RegisterKind RegKind);
1455 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1461 OperandMode_Default,
1465 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1467 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1468 const MCInstrInfo &MII)
1469 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1470 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1475 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1477 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1478 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1479 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1480 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1482 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1483 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1484 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1486 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1487 initializeGprCountSymbol(IS_VGPR);
1488 initializeGprCountSymbol(IS_SGPR);
1493 createConstantSymbol(Symbol, Code);
1495 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1496 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1497 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1545 bool hasBVHRayTracingInsts()
const {
1546 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1551 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1553 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1555 bool hasInv2PiInlineImm()
const {
1556 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1559 bool has64BitLiterals()
const {
1560 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1563 bool hasFlatOffsets()
const {
1564 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1567 bool hasTrue16Insts()
const {
1568 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1572 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1575 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1577 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1579 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1581 bool hasPartialNSAEncoding()
const {
1582 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1585 bool hasGloballyAddressableScratch()
const {
1586 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1599 AMDGPUTargetStreamer &getTargetStreamer() {
1600 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1601 return static_cast<AMDGPUTargetStreamer &
>(TS);
1607 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1610 const MCRegisterInfo *getMRI()
const {
1614 const MCInstrInfo *getMII()
const {
return &MII; }
1618 const FeatureBitset &getFeatureBits()
const {
1619 return getSTI().getFeatureBits();
1622 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1623 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1624 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1626 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1627 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1628 bool isForcedDPP()
const {
return ForcedDPP; }
1629 bool isForcedSDWA()
const {
return ForcedSDWA; }
1630 ArrayRef<unsigned> getMatchedVariants()
const;
1631 StringRef getMatchedVariantName()
const;
1633 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1634 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1635 bool RestoreOnFailure);
1636 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1637 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1638 SMLoc &EndLoc)
override;
1639 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1640 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1641 unsigned Kind)
override;
1642 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1645 bool MatchingInlineAsm)
override;
1646 bool ParseDirective(AsmToken DirectiveID)
override;
1647 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1648 void onEndOfFile()
override;
1650 OperandMode
Mode = OperandMode_Default);
1651 StringRef parseMnemonicSuffix(StringRef Name);
1652 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1658 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1662 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1663 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1665 ParseStatus parseOperandArrayWithPrefix(
1667 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1668 bool (*ConvertResult)(int64_t &) =
nullptr);
1672 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1673 bool IgnoreNegative =
false);
1674 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1678 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1682 ArrayRef<const char *> Ids,
1686 ArrayRef<const char *> Ids,
1687 AMDGPUOperand::ImmTy
Type);
1690 bool isOperandModifier(
const AsmToken &Token,
1691 const AsmToken &NextToken)
const;
1692 bool isRegOrOperandModifier(
const AsmToken &Token,
1693 const AsmToken &NextToken)
const;
1694 bool isNamedOperandModifier(
const AsmToken &Token,
1695 const AsmToken &NextToken)
const;
1696 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1697 const AsmToken &NextToken)
const;
1698 bool parseSP3NegModifier();
1705 bool AllowImm =
true);
1707 bool AllowImm =
true);
1713 AMDGPUOperand::ImmTy ImmTy);
1718 AMDGPUOperand::ImmTy
Type);
1722 AMDGPUOperand::ImmTy
Type);
1726 AMDGPUOperand::ImmTy
Type);
1730 ParseStatus parseDfmtNfmt(int64_t &
Format);
1731 ParseStatus parseUfmt(int64_t &
Format);
1732 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1734 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1737 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1738 ParseStatus parseNumericFormat(int64_t &
Format);
1742 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1743 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1748 bool parseCnt(int64_t &IntVal);
1751 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1752 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1755 bool parseDelay(int64_t &Delay);
1761 struct OperandInfoTy {
1764 bool IsSymbolic =
false;
1765 bool IsDefined =
false;
1767 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1770 struct StructuredOpField : OperandInfoTy {
1774 bool IsDefined =
false;
1776 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1777 unsigned Width, int64_t
Default)
1778 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1779 virtual ~StructuredOpField() =
default;
1781 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1782 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1786 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1788 return Error(Parser,
"not supported on this GPU");
1790 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1798 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1799 OperandInfoTy &Stream);
1800 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1801 const OperandInfoTy &Stream);
1803 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1804 OperandInfoTy &Width);
1809 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1816 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1818 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1822 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1828 bool validateConstantBusLimitations(
const MCInst &Inst,
1830 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1833 bool tryVOPD(
const MCInst &Inst);
1834 bool tryVOPD3(
const MCInst &Inst);
1835 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1837 bool validateIntClampSupported(
const MCInst &Inst);
1838 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1839 bool validateMIMGGatherDMask(
const MCInst &Inst);
1841 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1842 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1843 bool validateMIMGD16(
const MCInst &Inst);
1845 bool validateTensorR128(
const MCInst &Inst);
1846 bool validateMIMGMSAA(
const MCInst &Inst);
1847 bool validateOpSel(
const MCInst &Inst);
1848 bool validateTrue16OpSel(
const MCInst &Inst);
1849 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1851 bool validateVccOperand(MCRegister
Reg)
const;
1856 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1857 bool validateVGPRAlign(
const MCInst &Inst)
const;
1861 bool validateDivScale(
const MCInst &Inst);
1866 const unsigned CPol);
1871 bool validateClusterBarrierIsFirst(
const MCInst &Inst,
1873 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1874 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1875 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1876 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1878 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1879 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1880 ArrayRef<unsigned> Variants);
1881 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1883 bool isId(
const StringRef Id)
const;
1884 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1886 StringRef getId()
const;
1887 bool trySkipId(
const StringRef Id);
1888 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1892 bool parseString(StringRef &Val,
1893 const StringRef ErrMsg =
"expected a string");
1894 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1900 StringRef getTokenStr()
const;
1901 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1903 SMLoc getLoc()
const;
1907 void onBeginOfFile()
override;
1911 void emitTargetDirective();
1912 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1924 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1925 const unsigned MaxVal,
const Twine &ErrMsg,
1927 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1928 const unsigned MinVal,
const unsigned MaxVal,
1929 const StringRef ErrMsg);
1931 bool parseSwizzleOffset(int64_t &
Imm);
1932 bool parseSwizzleMacro(int64_t &
Imm);
1933 bool parseSwizzleQuadPerm(int64_t &
Imm);
1934 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1935 bool parseSwizzleBroadcast(int64_t &
Imm);
1936 bool parseSwizzleSwap(int64_t &
Imm);
1937 bool parseSwizzleReverse(int64_t &
Imm);
1938 bool parseSwizzleFFT(int64_t &
Imm);
1939 bool parseSwizzleRotate(int64_t &
Imm);
1942 int64_t parseGPRIdxMacro();
1945 cvtMubufImpl(Inst,
Operands,
false);
1948 cvtMubufImpl(Inst,
Operands,
true);
1954 OptionalImmIndexMap &OptionalIdx);
1963 OptionalImmIndexMap &OptionalIdx);
1965 OptionalImmIndexMap &OptionalIdx);
1969 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1971 bool parseDimId(
unsigned &Encoding);
1973 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1977 int64_t parseDPPCtrlSel(StringRef Ctrl);
1978 int64_t parseDPPCtrlPerm();
1984 bool IsDPP8 =
false);
1990 AMDGPUOperand::ImmTy
Type);
1998 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
2001 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
2002 bool SkipSrcVcc =
false);
2111bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2121 if (!isImmTy(ImmTyNone)) {
2126 if (getModifiers().
Lit != LitModifier::None)
2136 if (type == MVT::f64 || type == MVT::i64) {
2138 AsmParser->hasInv2PiInlineImm());
2141 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2160 APFloat::rmNearestTiesToEven, &Lost);
2167 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2169 AsmParser->hasInv2PiInlineImm());
2174 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2175 AsmParser->hasInv2PiInlineImm());
2179 if (type == MVT::f64 || type == MVT::i64) {
2181 AsmParser->hasInv2PiInlineImm());
2190 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2191 AsmParser->hasInv2PiInlineImm());
2195 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2196 AsmParser->hasInv2PiInlineImm());
2199bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2201 if (!isImmTy(ImmTyNone)) {
2206 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2211 if (type == MVT::f64 && hasFPModifiers()) {
2231 if (type == MVT::f64) {
2236 if (type == MVT::i64) {
2249 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2250 : (type == MVT::v2i16) ? MVT::f32
2251 : (type == MVT::v2f32) ? MVT::f32
2254 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2258bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2259 return isRegKind() &&
2260 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2263bool AMDGPUOperand::isVRegWithInputMods()
const {
2264 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2266 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2267 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2270template <
bool IsFake16>
2271bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2272 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2273 : AMDGPU::VGPR_16_Lo128RegClassID);
2276template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2277 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2278 : AMDGPU::VGPR_16RegClassID);
2281bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2282 if (AsmParser->isVI())
2284 if (AsmParser->isGFX9Plus())
2285 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2289bool AMDGPUOperand::isSDWAFP16Operand()
const {
2290 return isSDWAOperand(MVT::f16);
2293bool AMDGPUOperand::isSDWAFP32Operand()
const {
2294 return isSDWAOperand(MVT::f32);
2297bool AMDGPUOperand::isSDWAInt16Operand()
const {
2298 return isSDWAOperand(MVT::i16);
2301bool AMDGPUOperand::isSDWAInt32Operand()
const {
2302 return isSDWAOperand(MVT::i32);
2305bool AMDGPUOperand::isBoolReg()
const {
2306 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2307 (AsmParser->isWave32() && isSCSrc_b32()));
2311 unsigned Size)
const {
2312 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2327void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2328 bool ApplyModifiers)
const {
2338 addLiteralImmOperand(Inst,
Imm.Val,
2339 ApplyModifiers & isImmTy(ImmTyNone) &&
2340 Imm.Mods.hasFPModifiers());
2342 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2347void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2348 bool ApplyModifiers)
const {
2349 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2354 if (ApplyModifiers) {
2356 const unsigned Size =
2358 Val = applyInputFPModifiers(Val,
Size);
2362 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2364 bool CanUse64BitLiterals =
2367 MCContext &Ctx = AsmParser->getContext();
2378 if (
Lit == LitModifier::None &&
2380 AsmParser->hasInv2PiInlineImm())) {
2388 bool HasMandatoryLiteral =
2391 if (
Literal.getLoBits(32) != 0 &&
2392 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2393 !HasMandatoryLiteral) {
2394 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2396 "Can't encode literal as exact 64-bit floating-point operand. "
2397 "Low 32-bits will be set to zero");
2398 Val &= 0xffffffff00000000u;
2404 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2410 Lit = LitModifier::Lit64;
2411 }
else if (
Lit == LitModifier::Lit) {
2425 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2427 Lit = LitModifier::Lit64;
2434 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2435 Literal == 0x3fc45f306725feed) {
2474 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2481 if (
Lit != LitModifier::None) {
2512 if (
Lit == LitModifier::None &&
2522 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2530 if (
Lit == LitModifier::None &&
2538 if (!AsmParser->has64BitLiterals()) {
2539 Val =
static_cast<uint64_t>(Val) << 32;
2546 if (
Lit == LitModifier::Lit ||
2548 Val =
static_cast<uint64_t>(Val) << 32;
2552 if (
Lit == LitModifier::Lit)
2578 if (
Lit != LitModifier::None) {
2586void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2592bool AMDGPUOperand::isInlineValue()
const {
2600void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2611 if (Is == IS_VGPR) {
2616 return AMDGPU::VGPR_32RegClassID;
2618 return AMDGPU::VReg_64RegClassID;
2620 return AMDGPU::VReg_96RegClassID;
2622 return AMDGPU::VReg_128RegClassID;
2624 return AMDGPU::VReg_160RegClassID;
2626 return AMDGPU::VReg_192RegClassID;
2628 return AMDGPU::VReg_224RegClassID;
2630 return AMDGPU::VReg_256RegClassID;
2632 return AMDGPU::VReg_288RegClassID;
2634 return AMDGPU::VReg_320RegClassID;
2636 return AMDGPU::VReg_352RegClassID;
2638 return AMDGPU::VReg_384RegClassID;
2640 return AMDGPU::VReg_512RegClassID;
2642 return AMDGPU::VReg_1024RegClassID;
2644 }
else if (Is == IS_TTMP) {
2649 return AMDGPU::TTMP_32RegClassID;
2651 return AMDGPU::TTMP_64RegClassID;
2653 return AMDGPU::TTMP_128RegClassID;
2655 return AMDGPU::TTMP_256RegClassID;
2657 return AMDGPU::TTMP_512RegClassID;
2659 }
else if (Is == IS_SGPR) {
2664 return AMDGPU::SGPR_32RegClassID;
2666 return AMDGPU::SGPR_64RegClassID;
2668 return AMDGPU::SGPR_96RegClassID;
2670 return AMDGPU::SGPR_128RegClassID;
2672 return AMDGPU::SGPR_160RegClassID;
2674 return AMDGPU::SGPR_192RegClassID;
2676 return AMDGPU::SGPR_224RegClassID;
2678 return AMDGPU::SGPR_256RegClassID;
2680 return AMDGPU::SGPR_288RegClassID;
2682 return AMDGPU::SGPR_320RegClassID;
2684 return AMDGPU::SGPR_352RegClassID;
2686 return AMDGPU::SGPR_384RegClassID;
2688 return AMDGPU::SGPR_512RegClassID;
2690 }
else if (Is == IS_AGPR) {
2695 return AMDGPU::AGPR_32RegClassID;
2697 return AMDGPU::AReg_64RegClassID;
2699 return AMDGPU::AReg_96RegClassID;
2701 return AMDGPU::AReg_128RegClassID;
2703 return AMDGPU::AReg_160RegClassID;
2705 return AMDGPU::AReg_192RegClassID;
2707 return AMDGPU::AReg_224RegClassID;
2709 return AMDGPU::AReg_256RegClassID;
2711 return AMDGPU::AReg_288RegClassID;
2713 return AMDGPU::AReg_320RegClassID;
2715 return AMDGPU::AReg_352RegClassID;
2717 return AMDGPU::AReg_384RegClassID;
2719 return AMDGPU::AReg_512RegClassID;
2721 return AMDGPU::AReg_1024RegClassID;
2729 .
Case(
"exec", AMDGPU::EXEC)
2730 .
Case(
"vcc", AMDGPU::VCC)
2731 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2732 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2733 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2734 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2735 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2736 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2737 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2738 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2739 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2740 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2741 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2742 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2743 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2744 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2745 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2746 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2747 .
Case(
"m0", AMDGPU::M0)
2748 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2749 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2750 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2751 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2752 .
Case(
"scc", AMDGPU::SRC_SCC)
2753 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2754 .
Case(
"tba", AMDGPU::TBA)
2755 .
Case(
"tma", AMDGPU::TMA)
2756 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2757 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2758 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2759 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2760 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2761 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2762 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2763 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2764 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2765 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2766 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2767 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2768 .
Case(
"pc", AMDGPU::PC_REG)
2769 .
Case(
"null", AMDGPU::SGPR_NULL)
2773bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2774 SMLoc &EndLoc,
bool RestoreOnFailure) {
2775 auto R = parseRegister();
2779 RegNo =
R->getReg();
2780 StartLoc =
R->getStartLoc();
2781 EndLoc =
R->getEndLoc();
2785bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2787 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2790ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2792 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2793 bool PendingErrors = getParser().hasPendingError();
2794 getParser().clearPendingErrors();
2802bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2803 RegisterKind RegKind,
2805 RegisterKind RegKind1, SMLoc Loc) {
2807 if (RegKind == IS_SGPR) {
2808 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2809 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2810 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2816 if (RegKind != RegKind1) {
2817 Error(Loc,
"registers in a list must be of the same kind");
2823 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2828 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2829 Reg = AMDGPU::FLAT_SCR;
2833 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2834 Reg = AMDGPU::XNACK_MASK;
2838 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2843 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2848 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2853 Error(Loc,
"register does not fit in the list");
2859 if (Reg1 !=
Reg + RegWidth / 32) {
2860 Error(Loc,
"registers in a list must have consecutive indices");
2876 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2877 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2881 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2887 if (Str.starts_with(
Reg.Name))
2893 return !Str.getAsInteger(10, Num);
2896bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2897 const AsmToken &NextToken)
const {
2912 StringRef RegSuffix = Str.substr(
RegName.size());
2913 if (!RegSuffix.
empty()) {
2930bool AMDGPUAsmParser::isRegister() {
2931 return isRegister(
getToken(), peekToken());
2934MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2935 unsigned SubReg,
unsigned RegWidth,
2939 unsigned AlignSize = 1;
2940 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2946 if (RegNum % AlignSize != 0) {
2947 Error(Loc,
"invalid register alignment");
2948 return MCRegister();
2951 unsigned RegIdx = RegNum / AlignSize;
2954 Error(Loc,
"invalid or unsupported register size");
2955 return MCRegister();
2959 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2960 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2961 Error(Loc,
"register index is out of range");
2962 return AMDGPU::NoRegister;
2965 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2966 Error(Loc,
"register index is out of range");
2967 return MCRegister();
2983bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2985 int64_t RegLo, RegHi;
2989 SMLoc FirstIdxLoc = getLoc();
2996 SecondIdxLoc = getLoc();
3007 Error(FirstIdxLoc,
"invalid register index");
3012 Error(SecondIdxLoc,
"invalid register index");
3016 if (RegLo > RegHi) {
3017 Error(FirstIdxLoc,
"first register index should not exceed second index");
3021 if (RegHi == RegLo) {
3022 StringRef RegSuffix = getTokenStr();
3023 if (RegSuffix ==
".l") {
3024 SubReg = AMDGPU::lo16;
3026 }
else if (RegSuffix ==
".h") {
3027 SubReg = AMDGPU::hi16;
3032 Num =
static_cast<unsigned>(RegLo);
3033 RegWidth = 32 * ((RegHi - RegLo) + 1);
3038MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3041 SmallVectorImpl<AsmToken> &Tokens) {
3047 RegKind = IS_SPECIAL;
3054MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3057 SmallVectorImpl<AsmToken> &Tokens) {
3059 StringRef
RegName = getTokenStr();
3060 auto Loc = getLoc();
3064 Error(Loc,
"invalid register name");
3065 return MCRegister();
3073 unsigned SubReg = NoSubRegister;
3074 bool IsRange =
false;
3075 if (!RegSuffix.
empty()) {
3077 SubReg = AMDGPU::lo16;
3079 SubReg = AMDGPU::hi16;
3083 Error(Loc,
"invalid register index");
3084 return MCRegister();
3090 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3091 return MCRegister();
3095 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3096 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3097 if (RegKind == IS_SGPR && IsRange
3098 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3099 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3100 Error(Loc,
"register index is out of range");
3101 return MCRegister();
3107MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3108 unsigned &RegNum,
unsigned &RegWidth,
3109 SmallVectorImpl<AsmToken> &Tokens) {
3111 auto ListLoc = getLoc();
3114 "expected a register or a list of registers")) {
3115 return MCRegister();
3120 auto Loc = getLoc();
3121 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3122 return MCRegister();
3123 if (RegWidth != 32) {
3124 Error(Loc,
"expected a single 32-bit register");
3125 return MCRegister();
3129 RegisterKind NextRegKind;
3131 unsigned NextRegNum, NextRegWidth;
3134 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3136 return MCRegister();
3138 if (NextRegWidth != 32) {
3139 Error(Loc,
"expected a single 32-bit register");
3140 return MCRegister();
3142 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3144 return MCRegister();
3148 "expected a comma or a closing square bracket")) {
3149 return MCRegister();
3153 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3158bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3159 MCRegister &
Reg,
unsigned &RegNum,
3161 SmallVectorImpl<AsmToken> &Tokens) {
3162 auto Loc = getLoc();
3166 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3168 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3170 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3175 assert(Parser.hasPendingError());
3179 if (!subtargetHasRegister(*
TRI,
Reg)) {
3180 if (
Reg == AMDGPU::SGPR_NULL) {
3181 Error(Loc,
"'null' operand is not supported on this GPU");
3184 " register not available on this GPU");
3192bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3193 MCRegister &
Reg,
unsigned &RegNum,
3195 bool RestoreOnFailure ) {
3199 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3200 if (RestoreOnFailure) {
3201 while (!Tokens.
empty()) {
3210std::optional<StringRef>
3211AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3214 return StringRef(
".amdgcn.next_free_vgpr");
3216 return StringRef(
".amdgcn.next_free_sgpr");
3218 return std::nullopt;
3222void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3223 auto SymbolName = getGprCountSymbolName(RegKind);
3224 assert(SymbolName &&
"initializing invalid register kind");
3230bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3231 unsigned DwordRegIndex,
3232 unsigned RegWidth) {
3237 auto SymbolName = getGprCountSymbolName(RegKind);
3242 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3246 return !
Error(getLoc(),
3247 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3251 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3253 if (OldCount <= NewMax)
3259std::unique_ptr<AMDGPUOperand>
3260AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3262 SMLoc StartLoc = Tok.getLoc();
3263 SMLoc EndLoc = Tok.getEndLoc();
3264 RegisterKind RegKind;
3266 unsigned RegNum, RegWidth;
3268 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3272 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3275 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3276 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3283 if (isRegister() || isModifier())
3286 if (
Lit == LitModifier::None) {
3287 if (trySkipId(
"lit"))
3288 Lit = LitModifier::Lit;
3289 else if (trySkipId(
"lit64"))
3290 Lit = LitModifier::Lit64;
3292 if (
Lit != LitModifier::None) {
3295 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3304 const auto &NextTok = peekToken();
3307 bool Negate =
false;
3315 AMDGPUOperand::Modifiers Mods;
3323 StringRef Num = getTokenStr();
3326 APFloat RealVal(APFloat::IEEEdouble());
3327 auto roundMode = APFloat::rmNearestTiesToEven;
3328 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3331 RealVal.changeSign();
3334 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3335 S, AMDGPUOperand::ImmTyNone,
true));
3336 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3337 Op.setModifiers(Mods);
3346 if (HasSP3AbsModifier) {
3355 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3358 if (Parser.parseExpression(Expr))
3362 if (Expr->evaluateAsAbsolute(IntVal)) {
3364 return Error(S,
"literal value out of range");
3365 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3366 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3367 Op.setModifiers(Mods);
3369 if (
Lit != LitModifier::None)
3371 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3384 if (
auto R = parseRegister()) {
3394 ParseStatus Res = parseReg(
Operands);
3402bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3403 const AsmToken &NextToken)
const {
3406 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3411bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3412 const AsmToken &NextToken)
const {
3416bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3417 const AsmToken &NextToken)
const {
3418 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3421bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3422 const AsmToken &NextToken)
const {
3423 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3439bool AMDGPUAsmParser::isModifier() {
3442 AsmToken NextToken[2];
3443 peekTokens(NextToken);
3445 return isOperandModifier(Tok, NextToken[0]) ||
3447 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3448 isOpcodeModifierWithVal(Tok, NextToken[0]);
3473bool AMDGPUAsmParser::parseSP3NegModifier() {
3475 AsmToken NextToken[2];
3476 peekTokens(NextToken);
3479 (isRegister(NextToken[0], NextToken[1]) ||
3497 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3499 SP3Neg = parseSP3NegModifier();
3502 Neg = trySkipId(
"neg");
3504 return Error(Loc,
"expected register or immediate");
3508 Abs = trySkipId(
"abs");
3513 if (trySkipId(
"lit")) {
3514 Lit = LitModifier::Lit;
3517 }
else if (trySkipId(
"lit64")) {
3518 Lit = LitModifier::Lit64;
3521 if (!has64BitLiterals())
3522 return Error(Loc,
"lit64 is not supported on this GPU");
3528 return Error(Loc,
"expected register or immediate");
3537 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3541 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3542 Error(Loc,
"expected immediate with lit modifier");
3544 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3550 if (
Lit != LitModifier::None &&
3554 AMDGPUOperand::Modifiers Mods;
3555 Mods.Abs = Abs || SP3Abs;
3556 Mods.Neg = Neg || SP3Neg;
3559 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3560 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3562 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3563 Op.setModifiers(Mods);
3571 bool Sext = trySkipId(
"sext");
3572 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3587 AMDGPUOperand::Modifiers Mods;
3590 if (Mods.hasIntModifiers()) {
3591 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3593 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3594 Op.setModifiers(Mods);
3601 return parseRegOrImmWithFPInputMods(
Operands,
false);
3605 return parseRegOrImmWithIntInputMods(
Operands,
false);
3612 if (!trySkipId(
"rsrcidx"))
3618 SMLoc RegLoc = getLoc();
3619 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3627 if (!
Reg->isRsrcReg32())
3628 return Error(RegLoc,
"rsrcidx operand must be a 32-bit SGPR or VGPR");
3638 auto Loc = getLoc();
3639 if (trySkipId(
"off")) {
3641 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3648 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3657unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3662 return Match_InvalidOperand;
3664 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3665 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3668 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3670 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3671 return Match_InvalidOperand;
3679 if (tryAnotherVOPDEncoding(Inst))
3680 return Match_InvalidOperand;
3682 return Match_Success;
3686 static const unsigned Variants[] = {
3695ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3696 if (isForcedDPP() && isForcedVOP3()) {
3700 if (getForcedEncodingSize() == 32) {
3705 if (isForcedVOP3()) {
3710 if (isForcedSDWA()) {
3716 if (isForcedDPP()) {
3724StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3725 if (isForcedDPP() && isForcedVOP3())
3728 if (getForcedEncodingSize() == 32)
3744AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3748 case AMDGPU::FLAT_SCR:
3750 case AMDGPU::VCC_LO:
3751 case AMDGPU::VCC_HI:
3758 return MCRegister();
3765bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3766 unsigned OpIdx)
const {
3774 const MCOperand &MO = Inst.
getOperand(OpIdx);
3823unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3829 case AMDGPU::V_LSHLREV_B64_e64:
3830 case AMDGPU::V_LSHLREV_B64_gfx10:
3831 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3832 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3833 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3834 case AMDGPU::V_LSHRREV_B64_e64:
3835 case AMDGPU::V_LSHRREV_B64_gfx10:
3836 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3837 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3838 case AMDGPU::V_ASHRREV_I64_e64:
3839 case AMDGPU::V_ASHRREV_I64_gfx10:
3840 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3841 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3842 case AMDGPU::V_LSHL_B64_e64:
3843 case AMDGPU::V_LSHR_B64_e64:
3844 case AMDGPU::V_ASHR_I64_e64:
3857 bool AddMandatoryLiterals =
false) {
3860 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3864 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3866 return {getNamedOperandIdx(Opcode, OpName::src0X),
3867 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3868 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3869 getNamedOperandIdx(Opcode, OpName::src0Y),
3870 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3871 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3876 return {getNamedOperandIdx(Opcode, OpName::src0),
3877 getNamedOperandIdx(Opcode, OpName::src1),
3878 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3881bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3882 const MCOperand &MO = Inst.
getOperand(OpIdx);
3884 return !isInlineConstant(Inst, OpIdx);
3891 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3902 const unsigned Opcode = Inst.
getOpcode();
3903 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3906 if (!LaneSelOp.
isReg())
3909 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3912bool AMDGPUAsmParser::validateConstantBusLimitations(
3914 const unsigned Opcode = Inst.
getOpcode();
3915 const MCInstrDesc &
Desc = MII.
get(Opcode);
3916 MCRegister LastSGPR;
3917 unsigned ConstantBusUseCount = 0;
3918 unsigned NumLiterals = 0;
3919 unsigned LiteralSize;
3935 SmallDenseSet<MCRegister> SGPRsUsed;
3936 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3938 SGPRsUsed.
insert(SGPRUsed);
3939 ++ConstantBusUseCount;
3944 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3946 for (
int OpIdx : OpIndices) {
3950 const MCOperand &MO = Inst.
getOperand(OpIdx);
3951 if (usesConstantBus(Inst, OpIdx)) {
3960 if (SGPRsUsed.
insert(LastSGPR).second) {
3961 ++ConstantBusUseCount;
3981 if (NumLiterals == 0) {
3984 }
else if (LiteralSize !=
Size) {
3990 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3992 "invalid operand (violates constant bus restrictions)");
3999std::optional<unsigned>
4000AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
4002 const unsigned Opcode = Inst.
getOpcode();
4008 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
4009 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
4018 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
4019 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
4020 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
4021 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
4022 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
4023 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
4027 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
4028 int I = getNamedOperandIdx(Opcode, OpName);
4032 int64_t
Imm =
Op.getImm();
4038 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4039 OpName::vsrc2Y, OpName::imm}) {
4040 int I = getNamedOperandIdx(Opcode, OpName);
4050 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4051 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4053 return InvalidCompOprIdx;
4056bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4063 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4064 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4065 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4067 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4071 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4072 if (!InvalidCompOprIdx.has_value())
4075 auto CompOprIdx = *InvalidCompOprIdx;
4078 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4079 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4082 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4083 if (CompOprIdx == VOPD::Component::DST) {
4085 Error(Loc,
"dst registers must be distinct");
4087 Error(Loc,
"one dst register must be even and the other odd");
4089 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4090 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4091 " operands must use different VGPR banks");
4099bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4101 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4102 if (!InvalidCompOprIdx.has_value())
4106 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4107 if (InvalidCompOprIdx.has_value()) {
4112 if (*InvalidCompOprIdx == VOPD::Component::DST)
4125bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4126 const unsigned Opcode = Inst.
getOpcode();
4141 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4142 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4143 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4144 int I = getNamedOperandIdx(Opcode, OpName);
4151 return !tryVOPD3(Inst);
4156bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4161 return tryVOPD(Inst);
4162 return tryVOPD3(Inst);
4165bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4170 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4178bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4186 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4187 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4188 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4193 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4194 hasBVHRayTracingInsts())
4197 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4198 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4203 bool IsPackedD16 =
false;
4206 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4207 IsPackedD16 = D16Idx >= 0;
4209 DataSize = (DataSize + 1) / 2;
4212 if ((VDataSize / 4) == DataSize + TFESize)
4217 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4219 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4221 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4225bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4234 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4236 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4237 AMDGPU::OpName RSrcOpName =
4239 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4240 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4241 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4245 assert(SrsrcIdx > VAddr0Idx);
4248 if (BaseOpcode->
BVH) {
4249 if (IsA16 == BaseOpcode->
A16)
4251 Error(IDLoc,
"image address size does not match a16");
4257 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4258 unsigned ActualAddrSize =
4259 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4261 unsigned ExpectedAddrSize =
4265 if (hasPartialNSAEncoding() &&
4267 int VAddrLastIdx = SrsrcIdx - 1;
4268 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4270 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4273 if (ExpectedAddrSize > 12)
4274 ExpectedAddrSize = 16;
4279 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4283 if (ActualAddrSize == ExpectedAddrSize)
4286 Error(IDLoc,
"image address size does not match dim and a16");
4290bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4297 if (!
Desc.mayLoad() || !
Desc.mayStore())
4300 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4307 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4310bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4317 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4325 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4328bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4342 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4343 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4350bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4357 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4360 if (!BaseOpcode->
MSAA)
4363 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4369 return DimInfo->
MSAA;
4374 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4375 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4376 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4386bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4394 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4397 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4405 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4409bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4414 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4417 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4420 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4428 "source operand must be either a VGPR or an inline constant");
4435bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4440 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4443 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4447 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4449 "inline constants are not allowed for this operand");
4456bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4464 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4465 if (BlgpIdx != -1) {
4466 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4467 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4477 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4479 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4484 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4486 "wrong register tuple size for blgp value " + Twine(BLGP));
4494 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4498 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4502 MCRegister Src2Reg = Src2.
getReg();
4504 if (Src2Reg == DstReg)
4509 .getSizeInBits() <= 128)
4512 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4514 "source 2 operand must not partially overlap with dst");
4521bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4525 case V_DIV_SCALE_F32_gfx6_gfx7:
4526 case V_DIV_SCALE_F32_vi:
4527 case V_DIV_SCALE_F32_gfx10:
4528 case V_DIV_SCALE_F64_gfx6_gfx7:
4529 case V_DIV_SCALE_F64_vi:
4530 case V_DIV_SCALE_F64_gfx10:
4537 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4538 AMDGPU::OpName::src2_modifiers}) {
4549bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4556 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4565bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4571 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4578 case AMDGPU::V_SUBREV_F32_e32:
4579 case AMDGPU::V_SUBREV_F32_e64:
4580 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4581 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4582 case AMDGPU::V_SUBREV_F32_e32_vi:
4583 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4584 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4585 case AMDGPU::V_SUBREV_F32_e64_vi:
4587 case AMDGPU::V_SUBREV_CO_U32_e32:
4588 case AMDGPU::V_SUBREV_CO_U32_e64:
4589 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4590 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4592 case AMDGPU::V_SUBBREV_U32_e32:
4593 case AMDGPU::V_SUBBREV_U32_e64:
4594 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4595 case AMDGPU::V_SUBBREV_U32_e32_vi:
4596 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4597 case AMDGPU::V_SUBBREV_U32_e64_vi:
4599 case AMDGPU::V_SUBREV_U32_e32:
4600 case AMDGPU::V_SUBREV_U32_e64:
4601 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4602 case AMDGPU::V_SUBREV_U32_e32_vi:
4603 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4604 case AMDGPU::V_SUBREV_U32_e64_vi:
4606 case AMDGPU::V_SUBREV_F16_e32:
4607 case AMDGPU::V_SUBREV_F16_e64:
4608 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4609 case AMDGPU::V_SUBREV_F16_e32_vi:
4610 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4611 case AMDGPU::V_SUBREV_F16_e64_vi:
4613 case AMDGPU::V_SUBREV_U16_e32:
4614 case AMDGPU::V_SUBREV_U16_e64:
4615 case AMDGPU::V_SUBREV_U16_e32_vi:
4616 case AMDGPU::V_SUBREV_U16_e64_vi:
4618 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4619 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4620 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4622 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4623 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4625 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4626 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4628 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4629 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4631 case AMDGPU::V_LSHRREV_B32_e32:
4632 case AMDGPU::V_LSHRREV_B32_e64:
4633 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4634 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4635 case AMDGPU::V_LSHRREV_B32_e32_vi:
4636 case AMDGPU::V_LSHRREV_B32_e64_vi:
4637 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4638 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4640 case AMDGPU::V_ASHRREV_I32_e32:
4641 case AMDGPU::V_ASHRREV_I32_e64:
4642 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4643 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4644 case AMDGPU::V_ASHRREV_I32_e32_vi:
4645 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4646 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4647 case AMDGPU::V_ASHRREV_I32_e64_vi:
4649 case AMDGPU::V_LSHLREV_B32_e32:
4650 case AMDGPU::V_LSHLREV_B32_e64:
4651 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4652 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4653 case AMDGPU::V_LSHLREV_B32_e32_vi:
4654 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4655 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4656 case AMDGPU::V_LSHLREV_B32_e64_vi:
4658 case AMDGPU::V_LSHLREV_B16_e32:
4659 case AMDGPU::V_LSHLREV_B16_e64:
4660 case AMDGPU::V_LSHLREV_B16_e32_vi:
4661 case AMDGPU::V_LSHLREV_B16_e64_vi:
4662 case AMDGPU::V_LSHLREV_B16_gfx10:
4664 case AMDGPU::V_LSHRREV_B16_e32:
4665 case AMDGPU::V_LSHRREV_B16_e64:
4666 case AMDGPU::V_LSHRREV_B16_e32_vi:
4667 case AMDGPU::V_LSHRREV_B16_e64_vi:
4668 case AMDGPU::V_LSHRREV_B16_gfx10:
4670 case AMDGPU::V_ASHRREV_I16_e32:
4671 case AMDGPU::V_ASHRREV_I16_e64:
4672 case AMDGPU::V_ASHRREV_I16_e32_vi:
4673 case AMDGPU::V_ASHRREV_I16_e64_vi:
4674 case AMDGPU::V_ASHRREV_I16_gfx10:
4676 case AMDGPU::V_LSHLREV_B64_e64:
4677 case AMDGPU::V_LSHLREV_B64_gfx10:
4678 case AMDGPU::V_LSHLREV_B64_vi:
4680 case AMDGPU::V_LSHRREV_B64_e64:
4681 case AMDGPU::V_LSHRREV_B64_gfx10:
4682 case AMDGPU::V_LSHRREV_B64_vi:
4684 case AMDGPU::V_ASHRREV_I64_e64:
4685 case AMDGPU::V_ASHRREV_I64_gfx10:
4686 case AMDGPU::V_ASHRREV_I64_vi:
4688 case AMDGPU::V_PK_LSHLREV_B16:
4689 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4690 case AMDGPU::V_PK_LSHLREV_B16_vi:
4692 case AMDGPU::V_PK_LSHRREV_B16:
4693 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4694 case AMDGPU::V_PK_LSHRREV_B16_vi:
4695 case AMDGPU::V_PK_ASHRREV_I16:
4696 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4697 case AMDGPU::V_PK_ASHRREV_I16_vi:
4704bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4706 const unsigned Opcode = Inst.
getOpcode();
4715 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4716 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4720 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4724 "lds_direct is not supported on this GPU");
4730 "lds_direct cannot be used with this instruction");
4734 if (SrcName != OpName::src0) {
4736 "lds_direct may be used as src0 only");
4746 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4747 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4748 if (
Op.isFlatOffset())
4749 return Op.getStartLoc();
4754bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4757 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4762 return validateFlatOffset(Inst,
Operands);
4765 return validateSMEMOffset(Inst,
Operands);
4770 const unsigned OffsetSize = 24;
4771 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4773 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4774 "-bit unsigned offset for buffer ops");
4778 const unsigned OffsetSize = 16;
4779 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4781 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4788bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4794 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4798 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4800 "flat offset modifier is not supported on this GPU");
4807 bool AllowNegative =
4809 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4811 Twine(
"expected a ") +
4812 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4813 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4822 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4823 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4824 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4825 return Op.getStartLoc();
4830bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4839 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4855 ?
"expected a 23-bit unsigned offset for buffer ops"
4856 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4857 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4858 :
"expected a 21-bit signed offset");
4863bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4866 const MCInstrDesc &
Desc = MII.
get(Opcode);
4870 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4871 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4873 const int OpIndices[] = {Src0Idx, Src1Idx};
4875 unsigned NumExprs = 0;
4876 unsigned NumLiterals = 0;
4879 for (
int OpIdx : OpIndices) {
4883 const MCOperand &MO = Inst.
getOperand(OpIdx);
4887 std::optional<int64_t>
Imm;
4890 }
else if (MO.
isExpr()) {
4899 if (!
Imm.has_value()) {
4901 }
else if (!isInlineConstant(Inst, OpIdx)) {
4905 if (NumLiterals == 0 || LiteralValue !=
Value) {
4913 if (NumLiterals + NumExprs <= 1)
4917 "only one unique literal operand is allowed");
4921bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4924 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4932 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4933 if (OpSelIdx != -1) {
4937 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4938 if (OpSelHiIdx != -1) {
4947 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4957 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4958 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4959 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4960 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4962 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4963 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4969 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4971 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4981 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4982 if (Src2Idx != -1) {
4983 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4993bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4994 if (!hasTrue16Insts())
4996 const MCRegisterInfo *MRI = getMRI();
4998 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
5004 if (OpSelOpValue == 0)
5006 unsigned OpCount = 0;
5007 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
5008 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
5009 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
5016 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
5017 if (OpSelOpIsHi != VGPRSuffixIsHi)
5026bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
5027 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5039 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5050 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5051 AMDGPU::OpName::src1_modifiers,
5052 AMDGPU::OpName::src2_modifiers};
5054 for (
unsigned i = 0; i < 3; ++i) {
5064bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5067 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5068 if (DppCtrlIdx >= 0) {
5075 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5076 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5077 :
"DP ALU dpp only supports row_newbcast");
5082 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5083 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5086 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5088 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5092 "invalid operand for instruction");
5097 "src1 immediate operand invalid for instruction");
5107bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5108 return (
Reg == AMDGPU::VCC && isWave64()) ||
5109 (
Reg == AMDGPU::VCC_LO && isWave32());
5113bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5116 const MCInstrDesc &
Desc = MII.
get(Opcode);
5117 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5124 std::optional<unsigned> LiteralOpIdx;
5127 for (
int OpIdx : OpIndices) {
5131 const MCOperand &MO = Inst.
getOperand(OpIdx);
5137 std::optional<int64_t>
Imm;
5143 bool IsAnotherLiteral =
false;
5144 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5145 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5146 if (!
Imm.has_value()) {
5148 IsAnotherLiteral =
true;
5149 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5154 HasMandatoryLiteral);
5166 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5167 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5169 "invalid operand for instruction");
5174 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5175 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5177 "invalid operand for instruction");
5182 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5183 Value =
static_cast<uint32_t
>(
5191 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5192 !getFeatureBits()[FeatureVOP3Literal]) {
5194 "literal operands are not supported");
5198 if (LiteralOpIdx && IsAnotherLiteral) {
5200 getOperandLoc(
Operands, *LiteralOpIdx)),
5201 "only one unique literal operand is allowed");
5205 if (IsAnotherLiteral)
5206 LiteralOpIdx = OpIdx;
5215 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5229bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5235 ? AMDGPU::OpName::data0
5236 : AMDGPU::OpName::vdata;
5238 const MCRegisterInfo *MRI = getMRI();
5239 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5243 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5244 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5248 auto FB = getFeatureBits();
5249 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5250 if (DataAreg < 0 || DstAreg < 0)
5252 return DstAreg == DataAreg;
5255 return DstAreg < 1 && DataAreg < 1;
5258bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5259 auto FB = getFeatureBits();
5260 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5264 const MCRegisterInfo *MRI = getMRI();
5267 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5270 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5274 case AMDGPU::DS_LOAD_TR6_B96:
5275 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5279 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5280 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5284 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5285 if (VAddrIdx != -1) {
5288 if ((
Sub - AMDGPU::VGPR0) & 1)
5293 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5294 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5299 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5300 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5320 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5321 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5323 return Op.getStartLoc();
5328bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5331 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5334 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5337 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5338 auto FB = getFeatureBits();
5339 bool UsesNeg =
false;
5340 if (FB[AMDGPU::FeatureGFX940Insts]) {
5342 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5343 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5344 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5345 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5350 if (IsNeg == UsesNeg)
5353 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5354 :
"invalid modifier: neg is not supported");
5359bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5365 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5366 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5367 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5368 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5371 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5374 if (
Reg == AMDGPU::SGPR_NULL)
5377 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5381bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5386 return validateGWS(Inst,
Operands);
5391 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5396 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5397 Error(S,
"gds modifier is not supported on this GPU");
5405bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5407 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5411 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5412 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5415 const MCRegisterInfo *MRI = getMRI();
5416 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5418 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5421 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5423 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5430bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5434 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5442 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5445 Error(S,
"scale_offset is not supported on this GPU");
5448 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5451 Error(S,
"nv is not supported on this GPU");
5456 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5459 Error(S,
"scale_offset is not supported for this instruction");
5463 return validateTHAndScopeBits(Inst,
Operands, CPol);
5467 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5468 Error(S,
"cache policy is not supported for SMRD instructions");
5472 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5479 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5483 "scc modifier is not supported for this instruction on this GPU");
5494 :
"instruction must use glc");
5499 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5502 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5504 :
"instruction must not use glc");
5512bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5514 const unsigned CPol) {
5519 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5526 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5531 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5539 return PrintError(
"invalid th value for SMEM instruction");
5546 return PrintError(
"scope and th combination is not valid");
5552 return PrintError(
"invalid th value for atomic instructions");
5555 return PrintError(
"invalid th value for store instructions");
5558 return PrintError(
"invalid th value for load instructions");
5564bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5568 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5570 Error(Loc,
"TFE modifier has no meaning for store instructions");
5578bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5584 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5588 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5591 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5592 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5601 "wrong register tuple size for " +
5606 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5607 !validateFmt(BFmt, AMDGPU::OpName::src1))
5611 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5612 if (AScaleIdx == -1)
5616 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5620 "invalid matrix and scale format combination");
5627bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5630 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5631 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5634 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5637 "sleep forever is unsuported on the target");
5644bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5647 if (
Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5648 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5651 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
5658 "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5662bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5664 if (!validateLdsDirect(Inst,
Operands))
5666 if (!validateTrue16OpSel(Inst)) {
5668 "op_sel operand conflicts with 16-bit operand suffix");
5671 if (!validateSOPLiteral(Inst,
Operands))
5673 if (!validateVOPLiteral(Inst,
Operands)) {
5676 if (!validateConstantBusLimitations(Inst,
Operands)) {
5679 if (!validateVOPD(Inst,
Operands)) {
5682 if (!validateIntClampSupported(Inst)) {
5684 "integer clamping is not supported on this GPU");
5687 if (!validateOpSel(Inst)) {
5689 "invalid op_sel operand");
5692 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5694 "invalid neg_lo operand");
5697 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5699 "invalid neg_hi operand");
5702 if (!validateDPP(Inst,
Operands)) {
5706 if (!validateMIMGD16(Inst)) {
5708 "d16 modifier is not supported on this GPU");
5711 if (!validateMIMGDim(Inst,
Operands)) {
5712 Error(IDLoc,
"missing dim operand");
5715 if (!validateTensorR128(Inst)) {
5717 "instruction must set modifier r128=0");
5720 if (!validateMIMGMSAA(Inst)) {
5722 "invalid dim; must be MSAA type");
5725 if (!validateMIMGDataSize(Inst, IDLoc)) {
5728 if (!validateMIMGAddrSize(Inst, IDLoc))
5730 if (!validateMIMGAtomicDMask(Inst)) {
5732 "invalid atomic image dmask");
5735 if (!validateMIMGGatherDMask(Inst)) {
5737 "invalid image_gather dmask: only one bit must be set");
5740 if (!validateMovrels(Inst,
Operands)) {
5743 if (!validateOffset(Inst,
Operands)) {
5746 if (!validateMAIAccWrite(Inst,
Operands)) {
5749 if (!validateMAISrc2(Inst,
Operands)) {
5752 if (!validateMFMA(Inst,
Operands)) {
5755 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5759 if (!validateAGPRLdSt(Inst)) {
5762 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5763 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5764 :
"invalid register class: agpr loads and stores not supported on "
5768 if (!validateVGPRAlign(Inst)) {
5769 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5776 if (!validateBLGP(Inst,
Operands)) {
5780 if (!validateDivScale(Inst)) {
5781 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5784 if (!validateWaitCnt(Inst,
Operands)) {
5787 if (!validateTFE(Inst,
Operands)) {
5790 if (!validateWMMA(Inst,
Operands)) {
5793 if (!validateMonitorSleep(Inst,
Operands)) {
5796 if (!validateClusterBarrierIsFirst(Inst,
Operands)) {
5805 unsigned VariantID = 0);
5809 unsigned VariantID);
5811bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5816bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5817 const FeatureBitset &FBS,
5818 ArrayRef<unsigned> Variants) {
5819 for (
auto Variant : Variants) {
5827bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5829 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5832 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5837 getParser().clearPendingErrors();
5841 StringRef VariantName = getMatchedVariantName();
5842 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5843 return Error(IDLoc, Twine(VariantName,
5844 " variant of this instruction is not supported"));
5848 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5849 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5851 FeatureBitset FeaturesWS32 = getFeatureBits();
5852 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5853 .
flip(AMDGPU::FeatureWavefrontSize32);
5854 FeatureBitset AvailableFeaturesWS32 =
5855 ComputeAvailableFeatures(FeaturesWS32);
5857 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5858 return Error(IDLoc,
"instruction requires wavesize=32");
5862 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5863 return Error(IDLoc,
"instruction not supported on this GPU (" +
5864 getSTI().
getCPU() +
")" +
": " + Mnemo);
5869 return Error(IDLoc,
"invalid instruction" + Suggestion);
5875 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5876 if (
Op.isToken() && InvalidOprIdx > 1) {
5877 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5878 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5883bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5887 bool MatchingInlineAsm) {
5890 unsigned Result = Match_Success;
5891 for (
auto Variant : getMatchedVariants()) {
5894 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5898 if (R == Match_Success || R == Match_MissingFeature ||
5899 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5900 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5901 Result != Match_MissingFeature)) {
5905 if (R == Match_Success)
5909 if (Result == Match_Success) {
5910 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5913 emitTargetDirective();
5921 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5928 case Match_MissingFeature:
5932 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5934 case Match_InvalidOperand: {
5935 SMLoc ErrorLoc = IDLoc;
5936 if (ErrorInfo != ~0ULL) {
5937 if (ErrorInfo >=
Operands.size()) {
5938 return Error(IDLoc,
"too few operands for instruction");
5940 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5941 if (ErrorLoc == SMLoc())
5945 return Error(ErrorLoc,
"invalid VOPDY instruction");
5947 return Error(ErrorLoc,
"invalid operand for instruction");
5950 case Match_MnemonicFail:
5956bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5961 if (getParser().parseAbsoluteExpression(Tmp)) {
5964 Ret =
static_cast<uint32_t
>(Tmp);
5968bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5969 if (!getSTI().getTargetTriple().isAMDGCN())
5970 return TokError(
"directive only supported for amdgcn architecture");
5972 std::string TargetIDDirective;
5973 SMLoc TargetStart = getTok().getLoc();
5974 if (getParser().parseEscapedString(TargetIDDirective))
5977 std::optional<AMDGPU::TargetID> MaybeParsed =
5980 return getParser().Error(TargetStart,
5981 "malformed target id '" + TargetIDDirective +
"'");
5984 const Triple &
TT = getSTI().getTargetTriple();
5990 return getParser().Error(
5991 TargetStart,
"target id '" + TargetIDDirective +
5992 "' specifies a processor that is not valid for "
5994 TT.getArchName() +
"'");
5997 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5998 getTargetStreamer().getTargetID();
6001 const Triple &STITriple = getSTI().getTargetTriple();
6002 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6003 return getParser().Error(
6004 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
6005 " is incompatible with " +
6006 Twine(CurrentTargetID->toString()));
6010 StringRef DirectiveProcessor =
6013 if (DirectiveISA != ISA) {
6014 return getParser().Error(TargetStart,
6015 ".amdgcn_target directive processor " +
6016 Twine(DirectiveProcessor) +
6017 " does not match the specified processor " +
6018 Twine(getSTI().
getCPU()));
6024 CurrentTargetID->getXnackSetting())) {
6026 ".amdgcn_target directive has conflicting xnack settings");
6030 CurrentTargetID->getSramEccSetting())) {
6032 ".amdgcn_target directive has conflicting sramecc settings");
6038 getTargetStreamer().getTargetID()->setXnackSetting(
6040 getTargetStreamer().getTargetID()->setSramEccSetting(
6046bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
6050bool AMDGPUAsmParser::calculateGPRBlocks(
6051 const FeatureBitset &Features,
const MCExpr *VCCUsed,
6052 const MCExpr *FlatScrUsed,
bool XNACKUsed,
6053 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
6054 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6055 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6060 const MCExpr *
NumSGPRs = NextFreeSGPR;
6061 int64_t EvaluatedSGPRs;
6063 if (
ISA.Major >= 10)
6068 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6069 !Features.
test(FeatureSGPRInitBug) &&
6070 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6071 return OutOfRangeError(SGPRRange);
6073 const MCExpr *ExtraSGPRs =
6077 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6078 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6079 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6080 return OutOfRangeError(SGPRRange);
6082 if (Features.
test(FeatureSGPRInitBug))
6089 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6090 unsigned Granule) ->
const MCExpr * {
6094 const MCExpr *AlignToGPR =
6096 const MCExpr *DivGPR =
6102 VGPRBlocks = GetNumGPRBlocks(
6111bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6112 if (!getSTI().getTargetTriple().isAMDGCN())
6113 return TokError(
"directive only supported for amdgcn architecture");
6116 return TokError(
"directive only supported for amdhsa OS");
6118 StringRef KernelName;
6119 if (getParser().parseIdentifier(KernelName))
6126 AMDGPU::MCKernelDescriptor KD =
6136 const MCExpr *NextFreeVGPR = ZeroExpr;
6138 const MCExpr *NamedBarCnt = ZeroExpr;
6143 const MCExpr *NextFreeSGPR = ZeroExpr;
6146 unsigned ImpliedUserSGPRCount = 0;
6150 std::optional<unsigned> ExplicitUserSGPRCount;
6151 const MCExpr *ReserveVCC = OneExpr;
6152 const MCExpr *ReserveFlatScr = OneExpr;
6153 std::optional<bool> EnableWavefrontSize32;
6160 SMRange IDRange = getTok().getLocRange();
6161 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6164 if (ID ==
".end_amdhsa_kernel")
6167 if (!Seen.
insert(ID).second)
6168 return TokError(
".amdhsa_ directives cannot be repeated");
6170 SMLoc ValStart = getLoc();
6171 const MCExpr *ExprVal;
6172 if (getParser().parseExpression(ExprVal))
6174 SMLoc ValEnd = getLoc();
6175 SMRange ValRange = SMRange(ValStart, ValEnd);
6179 bool EvaluatableExpr;
6180 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6182 return OutOfRangeError(ValRange);
6186#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6187 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6188 return OutOfRangeError(RANGE); \
6189 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6194#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6196 return Error(IDRange.Start, "directive should have resolvable expression", \
6199 if (ID ==
".amdhsa_group_segment_fixed_size") {
6202 return OutOfRangeError(ValRange);
6204 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6207 return OutOfRangeError(ValRange);
6209 }
else if (ID ==
".amdhsa_kernarg_size") {
6211 return OutOfRangeError(ValRange);
6213 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6215 ExplicitUserSGPRCount = Val;
6216 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6220 "directive is not supported with architected flat scratch",
6223 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6226 ImpliedUserSGPRCount += 4;
6227 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6230 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6233 return OutOfRangeError(ValRange);
6237 ImpliedUserSGPRCount += Val;
6238 PreloadLength = Val;
6240 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6243 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6246 return OutOfRangeError(ValRange);
6250 PreloadOffset = Val;
6251 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6254 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6257 ImpliedUserSGPRCount += 2;
6258 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6261 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6264 ImpliedUserSGPRCount += 2;
6265 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6268 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6271 ImpliedUserSGPRCount += 2;
6272 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6275 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6278 ImpliedUserSGPRCount += 2;
6279 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6282 "directive is not supported with architected flat scratch",
6286 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6289 ImpliedUserSGPRCount += 2;
6290 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6293 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6296 ImpliedUserSGPRCount += 1;
6297 }
else if (ID ==
".amdhsa_wavefront_size32") {
6300 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6301 EnableWavefrontSize32 = Val;
6303 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6305 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6307 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6309 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6312 "directive is not supported with architected flat scratch",
6315 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6317 }
else if (ID ==
".amdhsa_enable_private_segment") {
6321 "directive is not supported without architected flat scratch",
6324 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6326 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6328 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6330 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6332 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6334 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6336 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6338 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6340 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6342 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6344 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6346 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6347 VGPRRange = ValRange;
6348 NextFreeVGPR = ExprVal;
6349 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6350 SGPRRange = ValRange;
6351 NextFreeSGPR = ExprVal;
6352 }
else if (ID ==
".amdhsa_accum_offset") {
6354 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6355 AccumOffset = ExprVal;
6356 }
else if (ID ==
".amdhsa_named_barrier_count") {
6358 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6359 NamedBarCnt = ExprVal;
6360 }
else if (ID ==
".amdhsa_reserve_vcc") {
6362 return OutOfRangeError(ValRange);
6363 ReserveVCC = ExprVal;
6364 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6366 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6369 "directive is not supported with architected flat scratch",
6372 return OutOfRangeError(ValRange);
6373 ReserveFlatScr = ExprVal;
6374 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6376 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6378 return OutOfRangeError(ValRange);
6379 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6380 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6381 if (Val != XnackOn) {
6382 return getParser().Error(
6384 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6386 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6388 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6390 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6392 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6394 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6396 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6398 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6400 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6402 }
else if (ID ==
".amdhsa_dx10_clamp") {
6403 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6404 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6407 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6409 }
else if (ID ==
".amdhsa_ieee_mode") {
6410 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6411 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6414 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6416 }
else if (ID ==
".amdhsa_fp16_overflow") {
6418 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6420 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6422 }
else if (ID ==
".amdhsa_tg_split") {
6424 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6427 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6430 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6432 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6434 }
else if (ID ==
".amdhsa_memory_ordered") {
6436 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6438 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6440 }
else if (ID ==
".amdhsa_forward_progress") {
6442 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6444 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6446 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6448 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6449 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6451 SharedVGPRCount = Val;
6453 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6455 }
else if (ID ==
".amdhsa_inst_pref_size") {
6457 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6458 if (
ISA.Major == 11) {
6460 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6464 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6467 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6470 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6472 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6474 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6476 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6479 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6481 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6483 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6485 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6487 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6489 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6491 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6493 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6495 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6497 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6499 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6501 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6504 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6507#undef PARSE_BITS_ENTRY
6510 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6511 return TokError(
".amdhsa_next_free_vgpr directive is required");
6513 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6514 return TokError(
".amdhsa_next_free_sgpr directive is required");
6516 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6518 return TokError(
"too many user SGPRs enabled, found " +
6519 Twine(UserSGPRCount) +
", but only " +
6525 if (PreloadLength) {
6531 const MCExpr *VGPRBlocks;
6532 const MCExpr *SGPRBlocks;
6533 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6534 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6535 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6536 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6539 int64_t EvaluatedVGPRBlocks;
6540 bool VGPRBlocksEvaluatable =
6541 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6542 if (VGPRBlocksEvaluatable &&
6544 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6545 return OutOfRangeError(VGPRRange);
6549 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6550 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6552 int64_t EvaluatedSGPRBlocks;
6553 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6555 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6556 return OutOfRangeError(SGPRRange);
6559 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6560 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6562 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6563 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6564 "enabled user SGPRs");
6570 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6571 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6576 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6577 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6582 return TokError(
"Kernarg size should be resolvable");
6584 if (PreloadLength && kernarg_size &&
6585 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6586 return TokError(
"Kernarg preload length + offset is larger than the "
6587 "kernarg segment size");
6590 if (!Seen.
contains(
".amdhsa_accum_offset"))
6591 return TokError(
".amdhsa_accum_offset directive is required");
6592 int64_t EvaluatedAccum;
6593 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6594 uint64_t UEvaluatedAccum = EvaluatedAccum;
6595 if (AccumEvaluatable &&
6596 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6597 return TokError(
"accum_offset should be in range [4..256] in "
6600 int64_t EvaluatedNumVGPR;
6601 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6605 return TokError(
"accum_offset exceeds total VGPR allocation");
6611 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6612 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6618 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6619 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6622 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6624 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6625 return TokError(
"shared_vgpr_count directive not valid on "
6626 "wavefront size 32");
6629 if (VGPRBlocksEvaluatable &&
6630 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6632 return TokError(
"shared_vgpr_count*2 + "
6633 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6638 emitTargetDirective();
6639 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6640 NextFreeVGPR, NextFreeSGPR,
6641 ReserveVCC, ReserveFlatScr);
6645bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6647 if (ParseAsAbsoluteExpression(
Version))
6650 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6651 emitTargetDirective();
6655bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6656 AMDGPUMCKernelCodeT &
C) {
6659 if (ID ==
"max_scratch_backing_memory_byte_size") {
6660 Parser.eatToEndOfStatement();
6664 SmallString<40> ErrStr;
6665 raw_svector_ostream Err(ErrStr);
6666 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6667 return TokError(Err.
str());
6671 if (ID ==
"enable_wavefront_size32") {
6674 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6676 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6679 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6683 if (ID ==
"wavefront_size") {
6684 if (
C.wavefront_size == 5) {
6686 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6688 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6689 }
else if (
C.wavefront_size == 6) {
6691 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6698bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6699 AMDGPUMCKernelCodeT KernelCode;
6709 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6712 if (ID ==
".end_amd_kernel_code_t")
6715 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6720 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6725bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6726 StringRef KernelName;
6727 if (!parseId(KernelName,
"expected symbol name"))
6730 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6737bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6738 if (!getSTI().getTargetTriple().isAMDGCN()) {
6739 return Error(getLoc(),
6740 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6744 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6746 std::optional<AMDGPU::TargetID> MaybeParsed =
6749 return Error(getParser().getTok().getLoc(),
6750 "malformed target id '" + TargetIDDirective +
"'");
6753 const Triple &
TT = getSTI().getTargetTriple();
6759 return Error(getParser().getTok().getLoc(),
6760 "target id '" + TargetIDDirective +
6761 "' specifies a processor that is not valid for subarch '" +
6762 TT.getArchName() +
"'");
6765 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6766 getTargetStreamer().getTargetID();
6769 const Triple &STITriple = getSTI().getTargetTriple();
6770 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6771 return Error(getParser().getTok().getLoc(),
6772 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6773 " is incompatible with " +
6774 Twine(CurrentTargetID->toString()));
6778 StringRef DirectiveProcessor =
6781 if (DirectiveISA != ISA) {
6782 return Error(getParser().getTok().getLoc(),
6783 ".amd_amdgpu_isa directive processor " +
6784 Twine(DirectiveProcessor) +
6785 " does not match the specified processor " +
6786 Twine(getSTI().
getCPU()));
6789 getTargetStreamer().EmitISAVersion();
6795bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6798 std::string HSAMetadataString;
6803 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6804 return Error(getLoc(),
"invalid HSA metadata");
6811bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6812 const char *AssemblerDirectiveEnd,
6813 std::string &CollectString) {
6815 raw_string_ostream CollectStream(CollectString);
6817 getLexer().setSkipSpace(
false);
6819 bool FoundEnd =
false;
6822 CollectStream << getTokenStr();
6826 if (trySkipId(AssemblerDirectiveEnd)) {
6831 CollectStream << Parser.parseStringToEndOfStatement()
6832 <<
getContext().getAsmInfo().getSeparatorString();
6834 Parser.eatToEndOfStatement();
6837 getLexer().setSkipSpace(
true);
6840 return TokError(Twine(
"expected directive ") +
6841 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6848bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6854 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6855 if (!PALMetadata->setFromString(
String))
6856 return Error(getLoc(),
"invalid PAL metadata");
6861bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6864 Twine(
" directive is "
6865 "not available on non-amdpal OSes"))
6869 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6870 PALMetadata->setLegacy();
6873 if (ParseAsAbsoluteExpression(
Key)) {
6874 return TokError(Twine(
"invalid value in ") +
6878 return TokError(Twine(
"expected an even number of values in ") +
6881 if (ParseAsAbsoluteExpression(
Value)) {
6882 return TokError(Twine(
"invalid value in ") +
6885 PALMetadata->setRegister(
Key,
Value);
6894bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6895 if (getParser().checkForValidSection())
6899 SMLoc NameLoc = getLoc();
6900 if (getParser().parseIdentifier(Name))
6901 return TokError(
"expected identifier in directive");
6904 if (getParser().parseComma())
6910 SMLoc SizeLoc = getLoc();
6911 if (getParser().parseAbsoluteExpression(
Size))
6914 return Error(SizeLoc,
"size must be non-negative");
6915 if (
Size > LocalMemorySize)
6916 return Error(SizeLoc,
"size is too large");
6920 SMLoc AlignLoc = getLoc();
6921 if (getParser().parseAbsoluteExpression(Alignment))
6924 return Error(AlignLoc,
"alignment must be a power of two");
6929 if (Alignment >= 1u << 31)
6930 return Error(AlignLoc,
"alignment is too large");
6936 Symbol->redefineIfPossible();
6937 if (!
Symbol->isUndefined())
6938 return Error(NameLoc,
"invalid symbol redefinition");
6940 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6944bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6945 if (getParser().checkForValidSection())
6949 if (getParser().parseIdentifier(FuncName))
6950 return TokError(
"expected symbol name after .amdgpu_info");
6953 AMDGPU::InfoSectionData ParsedInfoData;
6954 AMDGPU::FuncInfo FI;
6956 bool HasScalarAttrs =
false;
6963 SMLoc IDLoc = getLoc();
6964 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6967 if (ID ==
".end_amdgpu_info")
6975 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6977 if (Dir ==
"flags") {
6979 if (getParser().parseAbsoluteExpression(Val))
6982 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6984 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6986 HasScalarAttrs =
true;
6987 }
else if (Dir ==
"num_sgpr") {
6989 if (getParser().parseAbsoluteExpression(Val))
6991 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6992 HasScalarAttrs =
true;
6993 }
else if (Dir ==
"num_vgpr") {
6995 if (getParser().parseAbsoluteExpression(Val))
6998 HasScalarAttrs =
true;
6999 }
else if (Dir ==
"num_agpr") {
7001 if (getParser().parseAbsoluteExpression(Val))
7004 HasScalarAttrs =
true;
7005 }
else if (Dir ==
"private_segment_size") {
7007 if (getParser().parseAbsoluteExpression(Val))
7010 HasScalarAttrs =
true;
7011 }
else if (Dir ==
"use") {
7013 if (getParser().parseIdentifier(ResName))
7014 return TokError(
"expected resource symbol for .amdgpu_use");
7015 ParsedInfoData.
Uses.push_back(
7016 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
7017 }
else if (Dir ==
"call") {
7019 if (getParser().parseIdentifier(DstName))
7020 return TokError(
"expected callee symbol for .amdgpu_call");
7021 ParsedInfoData.
Calls.push_back(
7022 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
7023 }
else if (Dir ==
"indirect_call") {
7025 if (getParser().parseEscapedString(TypeId))
7026 return TokError(
"expected type ID string for .amdgpu_indirect_call");
7027 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
7028 }
else if (Dir ==
"typeid") {
7030 if (getParser().parseEscapedString(TypeId))
7031 return TokError(
"expected type ID string for .amdgpu_typeid");
7032 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
7034 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
7039 ParsedInfoData.
Funcs.push_back(std::move(FI));
7041 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
7042 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
7043 Data.Funcs.push_back(std::move(Func));
7044 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
7045 Data.Uses.push_back(Use);
7046 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
7048 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
7051 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
7052 Data.TypeIds.push_back(std::move(TypeId));
7057void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7064void AMDGPUAsmParser::checkKernelPrologues() {
7065 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7066 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7067 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7068 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7069 if (!AMDHSAKernelSymbols.
contains(Sym))
7071 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7072 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7076 "' does not begin with the required prologue "
7077 "sequence: s_mov_b64 followed by v_nop and "
7078 "global_prefetch_b8");
7082 OpcodeStream.
clear();
7083 OpcodeStreamSymbols.clear();
7084 AMDHSAKernelSymbols.
clear();
7087void AMDGPUAsmParser::onEndOfFile() {
7088 emitTargetDirective();
7089 checkKernelPrologues();
7091 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7094bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7095 StringRef IDVal = DirectiveID.
getString();
7098 if (IDVal ==
".amdhsa_kernel")
7099 return ParseDirectiveAMDHSAKernel();
7101 if (IDVal ==
".amdhsa_code_object_version")
7102 return ParseDirectiveAMDHSACodeObjectVersion();
7106 return ParseDirectiveHSAMetadata();
7108 if (IDVal ==
".amd_kernel_code_t")
7109 return ParseDirectiveAMDKernelCodeT();
7111 if (IDVal ==
".amdgpu_hsa_kernel")
7112 return ParseDirectiveAMDGPUHsaKernel();
7114 if (IDVal ==
".amd_amdgpu_isa")
7115 return ParseDirectiveISAVersion();
7119 Twine(
" directive is "
7120 "not available on non-amdhsa OSes"))
7125 if (IDVal ==
".amdgcn_target")
7126 return ParseDirectiveAMDGCNTarget();
7128 if (IDVal ==
".amdgpu_lds")
7129 return ParseDirectiveAMDGPULDS();
7131 if (IDVal ==
".amdgpu_info")
7132 return ParseDirectiveAMDGPUInfo();
7135 return ParseDirectivePALMetadataBegin();
7138 return ParseDirectivePALMetadata();
7143bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7150 return hasSGPR104_SGPR105();
7153 case SRC_SHARED_BASE_LO:
7154 case SRC_SHARED_BASE:
7155 case SRC_SHARED_LIMIT_LO:
7156 case SRC_SHARED_LIMIT:
7158 case SRC_PRIVATE_BASE_LO:
7159 case SRC_PRIVATE_BASE:
7160 case SRC_PRIVATE_LIMIT_LO:
7161 case SRC_PRIVATE_LIMIT:
7163 case SRC_FLAT_SCRATCH_BASE_LO:
7164 case SRC_FLAT_SCRATCH_BASE_HI:
7165 return hasGloballyAddressableScratch();
7166 case SRC_POPS_EXITING_WAVE_ID:
7179 getTargetStreamer().getTargetID()->isXnackSupported();
7209 return hasSGPR102_SGPR103();
7217 ParseStatus Res = parseVOPD(
Operands);
7222 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7234 SMLoc LBraceLoc = getLoc();
7239 auto Loc = getLoc();
7242 Error(Loc,
"expected a register");
7246 RBraceLoc = getLoc();
7251 "expected a comma or a closing square bracket"))
7255 if (
Operands.size() - Prefix > 1) {
7257 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7258 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7267StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7269 setForcedEncodingSize(0);
7270 setForcedDPP(
false);
7271 setForcedSDWA(
false);
7273 if (
Name.consume_back(
"_e64_dpp")) {
7275 setForcedEncodingSize(64);
7278 if (
Name.consume_back(
"_e64")) {
7279 setForcedEncodingSize(64);
7282 if (
Name.consume_back(
"_e32")) {
7283 setForcedEncodingSize(32);
7286 if (
Name.consume_back(
"_dpp")) {
7290 if (
Name.consume_back(
"_sdwa")) {
7291 setForcedSDWA(
true);
7299 unsigned VariantID);
7305 Name = parseMnemonicSuffix(Name);
7311 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7313 bool IsMIMG = Name.starts_with(
"image_");
7316 OperandMode
Mode = OperandMode_Default;
7318 Mode = OperandMode_NSA;
7322 checkUnsupportedInstruction(Name, NameLoc);
7323 if (!Parser.hasPendingError()) {
7326 :
"not a valid operand.";
7346ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7349 if (!trySkipId(Name))
7352 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7356ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7365ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7367 std::function<
bool(int64_t &)> ConvertResult) {
7371 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7375 if (ConvertResult && !ConvertResult(
Value)) {
7376 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7379 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7383ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7385 bool (*ConvertResult)(int64_t &)) {
7394 const unsigned MaxSize = 4;
7398 for (
int I = 0;; ++
I) {
7400 SMLoc Loc = getLoc();
7404 if (
Op != 0 &&
Op != 1)
7405 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7412 if (
I + 1 == MaxSize)
7413 return Error(getLoc(),
"expected a closing square bracket");
7419 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7423ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7425 AMDGPUOperand::ImmTy ImmTy,
7426 bool IgnoreNegative) {
7430 if (trySkipId(Name)) {
7432 }
else if (trySkipId(
"no", Name)) {
7441 return Error(S,
"r128 modifier is not supported on this GPU");
7442 if (Name ==
"a16" && !
hasA16())
7443 return Error(S,
"a16 modifier is not supported on this GPU");
7445 if (Bit == 0 && Name ==
"gds") {
7448 return Error(S,
"nogds is not allowed");
7451 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7452 ImmTy = AMDGPUOperand::ImmTyR128A16;
7454 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7458unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7459 bool &Disabling)
const {
7460 Disabling =
Id.consume_front(
"no");
7463 return StringSwitch<unsigned>(Id)
7470 return StringSwitch<unsigned>(Id)
7480 SMLoc StringLoc = getLoc();
7482 int64_t CPolVal = 0;
7502 ResScope = parseScope(
Operands, Scope);
7515 if (trySkipId(
"nv")) {
7519 }
else if (trySkipId(
"no",
"nv")) {
7526 if (trySkipId(
"scale_offset")) {
7530 }
else if (trySkipId(
"no",
"scale_offset")) {
7543 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7544 AMDGPUOperand::ImmTyCPol));
7549 SMLoc OpLoc = getLoc();
7550 unsigned Enabled = 0, Seen = 0;
7554 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7561 return Error(S,
"dlc modifier is not supported on this GPU");
7564 return Error(S,
"scc modifier is not supported on this GPU");
7567 return Error(S,
"duplicate cache policy modifier");
7579 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7588 ParseStatus Res = parseStringOrIntWithPrefix(
7589 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7603 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7607 if (
Value ==
"TH_DEFAULT")
7609 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7610 Value ==
"TH_LOAD_NT_WB") {
7611 return Error(StringLoc,
"invalid th value");
7612 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7614 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7616 }
else if (
Value.consume_front(
"TH_STORE_")) {
7619 return Error(StringLoc,
"invalid th value");
7622 if (
Value ==
"BYPASS")
7627 TH |= StringSwitch<int64_t>(
Value)
7637 .Default(0xffffffff);
7639 TH |= StringSwitch<int64_t>(
Value)
7650 .Default(0xffffffff);
7653 if (TH == 0xffffffff)
7654 return Error(StringLoc,
"invalid th value");
7661 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7662 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7663 std::optional<unsigned> InsertAt = std::nullopt) {
7664 auto i = OptionalIdx.find(ImmT);
7665 if (i != OptionalIdx.end()) {
7666 unsigned Idx = i->second;
7667 const AMDGPUOperand &
Op =
7668 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7672 Op.addImmOperands(Inst, 1);
7674 if (InsertAt.has_value())
7681ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7687 StringLoc = getLoc();
7692ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7698 SMLoc StringLoc = getLoc();
7702 Value = getTokenStr();
7706 if (
Value == Ids[IntVal])
7711 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7712 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7717ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7719 AMDGPUOperand::ImmTy
Type) {
7723 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7725 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7734bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7737 SMLoc Loc = getLoc();
7739 auto Res = parseIntWithPrefix(Pref, Val);
7745 if (Val < 0 || Val > MaxVal) {
7746 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7755 AMDGPUOperand::ImmTy ImmTy) {
7756 const char *Pref =
"index_key";
7758 SMLoc Loc = getLoc();
7759 auto Res = parseIntWithPrefix(Pref, ImmVal);
7763 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7764 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7765 (ImmVal < 0 || ImmVal > 1))
7766 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7768 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7769 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7771 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7776 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7780 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7784 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7789 AMDGPUOperand::ImmTy
Type) {
7795 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7796 AMDGPUOperand::ImmTyMatrixAFMT);
7800 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7801 AMDGPUOperand::ImmTyMatrixBFMT);
7806 AMDGPUOperand::ImmTy
Type) {
7812 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7813 AMDGPUOperand::ImmTyMatrixAScale);
7817 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7818 AMDGPUOperand::ImmTyMatrixBScale);
7823 AMDGPUOperand::ImmTy
Type) {
7829 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7830 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7834 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7835 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7840ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7841 using namespace llvm::AMDGPU::MTBUFFormat;
7847 for (
int I = 0;
I < 2; ++
I) {
7848 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7851 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7856 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7862 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7865 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7866 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7872ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7873 using namespace llvm::AMDGPU::MTBUFFormat;
7877 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7880 if (Fmt == UFMT_UNDEF)
7887bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7888 StringRef FormatStr, SMLoc Loc) {
7889 using namespace llvm::AMDGPU::MTBUFFormat;
7893 if (
Format != DFMT_UNDEF) {
7899 if (
Format != NFMT_UNDEF) {
7904 Error(Loc,
"unsupported format");
7908ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7911 using namespace llvm::AMDGPU::MTBUFFormat;
7915 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7920 SMLoc Loc = getLoc();
7921 if (!parseId(Str,
"expected a format string") ||
7922 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7924 if (Dfmt == DFMT_UNDEF)
7925 return Error(Loc,
"duplicate numeric format");
7926 if (Nfmt == NFMT_UNDEF)
7927 return Error(Loc,
"duplicate data format");
7930 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7931 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7935 if (Ufmt == UFMT_UNDEF)
7936 return Error(FormatLoc,
"unsupported format");
7945ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7948 using namespace llvm::AMDGPU::MTBUFFormat;
7951 if (Id == UFMT_UNDEF)
7955 return Error(Loc,
"unified format is not supported on this GPU");
7961ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7962 using namespace llvm::AMDGPU::MTBUFFormat;
7963 SMLoc Loc = getLoc();
7968 return Error(Loc,
"out of range format");
7973ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7974 using namespace llvm::AMDGPU::MTBUFFormat;
7980 StringRef FormatStr;
7981 SMLoc Loc = getLoc();
7982 if (!parseId(FormatStr,
"expected a format string"))
7985 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7987 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7997 return parseNumericFormat(
Format);
8001 using namespace llvm::AMDGPU::MTBUFFormat;
8005 SMLoc Loc = getLoc();
8015 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
8034 Res = parseSymbolicOrNumericFormat(
Format);
8039 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
8040 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
8047 return Error(getLoc(),
"duplicate format");
8053 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
8055 Res = parseIntWithPrefix(
"inst_offset",
Operands,
8056 AMDGPUOperand::ImmTyInstOffset);
8063 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8065 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8071 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8074 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8084 OptionalImmIndexMap OptionalIdx;
8086 unsigned OperandIdx[4];
8087 unsigned EnMask = 0;
8090 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8091 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8096 OperandIdx[SrcIdx] = Inst.
size();
8097 Op.addRegOperands(Inst, 1);
8104 OperandIdx[SrcIdx] = Inst.
size();
8110 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8111 Op.addImmOperands(Inst, 1);
8115 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8119 OptionalIdx[
Op.getImmTy()] = i;
8125 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8132 for (
auto i = 0; i < SrcIdx; ++i) {
8134 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8140 AMDGPUOperand::ImmTyExpCompr);
8150 int64_t CntVal,
bool Saturate,
8156 IntVal =
encode(ISA, IntVal, CntVal);
8157 if (CntVal !=
decode(ISA, IntVal)) {
8159 IntVal =
encode(ISA, IntVal, -1);
8167bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8169 SMLoc CntLoc = getLoc();
8170 StringRef CntName = getTokenStr();
8177 SMLoc ValLoc = getLoc();
8184 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8186 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8188 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8191 Error(CntLoc,
"invalid counter name " + CntName);
8196 Error(ValLoc,
"too large value for " + CntName);
8205 Error(getLoc(),
"expected a counter name");
8219 if (!parseCnt(Waitcnt))
8227 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8231bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8232 SMLoc FieldLoc = getLoc();
8233 StringRef FieldName = getTokenStr();
8238 SMLoc ValueLoc = getLoc();
8245 if (FieldName ==
"instid0") {
8247 }
else if (FieldName ==
"instskip") {
8249 }
else if (FieldName ==
"instid1") {
8252 Error(FieldLoc,
"invalid field name " + FieldName);
8271 .Case(
"VALU_DEP_1", 1)
8272 .Case(
"VALU_DEP_2", 2)
8273 .Case(
"VALU_DEP_3", 3)
8274 .Case(
"VALU_DEP_4", 4)
8275 .Case(
"TRANS32_DEP_1", 5)
8276 .Case(
"TRANS32_DEP_2", 6)
8277 .Case(
"TRANS32_DEP_3", 7)
8278 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8279 .Case(
"SALU_CYCLE_1", 9)
8280 .Case(
"SALU_CYCLE_2", 10)
8281 .Case(
"SALU_CYCLE_3", 11)
8289 Delay |=
Value << Shift;
8299 if (!parseDelay(Delay))
8307 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8311bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8313bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8319void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8320 StringRef DepCtrName) {
8323 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8326 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8329 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8332 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8339bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8341 using namespace llvm::AMDGPU::DepCtr;
8343 SMLoc DepCtrLoc = getLoc();
8344 StringRef DepCtrName = getTokenStr();
8354 unsigned PrevOprMask = UsedOprMask;
8355 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8358 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8367 Error(getLoc(),
"expected a counter name");
8372 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8373 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8378 using namespace llvm::AMDGPU::DepCtr;
8381 SMLoc Loc = getLoc();
8384 unsigned UsedOprMask = 0;
8386 if (!parseDepCtr(DepCtr, UsedOprMask))
8394 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8398bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8404ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8406 OperandInfoTy &Width) {
8407 using namespace llvm::AMDGPU::Hwreg;
8413 HwReg.Loc = getLoc();
8416 HwReg.IsSymbolic =
true;
8418 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8426 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8436 Width.Loc = getLoc();
8445 using namespace llvm::AMDGPU::Hwreg;
8448 SMLoc Loc = getLoc();
8450 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8452 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8453 HwregOffset::Default);
8454 struct : StructuredOpField {
8455 using StructuredOpField::StructuredOpField;
8456 bool validate(AMDGPUAsmParser &Parser)
const override {
8458 return Error(Parser,
"only values from 1 to 32 are legal");
8461 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8462 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8465 Res = parseHwregFunc(HwReg,
Offset, Width);
8468 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8470 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8474 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8481 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8483 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8487bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8493bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8494 OperandInfoTy &Stream) {
8495 using namespace llvm::AMDGPU::SendMsg;
8500 Msg.IsSymbolic =
true;
8507 Op.IsDefined =
true;
8513 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8518 Stream.IsDefined =
true;
8519 Stream.Loc = getLoc();
8528bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8529 const OperandInfoTy &
Op,
8530 const OperandInfoTy &Stream) {
8531 using namespace llvm::AMDGPU::SendMsg;
8540 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8545 Error(
Msg.Loc,
"invalid message id");
8551 Error(
Op.Loc,
"message does not support operations");
8553 Error(
Msg.Loc,
"missing message operation");
8559 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8561 Error(
Op.Loc,
"invalid operation id");
8566 Error(Stream.Loc,
"message operation does not support streams");
8570 Error(Stream.Loc,
"invalid message stream id");
8577 using namespace llvm::AMDGPU::SendMsg;
8580 SMLoc Loc = getLoc();
8584 OperandInfoTy
Op(OP_NONE_);
8585 OperandInfoTy Stream(STREAM_ID_NONE_);
8586 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8591 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8593 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8599 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8603bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8606 using namespace llvm::AMDGPU::WaitEvent;
8608 SMLoc Loc = getLoc();
8611 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8613 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8615 StructuredOpField *TargetBitfield =
8616 isGFX11() ? &DontWaitExportReady : &ExportReady;
8618 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8622 if (!validateStructuredOpFields({TargetBitfield}))
8624 ImmVal = TargetBitfield->Val;
8631 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8633 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8634 AMDGPUOperand::ImmTyWaitEvent));
8638bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8651 int Slot = StringSwitch<int>(Str)
8658 return Error(S,
"invalid interpolation slot");
8661 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8672 if (!Str.starts_with(
"attr"))
8673 return Error(S,
"invalid interpolation attribute");
8675 StringRef Chan = Str.take_back(2);
8676 int AttrChan = StringSwitch<int>(Chan)
8683 return Error(S,
"invalid or missing interpolation attribute channel");
8685 Str = Str.drop_back(2).drop_front(4);
8688 if (Str.getAsInteger(10, Attr))
8689 return Error(S,
"invalid or missing interpolation attribute number");
8692 return Error(S,
"out of bounds interpolation attribute number");
8697 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8698 Operands.push_back(AMDGPUOperand::CreateImm(
8699 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8708 using namespace llvm::AMDGPU::Exp;
8718 return Error(S, (Id == ET_INVALID)
8719 ?
"invalid exp target"
8720 :
"exp target is not supported on this GPU");
8723 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8731bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8735bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8740 return getTokenKind() ==
Kind;
8743StringRef AMDGPUAsmParser::getId()
const {
8747bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8755bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8757 StringRef Tok = getTokenStr();
8766bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8768 if (isId(Id) && peekToken().is(Kind)) {
8777 if (isToken(Kind)) {
8785 const StringRef ErrMsg) {
8786 if (!trySkipToken(Kind)) {
8787 Error(getLoc(), ErrMsg);
8793bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8797 if (Parser.parseExpression(Expr))
8800 if (Expr->evaluateAsAbsolute(
Imm))
8803 if (Expected.empty()) {
8804 Error(S,
"expected absolute expression");
8807 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8816 if (Parser.parseExpression(Expr))
8820 if (Expr->evaluateAsAbsolute(IntVal)) {
8821 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8823 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8828bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8830 Val =
getToken().getStringContents();
8834 Error(getLoc(), ErrMsg);
8838bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8840 Val = getTokenStr();
8844 if (!ErrMsg.
empty())
8845 Error(getLoc(), ErrMsg);
8849AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8851AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8854 : getLexer().peekTok(ShouldSkipSpace);
8858 auto TokCount = getLexer().peekTokens(Tokens);
8860 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8865 return getLexer().getKind();
8868SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8870StringRef AMDGPUAsmParser::getTokenStr()
const {
8874void AMDGPUAsmParser::lex() { Parser.Lex(); }
8876const AMDGPUOperand &
8878 int MCOpIdx)
const {
8880 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8881 if (TargetOp.getMCOpIdx() == MCOpIdx)
8888 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8892SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8897 int MCOpIdx)
const {
8898 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8901SMLoc AMDGPUAsmParser::getOperandLoc(
8902 std::function<
bool(
const AMDGPUOperand &)>
Test,
8904 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8905 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8907 return Op.getStartLoc();
8912SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8914 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8929 StringRef
Id = getTokenStr();
8930 SMLoc IdLoc = getLoc();
8936 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8937 if (
I == Fields.
end())
8938 return Error(IdLoc,
"unknown field");
8939 if ((*I)->IsDefined)
8940 return Error(IdLoc,
"duplicate field");
8943 (*I)->Loc = getLoc();
8946 (*I)->IsDefined =
true;
8953bool AMDGPUAsmParser::validateStructuredOpFields(
8955 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8956 return F->validate(*
this);
8966 const unsigned XorMask) {
8973bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8974 const unsigned MaxVal,
8975 const Twine &ErrMsg, SMLoc &Loc) {
8991bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8992 const unsigned MinVal,
8993 const unsigned MaxVal,
8994 const StringRef ErrMsg) {
8996 for (
unsigned i = 0; i < OpNum; ++i) {
8997 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
9004bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
9005 using namespace llvm::AMDGPU::Swizzle;
9008 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
9009 "expected a 2-bit lane id")) {
9019bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
9020 using namespace llvm::AMDGPU::Swizzle;
9026 if (!parseSwizzleOperand(GroupSize, 2, 32,
9027 "group size must be in the interval [2,32]", Loc)) {
9031 Error(Loc,
"group size must be a power of two");
9034 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
9035 "lane id must be in the interval [0,group size - 1]",
9043bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
9044 using namespace llvm::AMDGPU::Swizzle;
9049 if (!parseSwizzleOperand(GroupSize, 2, 32,
9050 "group size must be in the interval [2,32]", Loc)) {
9054 Error(Loc,
"group size must be a power of two");
9062bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9063 using namespace llvm::AMDGPU::Swizzle;
9068 if (!parseSwizzleOperand(GroupSize, 1, 16,
9069 "group size must be in the interval [1,16]", Loc)) {
9073 Error(Loc,
"group size must be a power of two");
9081bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9082 using namespace llvm::AMDGPU::Swizzle;
9089 SMLoc StrLoc = getLoc();
9090 if (!parseString(Ctl)) {
9093 if (Ctl.
size() != BITMASK_WIDTH) {
9094 Error(StrLoc,
"expected a 5-character mask");
9098 unsigned AndMask = 0;
9099 unsigned OrMask = 0;
9100 unsigned XorMask = 0;
9102 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9106 Error(StrLoc,
"invalid mask");
9127bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9128 using namespace llvm::AMDGPU::Swizzle;
9131 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9137 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9138 "FFT swizzle must be in the interval [0," +
9139 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9147bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9148 using namespace llvm::AMDGPU::Swizzle;
9151 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9158 if (!parseSwizzleOperand(
Direction, 0, 1,
9159 "direction must be 0 (left) or 1 (right)", Loc))
9163 if (!parseSwizzleOperand(
9164 RotateSize, 0, ROTATE_MAX_SIZE,
9165 "number of threads to rotate must be in the interval [0," +
9166 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9171 (RotateSize << ROTATE_SIZE_SHIFT);
9175bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9177 SMLoc OffsetLoc = getLoc();
9183 Error(OffsetLoc,
"expected a 16-bit offset");
9189bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9190 using namespace llvm::AMDGPU::Swizzle;
9194 SMLoc ModeLoc = getLoc();
9197 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9198 Ok = parseSwizzleQuadPerm(
Imm);
9199 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9200 Ok = parseSwizzleBitmaskPerm(
Imm);
9201 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9202 Ok = parseSwizzleBroadcast(
Imm);
9203 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9204 Ok = parseSwizzleSwap(
Imm);
9205 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9206 Ok = parseSwizzleReverse(
Imm);
9207 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9208 Ok = parseSwizzleFFT(
Imm);
9209 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9210 Ok = parseSwizzleRotate(
Imm);
9212 Error(ModeLoc,
"expected a swizzle mode");
9215 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9225 if (trySkipId(
"offset")) {
9229 if (trySkipId(
"swizzle")) {
9230 Ok = parseSwizzleMacro(
Imm);
9232 Ok = parseSwizzleOffset(
Imm);
9237 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9244bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9250int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9252 using namespace llvm::AMDGPU::VGPRIndexMode;
9264 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9265 if (trySkipId(IdSymbolic[ModeId])) {
9273 ?
"expected a VGPR index mode or a closing parenthesis"
9274 :
"expected a VGPR index mode");
9279 Error(S,
"duplicate VGPR index mode");
9287 "expected a comma or a closing parenthesis"))
9296 using namespace llvm::AMDGPU::VGPRIndexMode;
9302 Imm = parseGPRIdxMacro();
9306 if (getParser().parseAbsoluteExpression(
Imm))
9309 return Error(S,
"invalid immediate: only 4-bit values are legal");
9313 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9317bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9328 if (isRegister() || isModifier())
9335 assert(Opr.isImm() || Opr.isExpr());
9336 SMLoc Loc = Opr.getStartLoc();
9340 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9341 Error(Loc,
"expected an absolute expression or a label");
9342 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9343 Error(Loc,
"expected a 16-bit signed jump offset");
9363 OptionalImmIndexMap OptionalIdx;
9364 unsigned FirstOperandIdx = 1;
9365 bool IsAtomicReturn =
false;
9371 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9372 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9376 Op.addRegOperands(Inst, 1);
9380 if (IsAtomicReturn && i == FirstOperandIdx)
9381 Op.addRegOperands(Inst, 1);
9386 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9387 Op.addImmOperands(Inst, 1);
9399 OptionalIdx[
Op.getImmTy()] = i;
9403 AMDGPUOperand::ImmTyOffset);
9415bool AMDGPUOperand::isSMRDOffset8()
const {
9419bool AMDGPUOperand::isSMEMOffset()
const {
9421 return isImmLiteral();
9424bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9459bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9460 if (BoundCtrl == 0 || BoundCtrl == 1) {
9468void AMDGPUAsmParser::onBeginOfFile() {
9469 if (!getParser().getStreamer().getTargetStreamer())
9472 if (!getTargetStreamer().getTargetID())
9473 getTargetStreamer().initializeTargetID(getSTI(),
9477void AMDGPUAsmParser::emitTargetDirective() {
9478 if (TargetDirectiveEmitted)
9480 TargetDirectiveEmitted =
true;
9482 if (!getParser().getStreamer().getTargetStreamer() ||
9487 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9496bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9500 StringRef TokenId = getTokenStr();
9501 AGVK VK = StringSwitch<AGVK>(TokenId)
9502 .Case(
"max", AGVK::AGVK_Max)
9503 .Case(
"min", AGVK::AGVK_Min)
9504 .Case(
"or", AGVK::AGVK_Or)
9505 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9506 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9507 .Case(
"alignto", AGVK::AGVK_AlignTo)
9508 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9509 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9510 .Default(AGVK::AGVK_None);
9519 if (Exprs.
empty()) {
9521 "empty " + Twine(TokenId) +
" expression");
9524 if (CommaCount + 1 != Exprs.
size()) {
9526 "mismatch of commas in " + Twine(TokenId) +
" expression");
9530 Expected && Exprs.
size() != Expected) {
9531 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9532 Twine(Expected) +
" operands");
9539 if (getParser().parseExpression(Expr, EndLoc))
9543 if (LastTokenWasComma)
9547 "unexpected token in " + Twine(TokenId) +
" expression");
9553 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9557 StringRef
Name = getTokenStr();
9558 if (Name ==
"mul") {
9559 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9563 if (Name ==
"div") {
9564 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9575 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9580 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9581 AMDGPU::OpName::src2};
9589 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9594 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9596 if (
DstOp.isReg() &&
9601 if ((OpSel & (1 << SrcNum)) != 0)
9607void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9614 OptionalImmIndexMap &OptionalIdx) {
9615 cvtVOP3P(Inst,
Operands, OptionalIdx);
9624 &&
Desc.NumOperands > (OpNum + 1)
9626 &&
Desc.operands()[OpNum + 1].RegClass != -1
9628 &&
Desc.getOperandConstraint(OpNum + 1,
9632void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9634 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9635 AMDGPU::OpName::src2};
9636 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9637 AMDGPU::OpName::src1_modifiers,
9638 AMDGPU::OpName::src2_modifiers};
9639 for (
int J = 0; J < 3; ++J) {
9640 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9646 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9649 if ((OpSel & (1 << J)) != 0)
9652 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9659void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9661 OptionalImmIndexMap OptionalIdx;
9666 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9667 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9671 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9673 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9674 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9675 Op.isInterpAttrChan()) {
9677 }
else if (
Op.isImmModifier()) {
9678 OptionalIdx[
Op.getImmTy()] =
I;
9686 AMDGPUOperand::ImmTyHigh);
9690 AMDGPUOperand::ImmTyClamp);
9694 AMDGPUOperand::ImmTyOModSI);
9699 AMDGPUOperand::ImmTyOpSel);
9700 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9703 cvtOpSelHelper(Inst, OpSel);
9708 OptionalImmIndexMap OptionalIdx;
9713 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9714 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9718 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9720 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9721 }
else if (
Op.isImmModifier()) {
9722 OptionalIdx[
Op.getImmTy()] =
I;
9730 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9733 AMDGPUOperand::ImmTyOpSel);
9736 AMDGPUOperand::ImmTyWaitEXP);
9742 cvtOpSelHelper(Inst, OpSel);
9745void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9747 OptionalImmIndexMap OptionalIdx;
9750 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9754 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9755 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9758 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9763 if (NumOperands == CbszOpIdx) {
9768 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9769 }
else if (
Op.isImmModifier()) {
9770 OptionalIdx[
Op.getImmTy()] =
I;
9772 Op.addRegOrImmOperands(Inst, 1);
9777 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9778 if (CbszIdx != OptionalIdx.end()) {
9779 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9783 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9784 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9785 if (BlgpIdx != OptionalIdx.end()) {
9786 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9797 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9798 if (OpselIdx != OptionalIdx.end()) {
9799 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9803 unsigned OpSelHi = 0;
9804 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9805 if (OpselHiIdx != OptionalIdx.end()) {
9806 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9809 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9810 AMDGPU::OpName::src1_modifiers};
9812 for (
unsigned J = 0; J < 2; ++J) {
9813 unsigned ModVal = 0;
9814 if (OpSel & (1 << J))
9816 if (OpSelHi & (1 << J))
9819 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9825 OptionalImmIndexMap &OptionalIdx) {
9830 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9831 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9835 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9837 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9838 }
else if (
Op.isImmModifier()) {
9839 OptionalIdx[
Op.getImmTy()] =
I;
9841 Op.addRegOrImmOperands(Inst, 1);
9847 AMDGPUOperand::ImmTyScaleSel);
9851 AMDGPUOperand::ImmTyClamp);
9857 AMDGPUOperand::ImmTyByteSel);
9862 AMDGPUOperand::ImmTyOModSI);
9869 auto *it = Inst.
begin();
9871 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9880 OptionalImmIndexMap OptionalIdx;
9881 cvtVOP3(Inst,
Operands, OptionalIdx);
9885 OptionalImmIndexMap &OptIdx) {
9890 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9891 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9892 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9893 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9894 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9895 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9896 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9897 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9898 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9899 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9908 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9909 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9912 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9913 if (BitOp3Idx != -1) {
9920 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9921 if (OpSelIdx != -1) {
9925 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9926 if (OpSelHiIdx != -1) {
9927 int DefaultVal =
IsPacked ? -1 : 0;
9933 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9934 if (MatrixAFMTIdx != -1) {
9936 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9940 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9941 if (MatrixBFMTIdx != -1) {
9943 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9946 int MatrixAScaleIdx =
9947 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9948 if (MatrixAScaleIdx != -1) {
9950 AMDGPUOperand::ImmTyMatrixAScale, 0);
9953 int MatrixBScaleIdx =
9954 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9955 if (MatrixBScaleIdx != -1) {
9957 AMDGPUOperand::ImmTyMatrixBScale, 0);
9960 int MatrixAScaleFmtIdx =
9961 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9962 if (MatrixAScaleFmtIdx != -1) {
9964 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9967 int MatrixBScaleFmtIdx =
9968 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9969 if (MatrixBScaleFmtIdx != -1) {
9971 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9976 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9980 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9982 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9986 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9990 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9991 AMDGPU::OpName::src2};
9992 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9993 AMDGPU::OpName::src1_modifiers,
9994 AMDGPU::OpName::src2_modifiers};
9997 unsigned OpSelHi = 0;
10001 if (OpSelIdx != -1)
10004 if (OpSelHiIdx != -1)
10007 if (NegLoIdx != -1)
10010 if (NegHiIdx != -1)
10013 for (
int J = 0; J < 3; ++J) {
10014 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
10018 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
10028 uint32_t ModVal = 0;
10030 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
10031 if (SrcOp.
isReg() && getMRI()
10035 if (VGPRSuffixIsHi)
10038 if ((OpSel & (1 << J)) != 0)
10042 if ((OpSelHi & (1 << J)) != 0)
10045 if ((NegLo & (1 << J)) != 0)
10048 if ((NegHi & (1 << J)) != 0)
10056 OptionalImmIndexMap OptIdx;
10062 unsigned i,
unsigned Opc,
10063 AMDGPU::OpName
OpName) {
10064 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10065 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10067 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10073 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10076 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10077 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10079 OptionalImmIndexMap OptIdx;
10080 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10081 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10082 OptIdx[
Op.getImmTy()] = i;
10087 AMDGPUOperand::ImmTyIndexKey8bit);
10091 AMDGPUOperand::ImmTyIndexKey16bit);
10095 AMDGPUOperand::ImmTyIndexKey32bit);
10112 SMLoc S = getLoc();
10115 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10116 SMLoc OpYLoc = getLoc();
10119 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10122 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10131 auto addOp = [&](uint16_t ParsedOprIdx) {
10132 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10134 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10138 Op.addRegOperands(Inst, 1);
10142 Op.addImmOperands(Inst, 1);
10154 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10158 const auto &CInfo = InstInfo[CompIdx];
10159 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10160 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10161 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10162 if (CInfo.hasSrc2Acc())
10163 addOp(CInfo.getIndexOfDstInParsedOperands());
10167 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10168 if (BitOp3Idx != -1) {
10169 OptionalImmIndexMap OptIdx;
10170 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10172 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10182bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10184bool AMDGPUOperand::isDPPCtrl()
const {
10185 using namespace AMDGPU::DPP;
10187 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10190 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10191 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10192 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10193 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10194 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10195 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10196 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10197 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10198 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10199 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10200 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10201 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10210bool AMDGPUOperand::isBLGP()
const {
10214bool AMDGPUOperand::isS16Imm()
const {
10218bool AMDGPUOperand::isU16Imm()
const {
10226bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10231 SMLoc Loc =
getToken().getEndLoc();
10232 Token = std::string(getTokenStr());
10234 if (getLoc() != Loc)
10239 if (!parseId(Suffix))
10243 StringRef DimId = Token;
10258 SMLoc S = getLoc();
10264 SMLoc Loc = getLoc();
10265 if (!parseDimId(Encoding))
10266 return Error(Loc,
"invalid dim value");
10269 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10278 SMLoc S = getLoc();
10287 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10290 for (
size_t i = 0; i < 8; ++i) {
10294 SMLoc Loc = getLoc();
10295 if (getParser().parseAbsoluteExpression(Sels[i]))
10297 if (0 > Sels[i] || 7 < Sels[i])
10298 return Error(Loc,
"expected a 3-bit value");
10301 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10305 for (
size_t i = 0; i < 8; ++i)
10306 DPP8 |= (Sels[i] << (i * 3));
10309 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10313bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10315 if (Ctrl ==
"row_newbcast")
10318 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10321 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10322 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10325 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10326 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10330int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10333 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10337 for (
int i = 0; i < 4; ++i) {
10342 SMLoc Loc = getLoc();
10343 if (getParser().parseAbsoluteExpression(Temp))
10345 if (Temp < 0 || Temp > 3) {
10346 Error(Loc,
"expected a 2-bit value");
10350 Val += (Temp << i * 2);
10353 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10359int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10360 using namespace AMDGPU::DPP;
10365 SMLoc Loc = getLoc();
10367 if (getParser().parseAbsoluteExpression(Val))
10370 struct DppCtrlCheck {
10376 DppCtrlCheck
Check =
10377 StringSwitch<DppCtrlCheck>(Ctrl)
10378 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10379 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10380 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10381 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10382 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10383 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10384 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10385 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10386 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10387 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10391 if (
Check.Ctrl == -1) {
10392 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10400 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10408 using namespace AMDGPU::DPP;
10411 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10414 SMLoc S = getLoc();
10420 if (Ctrl ==
"row_mirror") {
10421 Val = DppCtrl::ROW_MIRROR;
10422 }
else if (Ctrl ==
"row_half_mirror") {
10423 Val = DppCtrl::ROW_HALF_MIRROR;
10426 if (Ctrl ==
"quad_perm") {
10427 Val = parseDPPCtrlPerm();
10429 Val = parseDPPCtrlSel(Ctrl);
10438 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10444 OptionalImmIndexMap OptionalIdx;
10451 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10453 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10454 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10458 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10459 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10463 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10464 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10465 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10466 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10467 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10468 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10469 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10470 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10471 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10477 if (OldIdx == NumOperands) {
10479 constexpr int DST_IDX = 0;
10481 }
else if (Src2ModIdx == NumOperands) {
10491 if (IsVOP3CvtSrDpp) {
10500 if (TiedTo != -1) {
10505 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10507 if (IsDPP8 &&
Op.isDppFI()) {
10510 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10511 }
else if (
Op.isReg()) {
10512 Op.addRegOperands(Inst, 1);
10513 }
else if (
Op.isImm() &&
10515 Op.addImmOperands(Inst, 1);
10516 }
else if (
Op.isImm()) {
10517 OptionalIdx[
Op.getImmTy()] =
I;
10525 AMDGPUOperand::ImmTyClamp);
10531 AMDGPUOperand::ImmTyByteSel);
10536 AMDGPUOperand::ImmTyOModSI);