70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
248 return isRegOrInline(RCID, type) || isLiteralImm(type);
251 bool isRegOrImmWithInt16InputMods()
const {
255 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
257 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
260 bool isRegOrImmWithInt32InputMods()
const {
264 bool isRegOrInlineImmWithInt16InputMods()
const {
265 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFP32InputMods()
const {
294 bool isRegOrImmWithFP64InputMods()
const {
298 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
299 return isRegOrInline(
300 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
303 bool isRegOrInlineImmWithFP32InputMods()
const {
304 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
307 bool isRegOrInlineImmWithFP64InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_64RegClassID, MVT::f64);
311 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
313 bool isVRegWithFP32InputMods()
const {
314 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
317 bool isVRegWithFP64InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VReg_64RegClassID);
321 bool isPackedFP16InputMods()
const {
325 bool isPackedVGPRFP32InputMods()
const {
329 bool isVReg()
const {
330 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
331 isRegClass(AMDGPU::VReg_64RegClassID) ||
332 isRegClass(AMDGPU::VReg_96RegClassID) ||
333 isRegClass(AMDGPU::VReg_128RegClassID) ||
334 isRegClass(AMDGPU::VReg_160RegClassID) ||
335 isRegClass(AMDGPU::VReg_192RegClassID) ||
336 isRegClass(AMDGPU::VReg_256RegClassID) ||
337 isRegClass(AMDGPU::VReg_512RegClassID) ||
338 isRegClass(AMDGPU::VReg_1024RegClassID);
341 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
343 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
345 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
347 bool isAV_LdSt_32_Align2_RegOp()
const {
348 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
349 isRegClass(AMDGPU::AGPR_32RegClassID);
352 bool isVRegWithInputMods()
const;
353 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
354 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
356 bool isSDWAOperand(MVT type)
const;
357 bool isSDWAFP16Operand()
const;
358 bool isSDWAFP32Operand()
const;
359 bool isSDWAInt16Operand()
const;
360 bool isSDWAInt32Operand()
const;
362 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
364 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
366 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
368 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
370 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
371 bool isDim()
const {
return isImmTy(ImmTyDim); }
372 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
373 bool isOff()
const {
return isImmTy(ImmTyOff); }
374 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
375 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
376 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
377 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
378 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
379 bool isFlatOffset()
const {
380 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
382 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
383 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
384 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
385 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
386 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
387 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
388 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
389 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
390 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
391 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
392 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
393 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
394 bool isMatrixAReuse()
const {
return isImmTy(ImmTyMatrixAReuse); }
395 bool isMatrixBReuse()
const {
return isImmTy(ImmTyMatrixBReuse); }
396 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
397 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
398 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
399 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
400 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
401 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
402 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
403 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
404 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
405 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
406 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
407 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
408 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
409 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
410 bool isBitOp3()
const {
return isImmTy(ImmTyBitOp3) &&
isUInt<8>(
getImm()); }
411 bool isDone()
const {
return isImmTy(ImmTyDone); }
412 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
414 bool isRegOrImm()
const {
return isReg() || isImm(); }
416 bool isRegClass(
unsigned RCID)
const;
420 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
421 return isRegOrInline(RCID, type) && !hasModifiers();
424 bool isSCSrcB16()
const {
425 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
428 bool isSCSrcV2B16()
const {
return isSCSrcB16(); }
430 bool isSCSrc_b32()
const {
431 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
434 bool isSCSrc_b64()
const {
435 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
438 bool isBoolReg()
const;
440 bool isSCSrcF16()
const {
441 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
444 bool isSCSrcV2F16()
const {
return isSCSrcF16(); }
446 bool isSCSrcF32()
const {
447 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
450 bool isSCSrcF64()
const {
451 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
454 bool isSSrc_b32()
const {
455 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
458 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
460 bool isSSrcV2B16()
const {
465 bool isSSrc_b64()
const {
468 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
469 (((
const MCTargetAsmParser *)AsmParser)
470 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
474 bool isSSrc_f32()
const {
475 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
478 bool isSSrcF64()
const {
return isSCSrc_b64() || isLiteralImm(MVT::f64); }
480 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
482 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
484 bool isSSrcV2F16()
const {
489 bool isSSrcV2FP32()
const {
494 bool isSCSrcV2FP32()
const {
499 bool isSSrcV2INT32()
const {
504 bool isSCSrcV2INT32()
const {
506 return isSCSrc_b32();
509 bool isSSrcOrLds_b32()
const {
510 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
511 isLiteralImm(MVT::i32) || isExpr();
514 bool isVCSrc_b32()
const {
515 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
518 bool isVCSrc_b32_Lo256()
const {
519 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
522 bool isVCSrc_b64_Lo256()
const {
523 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
526 bool isVCSrc_b64()
const {
527 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
530 bool isVCSrcT_b16()
const {
531 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
534 bool isVCSrcTB16_Lo128()
const {
535 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
538 bool isVCSrcFake16B16_Lo128()
const {
539 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
542 bool isVCSrc_b16()
const {
543 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
546 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
548 bool isVCSrc_f32()
const {
549 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
552 bool isVCSrc_f64()
const {
553 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
556 bool isVCSrcTBF16()
const {
557 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
560 bool isVCSrcT_f16()
const {
561 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
564 bool isVCSrcT_bf16()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
568 bool isVCSrcTBF16_Lo128()
const {
569 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
572 bool isVCSrcTF16_Lo128()
const {
573 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
576 bool isVCSrcFake16BF16_Lo128()
const {
577 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
580 bool isVCSrcFake16F16_Lo128()
const {
581 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
584 bool isVCSrc_bf16()
const {
585 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
588 bool isVCSrc_f16()
const {
589 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
592 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
594 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
596 bool isVSrc_b32()
const {
597 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
600 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
602 bool isVSrc_v2b64()
const {
603 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
604 isLiteralImm(MVT::i64);
607 bool isVSrc_v2f64()
const {
608 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
609 isLiteralImm(MVT::f64);
612 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
614 bool isVSrcT_b16_Lo128()
const {
615 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
618 bool isVSrcFake16_b16_Lo128()
const {
619 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
622 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
624 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
626 bool isVCSrcV2FP32()
const {
return isVCSrc_f64(); }
628 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
630 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
632 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
634 bool isVSrc_f32()
const {
635 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
638 bool isVSrc_f64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::f64); }
640 bool isVSrcT_bf16()
const {
641 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
644 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
646 bool isVSrcT_bf16_Lo128()
const {
647 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
650 bool isVSrcT_f16_Lo128()
const {
651 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
654 bool isVSrcFake16_bf16_Lo128()
const {
655 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
658 bool isVSrcFake16_f16_Lo128()
const {
659 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
662 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
664 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
666 bool isVSrc_v2bf16()
const {
667 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
670 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
672 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
674 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
676 bool isVISrcB32()
const {
677 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
680 bool isVISrcB16()
const {
681 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
684 bool isVISrcV2B16()
const {
return isVISrcB16(); }
686 bool isVISrcF32()
const {
687 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
690 bool isVISrcF16()
const {
691 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
694 bool isVISrcV2F16()
const {
return isVISrcF16() || isVISrcB32(); }
696 bool isVISrc_64_bf16()
const {
697 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
700 bool isVISrc_64_f16()
const {
701 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
704 bool isVISrc_64_b32()
const {
705 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
708 bool isVISrc_64B64()
const {
709 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
712 bool isVISrc_64_f64()
const {
713 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
716 bool isVISrc_64V2FP32()
const {
717 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
720 bool isVISrc_64V2INT32()
const {
721 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
724 bool isVISrc_256_b32()
const {
725 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
728 bool isVISrc_256_f32()
const {
729 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
732 bool isVISrc_256B64()
const {
733 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
736 bool isVISrc_256_f64()
const {
737 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
740 bool isVISrc_512_f64()
const {
741 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f64);
744 bool isVISrc_128B16()
const {
745 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
748 bool isVISrc_128V2B16()
const {
return isVISrc_128B16(); }
750 bool isVISrc_128_b32()
const {
751 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
754 bool isVISrc_128_f32()
const {
755 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
758 bool isVISrc_256V2FP32()
const {
759 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
762 bool isVISrc_256V2INT32()
const {
763 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
766 bool isVISrc_512_b32()
const {
767 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
770 bool isVISrc_512B16()
const {
771 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
774 bool isVISrc_512V2B16()
const {
return isVISrc_512B16(); }
776 bool isVISrc_512_f32()
const {
777 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
780 bool isVISrc_512F16()
const {
781 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
784 bool isVISrc_512V2F16()
const {
785 return isVISrc_512F16() || isVISrc_512_b32();
788 bool isVISrc_1024_b32()
const {
789 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
792 bool isVISrc_1024B16()
const {
793 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
796 bool isVISrc_1024V2B16()
const {
return isVISrc_1024B16(); }
798 bool isVISrc_1024_f32()
const {
799 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
802 bool isVISrc_1024F16()
const {
803 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
806 bool isVISrc_1024V2F16()
const {
807 return isVISrc_1024F16() || isVISrc_1024_b32();
810 bool isAISrcB32()
const {
811 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
814 bool isAISrcB16()
const {
815 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
818 bool isAISrcV2B16()
const {
return isAISrcB16(); }
820 bool isAISrcF32()
const {
821 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
824 bool isAISrcF16()
const {
825 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
828 bool isAISrcV2F16()
const {
return isAISrcF16() || isAISrcB32(); }
830 bool isAISrc_64B64()
const {
831 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
834 bool isAISrc_64_f64()
const {
835 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
838 bool isAISrc_128_b32()
const {
839 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
842 bool isAISrc_128B16()
const {
843 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
846 bool isAISrc_128V2B16()
const {
return isAISrc_128B16(); }
848 bool isAISrc_128_f32()
const {
849 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
852 bool isAISrc_128F16()
const {
853 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
856 bool isAISrc_128V2F16()
const {
857 return isAISrc_128F16() || isAISrc_128_b32();
860 bool isVISrc_128_bf16()
const {
861 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
864 bool isVISrc_128_f16()
const {
865 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
868 bool isVISrc_128V2F16()
const {
869 return isVISrc_128_f16() || isVISrc_128_b32();
872 bool isAISrc_256B64()
const {
873 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
876 bool isAISrc_256_f64()
const {
877 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
880 bool isAISrc_512_b32()
const {
881 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
884 bool isAISrc_512B16()
const {
885 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
888 bool isAISrc_512V2B16()
const {
return isAISrc_512B16(); }
890 bool isAISrc_512_f32()
const {
891 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
894 bool isAISrc_512F16()
const {
895 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
898 bool isAISrc_512V2F16()
const {
899 return isAISrc_512F16() || isAISrc_512_b32();
902 bool isAISrc_1024_b32()
const {
903 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
906 bool isAISrc_1024B16()
const {
907 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
910 bool isAISrc_1024V2B16()
const {
return isAISrc_1024B16(); }
912 bool isAISrc_1024_f32()
const {
913 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
916 bool isAISrc_1024F16()
const {
917 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
920 bool isAISrc_1024V2F16()
const {
921 return isAISrc_1024F16() || isAISrc_1024_b32();
924 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
926 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
928 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
930 bool isMem()
const override {
return false; }
932 bool isExpr()
const {
return Kind == Expression; }
934 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
936 bool isSWaitCnt()
const;
937 bool isDepCtr()
const;
938 bool isSDelayALU()
const;
939 bool isHwreg()
const;
940 bool isSendMsg()
const;
941 bool isWaitEvent()
const;
942 bool isSplitBarrier()
const;
943 bool isSwizzle()
const;
944 bool isSMRDOffset8()
const;
945 bool isSMEMOffset()
const;
946 bool isSMRDLiteralOffset()
const;
948 bool isDPPCtrl()
const;
950 bool isGPRIdxMode()
const;
951 bool isS16Imm()
const;
952 bool isU16Imm()
const;
953 bool isEndpgm()
const;
955 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
956 return [
this,
P]() {
return P(*
this); };
961 return StringRef(Tok.Data, Tok.Length);
969 void setImm(int64_t Val) {
974 ImmTy getImmTy()
const {
979 MCRegister
getReg()
const override {
984 SMLoc getStartLoc()
const override {
return StartLoc; }
986 SMLoc getEndLoc()
const override {
return EndLoc; }
988 SMRange getLocRange()
const {
return SMRange(StartLoc, EndLoc); }
990 int getMCOpIdx()
const {
return MCOpIdx; }
992 Modifiers getModifiers()
const {
993 assert(isRegKind() || isImmTy(ImmTyNone));
994 return isRegKind() ?
Reg.Mods :
Imm.Mods;
997 void setModifiers(Modifiers Mods) {
998 assert(isRegKind() || isImmTy(ImmTyNone));
1005 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
1007 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
1009 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
1011 bool isForcedLit()
const {
1012 return isImmLiteral() && getModifiers().isForcedLit();
1015 bool isForcedLit64()
const {
1016 return isImmLiteral() && getModifiers().isForcedLit64();
1021 void addImmOperands(MCInst &Inst,
unsigned N,
1022 bool ApplyModifiers =
true)
const;
1024 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1025 bool ApplyModifiers)
const;
1027 void addRegOperands(MCInst &Inst,
unsigned N)
const;
1029 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
1031 addRegOperands(Inst,
N);
1033 addImmOperands(Inst,
N);
1036 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1037 Modifiers Mods = getModifiers();
1040 addRegOperands(Inst,
N);
1042 addImmOperands(Inst,
N,
false);
1046 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1047 assert(!hasIntModifiers());
1048 addRegOrImmWithInputModsOperands(Inst,
N);
1051 void addRegOrImmWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1052 assert(!hasFPModifiers());
1053 addRegOrImmWithInputModsOperands(Inst,
N);
1056 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
1057 Modifiers Mods = getModifiers();
1060 addRegOperands(Inst,
N);
1063 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
1064 assert(!hasIntModifiers());
1065 addRegWithInputModsOperands(Inst,
N);
1068 void addRegWithIntInputModsOperands(MCInst &Inst,
unsigned N)
const {
1069 assert(!hasFPModifiers());
1070 addRegWithInputModsOperands(Inst,
N);
1073 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
1076 case ImmTyNone: OS <<
"None";
break;
1077 case ImmTyGDS: OS <<
"GDS";
break;
1078 case ImmTyLDS: OS <<
"LDS";
break;
1079 case ImmTyOffen: OS <<
"Offen";
break;
1080 case ImmTyIdxen: OS <<
"Idxen";
break;
1081 case ImmTyAddr64: OS <<
"Addr64";
break;
1082 case ImmTyOffset: OS <<
"Offset";
break;
1083 case ImmTyInstOffset: OS <<
"InstOffset";
break;
1084 case ImmTyOffset0: OS <<
"Offset0";
break;
1085 case ImmTyOffset1: OS <<
"Offset1";
break;
1086 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
1087 case ImmTyCPol: OS <<
"CPol";
break;
1088 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
1089 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
1090 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
1091 case ImmTyTFE: OS <<
"TFE";
break;
1092 case ImmTyIsAsync: OS <<
"IsAsync";
break;
1093 case ImmTyD16: OS <<
"D16";
break;
1094 case ImmTyFORMAT: OS <<
"FORMAT";
break;
1095 case ImmTyClamp: OS <<
"Clamp";
break;
1096 case ImmTyOModSI: OS <<
"OModSI";
break;
1097 case ImmTyDPP8: OS <<
"DPP8";
break;
1098 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
1099 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
1100 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
1101 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
1102 case ImmTyDppFI: OS <<
"DppFI";
break;
1103 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
1104 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
1105 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
1106 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
1107 case ImmTyDMask: OS <<
"DMask";
break;
1108 case ImmTyDim: OS <<
"Dim";
break;
1109 case ImmTyUNorm: OS <<
"UNorm";
break;
1110 case ImmTyDA: OS <<
"DA";
break;
1111 case ImmTyR128A16: OS <<
"R128A16";
break;
1112 case ImmTyA16: OS <<
"A16";
break;
1113 case ImmTyLWE: OS <<
"LWE";
break;
1114 case ImmTyOff: OS <<
"Off";
break;
1115 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
1116 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
1117 case ImmTyExpVM: OS <<
"ExpVM";
break;
1118 case ImmTyDone: OS <<
"Done";
break;
1119 case ImmTyRowEn: OS <<
"RowEn";
break;
1120 case ImmTyHwreg: OS <<
"Hwreg";
break;
1121 case ImmTySendMsg: OS <<
"SendMsg";
break;
1122 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
1123 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
1124 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
1125 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
1126 case ImmTyOpSel: OS <<
"OpSel";
break;
1127 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
1128 case ImmTyNegLo: OS <<
"NegLo";
break;
1129 case ImmTyNegHi: OS <<
"NegHi";
break;
1130 case ImmTySwizzle: OS <<
"Swizzle";
break;
1131 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
1132 case ImmTyHigh: OS <<
"High";
break;
1133 case ImmTyBLGP: OS <<
"BLGP";
break;
1134 case ImmTyCBSZ: OS <<
"CBSZ";
break;
1135 case ImmTyABID: OS <<
"ABID";
break;
1136 case ImmTyEndpgm: OS <<
"Endpgm";
break;
1137 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
1138 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
1139 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
1140 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
1141 case ImmTyBitOp3: OS <<
"BitOp3";
break;
1142 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
1143 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
1144 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
1145 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
1146 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
1147 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
1148 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
1149 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
1150 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
1151 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
1156 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
1160 <<
" mods: " <<
Reg.Mods <<
'>';
1164 if (getImmTy() != ImmTyNone) {
1166 printImmTy(OS, getImmTy());
1168 OS <<
" mods: " <<
Imm.Mods <<
'>';
1181 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
1182 int64_t Val, SMLoc Loc,
1183 ImmTy
Type = ImmTyNone,
1184 bool IsFPImm =
false) {
1185 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1187 Op->Imm.IsFPImm = IsFPImm;
1189 Op->Imm.Mods = Modifiers();
1195 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
1196 StringRef Str, SMLoc Loc,
1197 bool HasExplicitEncodingSize =
true) {
1198 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1199 Res->Tok.Data = Str.data();
1200 Res->Tok.Length = Str.size();
1201 Res->StartLoc = Loc;
1206 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1207 MCRegister
Reg, SMLoc S, SMLoc
E) {
1208 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1209 Op->Reg.RegNo =
Reg;
1210 Op->Reg.Mods = Modifiers();
1216 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1217 const class MCExpr *Expr, SMLoc S) {
1218 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1227 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1236#define GET_REGISTER_MATCHER
1237#include "AMDGPUGenAsmMatcher.inc"
1238#undef GET_REGISTER_MATCHER
1239#undef GET_SUBTARGET_FEATURE_NAME
1244class KernelScopeInfo {
1245 int SgprIndexUnusedMin = -1;
1246 int VgprIndexUnusedMin = -1;
1247 int AgprIndexUnusedMin = -1;
1251 void usesSgprAt(
int i) {
1252 if (i >= SgprIndexUnusedMin) {
1253 SgprIndexUnusedMin = ++i;
1256 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1262 void usesVgprAt(
int i) {
1263 if (i >= VgprIndexUnusedMin) {
1264 VgprIndexUnusedMin = ++i;
1267 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1269 VgprIndexUnusedMin);
1275 void usesAgprAt(
int i) {
1280 if (i >= AgprIndexUnusedMin) {
1281 AgprIndexUnusedMin = ++i;
1284 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1289 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1291 VgprIndexUnusedMin);
1298 KernelScopeInfo() =
default;
1302 MSTI = Ctx->getSubtargetInfo();
1304 usesSgprAt(SgprIndexUnusedMin = -1);
1305 usesVgprAt(VgprIndexUnusedMin = -1);
1307 usesAgprAt(AgprIndexUnusedMin = -1);
1311 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1312 unsigned RegWidth) {
1315 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1318 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1321 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1330 MCAsmParser &Parser;
1332 unsigned ForcedEncodingSize = 0;
1333 bool ForcedDPP =
false;
1334 bool ForcedSDWA =
false;
1335 KernelScopeInfo KernelScope;
1336 const unsigned HwMode;
1338 const AMDGPU::IsaVersion ISA;
1343#define GET_ASSEMBLER_HEADER
1344#include "AMDGPUGenAsmMatcher.inc"
1349 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1351 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1355 std::optional<AMDGPU::InfoSectionData> InfoData;
1362 bool TargetDirectiveEmitted =
false;
1371 SmallVector<unsigned> OpcodeStream;
1373 OpcodeStreamSymbols;
1374 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1377 void checkKernelPrologues();
1380 void createConstantSymbol(StringRef Id, int64_t Val);
1382 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1383 bool OutOfRangeError(SMRange
Range);
1399 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1400 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1401 std::optional<bool> EnableWavefrontSize32,
1402 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1403 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1404 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1405 bool ParseDirectiveAMDGCNTarget();
1406 bool ParseDirectiveAMDHSACodeObjectVersion();
1407 bool ParseDirectiveAMDHSAKernel();
1408 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1409 bool ParseDirectiveAMDKernelCodeT();
1411 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1412 bool ParseDirectiveAMDGPUHsaKernel();
1414 bool ParseDirectiveISAVersion();
1415 bool ParseDirectiveHSAMetadata();
1416 bool ParseDirectivePALMetadataBegin();
1417 bool ParseDirectivePALMetadata();
1418 bool ParseDirectiveAMDGPULDS();
1419 bool ParseDirectiveAMDGPUInfo();
1423 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1424 const char *AssemblerDirectiveEnd,
1425 std::string &CollectString);
1427 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1428 RegisterKind RegKind, MCRegister Reg1,
1429 RegisterKind RegKind1, SMLoc Loc);
1430 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1431 unsigned &RegNum,
unsigned &RegWidth,
1432 bool RestoreOnFailure =
false);
1433 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1434 unsigned &RegNum,
unsigned &RegWidth,
1435 SmallVectorImpl<AsmToken> &Tokens);
1436 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1438 SmallVectorImpl<AsmToken> &Tokens);
1439 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1441 SmallVectorImpl<AsmToken> &Tokens);
1442 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1444 SmallVectorImpl<AsmToken> &Tokens);
1445 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1446 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1447 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1450 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1451 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1452 void initializeGprCountSymbol(RegisterKind RegKind);
1453 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1459 OperandMode_Default,
1463 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1465 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1466 const MCInstrInfo &MII)
1467 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1468 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1473 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1475 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1476 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1477 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1478 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1480 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1481 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1482 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1484 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1485 initializeGprCountSymbol(IS_VGPR);
1486 initializeGprCountSymbol(IS_SGPR);
1491 createConstantSymbol(Symbol, Code);
1493 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1494 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1495 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1543 bool hasBVHRayTracingInsts()
const {
1544 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1549 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1551 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1553 bool hasInv2PiInlineImm()
const {
1554 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1557 bool has64BitLiterals()
const {
1558 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1561 bool hasFlatOffsets()
const {
1562 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1565 bool hasTrue16Insts()
const {
1566 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1570 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1573 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1575 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1577 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1579 bool hasPartialNSAEncoding()
const {
1580 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1583 bool hasGloballyAddressableScratch()
const {
1584 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1597 AMDGPUTargetStreamer &getTargetStreamer() {
1598 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1599 return static_cast<AMDGPUTargetStreamer &
>(TS);
1605 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1608 const MCRegisterInfo *getMRI()
const {
1612 const MCInstrInfo *getMII()
const {
return &MII; }
1616 const FeatureBitset &getFeatureBits()
const {
1617 return getSTI().getFeatureBits();
1620 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1621 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1622 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1624 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1625 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1626 bool isForcedDPP()
const {
return ForcedDPP; }
1627 bool isForcedSDWA()
const {
return ForcedSDWA; }
1628 ArrayRef<unsigned> getMatchedVariants()
const;
1629 StringRef getMatchedVariantName()
const;
1631 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1632 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1633 bool RestoreOnFailure);
1634 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1635 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1636 SMLoc &EndLoc)
override;
1637 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1638 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1639 unsigned Kind)
override;
1640 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1643 bool MatchingInlineAsm)
override;
1644 bool ParseDirective(AsmToken DirectiveID)
override;
1645 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1646 void onEndOfFile()
override;
1648 OperandMode
Mode = OperandMode_Default);
1649 StringRef parseMnemonicSuffix(StringRef Name);
1650 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1656 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1660 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1661 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1663 ParseStatus parseOperandArrayWithPrefix(
1665 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1666 bool (*ConvertResult)(int64_t &) =
nullptr);
1670 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1671 bool IgnoreNegative =
false);
1672 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1676 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1680 ArrayRef<const char *> Ids,
1684 ArrayRef<const char *> Ids,
1685 AMDGPUOperand::ImmTy
Type);
1688 bool isOperandModifier(
const AsmToken &Token,
1689 const AsmToken &NextToken)
const;
1690 bool isRegOrOperandModifier(
const AsmToken &Token,
1691 const AsmToken &NextToken)
const;
1692 bool isNamedOperandModifier(
const AsmToken &Token,
1693 const AsmToken &NextToken)
const;
1694 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1695 const AsmToken &NextToken)
const;
1696 bool parseSP3NegModifier();
1703 bool AllowImm =
true);
1705 bool AllowImm =
true);
1710 AMDGPUOperand::ImmTy ImmTy);
1715 AMDGPUOperand::ImmTy
Type);
1719 AMDGPUOperand::ImmTy
Type);
1723 AMDGPUOperand::ImmTy
Type);
1727 ParseStatus parseDfmtNfmt(int64_t &
Format);
1728 ParseStatus parseUfmt(int64_t &
Format);
1729 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1731 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1734 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1735 ParseStatus parseNumericFormat(int64_t &
Format);
1739 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1740 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1745 bool parseCnt(int64_t &IntVal);
1748 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1749 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1752 bool parseDelay(int64_t &Delay);
1758 struct OperandInfoTy {
1761 bool IsSymbolic =
false;
1762 bool IsDefined =
false;
1764 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1767 struct StructuredOpField : OperandInfoTy {
1771 bool IsDefined =
false;
1773 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1774 unsigned Width, int64_t
Default)
1775 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1776 virtual ~StructuredOpField() =
default;
1778 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1779 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1783 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1785 return Error(Parser,
"not supported on this GPU");
1787 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1795 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1796 OperandInfoTy &Stream);
1797 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1798 const OperandInfoTy &Stream);
1800 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1801 OperandInfoTy &Width);
1806 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1813 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1815 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1819 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1825 bool validateConstantBusLimitations(
const MCInst &Inst,
1827 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1830 bool tryVOPD(
const MCInst &Inst);
1831 bool tryVOPD3(
const MCInst &Inst);
1832 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1834 bool validateIntClampSupported(
const MCInst &Inst);
1835 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1836 bool validateMIMGGatherDMask(
const MCInst &Inst);
1838 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1839 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1840 bool validateMIMGD16(
const MCInst &Inst);
1842 bool validateTensorR128(
const MCInst &Inst);
1843 bool validateMIMGMSAA(
const MCInst &Inst);
1844 bool validateOpSel(
const MCInst &Inst);
1845 bool validateTrue16OpSel(
const MCInst &Inst);
1846 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1848 bool validateVccOperand(MCRegister
Reg)
const;
1853 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1854 bool validateVGPRAlign(
const MCInst &Inst)
const;
1858 bool validateDivScale(
const MCInst &Inst);
1863 const unsigned CPol);
1867 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1868 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1869 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1870 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1872 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1873 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1874 ArrayRef<unsigned> Variants);
1875 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1877 bool isId(
const StringRef Id)
const;
1878 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1880 StringRef getId()
const;
1881 bool trySkipId(
const StringRef Id);
1882 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1886 bool parseString(StringRef &Val,
1887 const StringRef ErrMsg =
"expected a string");
1888 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1894 StringRef getTokenStr()
const;
1895 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1897 SMLoc getLoc()
const;
1901 void onBeginOfFile()
override;
1905 void emitTargetDirective();
1906 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1918 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1919 const unsigned MaxVal,
const Twine &ErrMsg,
1921 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1922 const unsigned MinVal,
const unsigned MaxVal,
1923 const StringRef ErrMsg);
1925 bool parseSwizzleOffset(int64_t &
Imm);
1926 bool parseSwizzleMacro(int64_t &
Imm);
1927 bool parseSwizzleQuadPerm(int64_t &
Imm);
1928 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1929 bool parseSwizzleBroadcast(int64_t &
Imm);
1930 bool parseSwizzleSwap(int64_t &
Imm);
1931 bool parseSwizzleReverse(int64_t &
Imm);
1932 bool parseSwizzleFFT(int64_t &
Imm);
1933 bool parseSwizzleRotate(int64_t &
Imm);
1936 int64_t parseGPRIdxMacro();
1939 cvtMubufImpl(Inst,
Operands,
false);
1942 cvtMubufImpl(Inst,
Operands,
true);
1948 OptionalImmIndexMap &OptionalIdx);
1957 OptionalImmIndexMap &OptionalIdx);
1959 OptionalImmIndexMap &OptionalIdx);
1963 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1965 bool parseDimId(
unsigned &Encoding);
1967 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1971 int64_t parseDPPCtrlSel(StringRef Ctrl);
1972 int64_t parseDPPCtrlPerm();
1978 bool IsDPP8 =
false);
1984 AMDGPUOperand::ImmTy
Type);
1992 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1995 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1996 bool SkipSrcVcc =
false);
2105bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
2115 if (!isImmTy(ImmTyNone)) {
2120 if (getModifiers().
Lit != LitModifier::None)
2130 if (type == MVT::f64 || type == MVT::i64) {
2132 AsmParser->hasInv2PiInlineImm());
2135 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2154 APFloat::rmNearestTiesToEven, &Lost);
2161 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2163 AsmParser->hasInv2PiInlineImm());
2168 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
2169 AsmParser->hasInv2PiInlineImm());
2173 if (type == MVT::f64 || type == MVT::i64) {
2175 AsmParser->hasInv2PiInlineImm());
2184 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
2185 AsmParser->hasInv2PiInlineImm());
2189 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
2190 AsmParser->hasInv2PiInlineImm());
2193bool AMDGPUOperand::isLiteralImm(MVT type)
const {
2195 if (!isImmTy(ImmTyNone)) {
2200 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2205 if (type == MVT::f64 && hasFPModifiers()) {
2225 if (type == MVT::f64) {
2230 if (type == MVT::i64) {
2243 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2244 : (type == MVT::v2i16) ? MVT::f32
2245 : (type == MVT::v2f32) ? MVT::f32
2248 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2252bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2253 return isRegKind() &&
2254 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2257bool AMDGPUOperand::isVRegWithInputMods()
const {
2258 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2260 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2261 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2264template <
bool IsFake16>
2265bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2266 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2267 : AMDGPU::VGPR_16_Lo128RegClassID);
2270template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2271 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2272 : AMDGPU::VGPR_16RegClassID);
2275bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2276 if (AsmParser->isVI())
2278 if (AsmParser->isGFX9Plus())
2279 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2283bool AMDGPUOperand::isSDWAFP16Operand()
const {
2284 return isSDWAOperand(MVT::f16);
2287bool AMDGPUOperand::isSDWAFP32Operand()
const {
2288 return isSDWAOperand(MVT::f32);
2291bool AMDGPUOperand::isSDWAInt16Operand()
const {
2292 return isSDWAOperand(MVT::i16);
2295bool AMDGPUOperand::isSDWAInt32Operand()
const {
2296 return isSDWAOperand(MVT::i32);
2299bool AMDGPUOperand::isBoolReg()
const {
2300 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2301 (AsmParser->isWave32() && isSCSrc_b32()));
2305 unsigned Size)
const {
2306 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2321void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2322 bool ApplyModifiers)
const {
2332 addLiteralImmOperand(Inst,
Imm.Val,
2333 ApplyModifiers & isImmTy(ImmTyNone) &&
2334 Imm.Mods.hasFPModifiers());
2336 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2341void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2342 bool ApplyModifiers)
const {
2343 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2348 if (ApplyModifiers) {
2350 const unsigned Size =
2352 Val = applyInputFPModifiers(Val,
Size);
2356 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2358 bool CanUse64BitLiterals =
2361 MCContext &Ctx = AsmParser->getContext();
2372 if (
Lit == LitModifier::None &&
2374 AsmParser->hasInv2PiInlineImm())) {
2382 bool HasMandatoryLiteral =
2385 if (
Literal.getLoBits(32) != 0 &&
2386 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2387 !HasMandatoryLiteral) {
2388 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2390 "Can't encode literal as exact 64-bit floating-point operand. "
2391 "Low 32-bits will be set to zero");
2392 Val &= 0xffffffff00000000u;
2398 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2404 Lit = LitModifier::Lit64;
2405 }
else if (
Lit == LitModifier::Lit) {
2419 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2421 Lit = LitModifier::Lit64;
2428 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2429 Literal == 0x3fc45f306725feed) {
2468 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2475 if (
Lit != LitModifier::None) {
2506 if (
Lit == LitModifier::None &&
2516 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2524 if (
Lit == LitModifier::None &&
2532 if (!AsmParser->has64BitLiterals()) {
2533 Val =
static_cast<uint64_t>(Val) << 32;
2540 if (
Lit == LitModifier::Lit ||
2542 Val =
static_cast<uint64_t>(Val) << 32;
2546 if (
Lit == LitModifier::Lit)
2572 if (
Lit != LitModifier::None) {
2580void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2586bool AMDGPUOperand::isInlineValue()
const {
2594void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2605 if (Is == IS_VGPR) {
2610 return AMDGPU::VGPR_32RegClassID;
2612 return AMDGPU::VReg_64RegClassID;
2614 return AMDGPU::VReg_96RegClassID;
2616 return AMDGPU::VReg_128RegClassID;
2618 return AMDGPU::VReg_160RegClassID;
2620 return AMDGPU::VReg_192RegClassID;
2622 return AMDGPU::VReg_224RegClassID;
2624 return AMDGPU::VReg_256RegClassID;
2626 return AMDGPU::VReg_288RegClassID;
2628 return AMDGPU::VReg_320RegClassID;
2630 return AMDGPU::VReg_352RegClassID;
2632 return AMDGPU::VReg_384RegClassID;
2634 return AMDGPU::VReg_512RegClassID;
2636 return AMDGPU::VReg_1024RegClassID;
2638 }
else if (Is == IS_TTMP) {
2643 return AMDGPU::TTMP_32RegClassID;
2645 return AMDGPU::TTMP_64RegClassID;
2647 return AMDGPU::TTMP_128RegClassID;
2649 return AMDGPU::TTMP_256RegClassID;
2651 return AMDGPU::TTMP_512RegClassID;
2653 }
else if (Is == IS_SGPR) {
2658 return AMDGPU::SGPR_32RegClassID;
2660 return AMDGPU::SGPR_64RegClassID;
2662 return AMDGPU::SGPR_96RegClassID;
2664 return AMDGPU::SGPR_128RegClassID;
2666 return AMDGPU::SGPR_160RegClassID;
2668 return AMDGPU::SGPR_192RegClassID;
2670 return AMDGPU::SGPR_224RegClassID;
2672 return AMDGPU::SGPR_256RegClassID;
2674 return AMDGPU::SGPR_288RegClassID;
2676 return AMDGPU::SGPR_320RegClassID;
2678 return AMDGPU::SGPR_352RegClassID;
2680 return AMDGPU::SGPR_384RegClassID;
2682 return AMDGPU::SGPR_512RegClassID;
2684 }
else if (Is == IS_AGPR) {
2689 return AMDGPU::AGPR_32RegClassID;
2691 return AMDGPU::AReg_64RegClassID;
2693 return AMDGPU::AReg_96RegClassID;
2695 return AMDGPU::AReg_128RegClassID;
2697 return AMDGPU::AReg_160RegClassID;
2699 return AMDGPU::AReg_192RegClassID;
2701 return AMDGPU::AReg_224RegClassID;
2703 return AMDGPU::AReg_256RegClassID;
2705 return AMDGPU::AReg_288RegClassID;
2707 return AMDGPU::AReg_320RegClassID;
2709 return AMDGPU::AReg_352RegClassID;
2711 return AMDGPU::AReg_384RegClassID;
2713 return AMDGPU::AReg_512RegClassID;
2715 return AMDGPU::AReg_1024RegClassID;
2723 .
Case(
"exec", AMDGPU::EXEC)
2724 .
Case(
"vcc", AMDGPU::VCC)
2725 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2726 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2727 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2728 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2729 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2730 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2731 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2732 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2733 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2734 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2735 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2736 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2737 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2738 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2739 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2740 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2741 .
Case(
"m0", AMDGPU::M0)
2742 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2743 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2744 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2745 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2746 .
Case(
"scc", AMDGPU::SRC_SCC)
2747 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2748 .
Case(
"tba", AMDGPU::TBA)
2749 .
Case(
"tma", AMDGPU::TMA)
2750 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2751 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2752 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2753 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2754 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2755 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2756 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2757 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2758 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2759 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2760 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2761 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2762 .
Case(
"pc", AMDGPU::PC_REG)
2763 .
Case(
"null", AMDGPU::SGPR_NULL)
2767bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2768 SMLoc &EndLoc,
bool RestoreOnFailure) {
2769 auto R = parseRegister();
2773 RegNo =
R->getReg();
2774 StartLoc =
R->getStartLoc();
2775 EndLoc =
R->getEndLoc();
2779bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2781 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2784ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2786 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2787 bool PendingErrors = getParser().hasPendingError();
2788 getParser().clearPendingErrors();
2796bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2797 RegisterKind RegKind,
2799 RegisterKind RegKind1, SMLoc Loc) {
2801 if (RegKind == IS_SGPR) {
2802 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2803 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2804 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2810 if (RegKind != RegKind1) {
2811 Error(Loc,
"registers in a list must be of the same kind");
2817 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2822 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2823 Reg = AMDGPU::FLAT_SCR;
2827 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2828 Reg = AMDGPU::XNACK_MASK;
2832 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2837 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2842 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2847 Error(Loc,
"register does not fit in the list");
2853 if (Reg1 !=
Reg + RegWidth / 32) {
2854 Error(Loc,
"registers in a list must have consecutive indices");
2870 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2871 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2875 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2881 if (Str.starts_with(
Reg.Name))
2887 return !Str.getAsInteger(10, Num);
2890bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2891 const AsmToken &NextToken)
const {
2906 StringRef RegSuffix = Str.substr(
RegName.size());
2907 if (!RegSuffix.
empty()) {
2924bool AMDGPUAsmParser::isRegister() {
2925 return isRegister(
getToken(), peekToken());
2928MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2929 unsigned SubReg,
unsigned RegWidth,
2933 unsigned AlignSize = 1;
2934 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2940 if (RegNum % AlignSize != 0) {
2941 Error(Loc,
"invalid register alignment");
2942 return MCRegister();
2945 unsigned RegIdx = RegNum / AlignSize;
2948 Error(Loc,
"invalid or unsupported register size");
2949 return MCRegister();
2953 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2954 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2955 Error(Loc,
"register index is out of range");
2956 return AMDGPU::NoRegister;
2959 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2960 Error(Loc,
"register index is out of range");
2961 return MCRegister();
2977bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2979 int64_t RegLo, RegHi;
2983 SMLoc FirstIdxLoc = getLoc();
2990 SecondIdxLoc = getLoc();
3001 Error(FirstIdxLoc,
"invalid register index");
3006 Error(SecondIdxLoc,
"invalid register index");
3010 if (RegLo > RegHi) {
3011 Error(FirstIdxLoc,
"first register index should not exceed second index");
3015 if (RegHi == RegLo) {
3016 StringRef RegSuffix = getTokenStr();
3017 if (RegSuffix ==
".l") {
3018 SubReg = AMDGPU::lo16;
3020 }
else if (RegSuffix ==
".h") {
3021 SubReg = AMDGPU::hi16;
3026 Num =
static_cast<unsigned>(RegLo);
3027 RegWidth = 32 * ((RegHi - RegLo) + 1);
3032MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3035 SmallVectorImpl<AsmToken> &Tokens) {
3041 RegKind = IS_SPECIAL;
3048MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3051 SmallVectorImpl<AsmToken> &Tokens) {
3053 StringRef
RegName = getTokenStr();
3054 auto Loc = getLoc();
3058 Error(Loc,
"invalid register name");
3059 return MCRegister();
3067 unsigned SubReg = NoSubRegister;
3068 bool IsRange =
false;
3069 if (!RegSuffix.
empty()) {
3071 SubReg = AMDGPU::lo16;
3073 SubReg = AMDGPU::hi16;
3077 Error(Loc,
"invalid register index");
3078 return MCRegister();
3084 if (!ParseRegRange(RegNum, RegWidth, SubReg))
3085 return MCRegister();
3089 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3090 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
3091 if (RegKind == IS_SGPR && IsRange
3092 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
3093 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
3094 Error(Loc,
"register index is out of range");
3095 return MCRegister();
3101MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3102 unsigned &RegNum,
unsigned &RegWidth,
3103 SmallVectorImpl<AsmToken> &Tokens) {
3105 auto ListLoc = getLoc();
3108 "expected a register or a list of registers")) {
3109 return MCRegister();
3114 auto Loc = getLoc();
3115 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
3116 return MCRegister();
3117 if (RegWidth != 32) {
3118 Error(Loc,
"expected a single 32-bit register");
3119 return MCRegister();
3123 RegisterKind NextRegKind;
3125 unsigned NextRegNum, NextRegWidth;
3128 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
3130 return MCRegister();
3132 if (NextRegWidth != 32) {
3133 Error(Loc,
"expected a single 32-bit register");
3134 return MCRegister();
3136 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
3138 return MCRegister();
3142 "expected a comma or a closing square bracket")) {
3143 return MCRegister();
3147 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
3152bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3153 MCRegister &
Reg,
unsigned &RegNum,
3155 SmallVectorImpl<AsmToken> &Tokens) {
3156 auto Loc = getLoc();
3160 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3162 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3164 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3169 assert(Parser.hasPendingError());
3173 if (!subtargetHasRegister(*
TRI,
Reg)) {
3174 if (
Reg == AMDGPU::SGPR_NULL) {
3175 Error(Loc,
"'null' operand is not supported on this GPU");
3178 " register not available on this GPU");
3186bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3187 MCRegister &
Reg,
unsigned &RegNum,
3189 bool RestoreOnFailure ) {
3193 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3194 if (RestoreOnFailure) {
3195 while (!Tokens.
empty()) {
3204std::optional<StringRef>
3205AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3208 return StringRef(
".amdgcn.next_free_vgpr");
3210 return StringRef(
".amdgcn.next_free_sgpr");
3212 return std::nullopt;
3216void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3217 auto SymbolName = getGprCountSymbolName(RegKind);
3218 assert(SymbolName &&
"initializing invalid register kind");
3224bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3225 unsigned DwordRegIndex,
3226 unsigned RegWidth) {
3231 auto SymbolName = getGprCountSymbolName(RegKind);
3236 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3240 return !
Error(getLoc(),
3241 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3245 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3247 if (OldCount <= NewMax)
3253std::unique_ptr<AMDGPUOperand>
3254AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3256 SMLoc StartLoc = Tok.getLoc();
3257 SMLoc EndLoc = Tok.getEndLoc();
3258 RegisterKind RegKind;
3260 unsigned RegNum, RegWidth;
3262 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3266 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3269 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3270 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3277 if (isRegister() || isModifier())
3280 if (
Lit == LitModifier::None) {
3281 if (trySkipId(
"lit"))
3282 Lit = LitModifier::Lit;
3283 else if (trySkipId(
"lit64"))
3284 Lit = LitModifier::Lit64;
3286 if (
Lit != LitModifier::None) {
3289 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3298 const auto &NextTok = peekToken();
3301 bool Negate =
false;
3309 AMDGPUOperand::Modifiers Mods;
3317 StringRef Num = getTokenStr();
3320 APFloat RealVal(APFloat::IEEEdouble());
3321 auto roundMode = APFloat::rmNearestTiesToEven;
3322 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3325 RealVal.changeSign();
3328 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3329 S, AMDGPUOperand::ImmTyNone,
true));
3330 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3331 Op.setModifiers(Mods);
3340 if (HasSP3AbsModifier) {
3349 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3352 if (Parser.parseExpression(Expr))
3356 if (Expr->evaluateAsAbsolute(IntVal)) {
3358 return Error(S,
"literal value out of range");
3359 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3360 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3361 Op.setModifiers(Mods);
3363 if (
Lit != LitModifier::None)
3365 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3378 if (
auto R = parseRegister()) {
3388 ParseStatus Res = parseReg(
Operands);
3396bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3397 const AsmToken &NextToken)
const {
3400 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3405bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3406 const AsmToken &NextToken)
const {
3410bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3411 const AsmToken &NextToken)
const {
3412 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3415bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3416 const AsmToken &NextToken)
const {
3417 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3433bool AMDGPUAsmParser::isModifier() {
3436 AsmToken NextToken[2];
3437 peekTokens(NextToken);
3439 return isOperandModifier(Tok, NextToken[0]) ||
3441 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3442 isOpcodeModifierWithVal(Tok, NextToken[0]);
3467bool AMDGPUAsmParser::parseSP3NegModifier() {
3469 AsmToken NextToken[2];
3470 peekTokens(NextToken);
3473 (isRegister(NextToken[0], NextToken[1]) ||
3491 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3493 SP3Neg = parseSP3NegModifier();
3496 Neg = trySkipId(
"neg");
3498 return Error(Loc,
"expected register or immediate");
3502 Abs = trySkipId(
"abs");
3507 if (trySkipId(
"lit")) {
3508 Lit = LitModifier::Lit;
3511 }
else if (trySkipId(
"lit64")) {
3512 Lit = LitModifier::Lit64;
3515 if (!has64BitLiterals())
3516 return Error(Loc,
"lit64 is not supported on this GPU");
3522 return Error(Loc,
"expected register or immediate");
3531 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3535 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3536 Error(Loc,
"expected immediate with lit modifier");
3538 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3544 if (
Lit != LitModifier::None &&
3548 AMDGPUOperand::Modifiers Mods;
3549 Mods.Abs = Abs || SP3Abs;
3550 Mods.Neg = Neg || SP3Neg;
3553 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3554 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3556 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3557 Op.setModifiers(Mods);
3565 bool Sext = trySkipId(
"sext");
3566 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3581 AMDGPUOperand::Modifiers Mods;
3584 if (Mods.hasIntModifiers()) {
3585 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3587 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3588 Op.setModifiers(Mods);
3595 return parseRegOrImmWithFPInputMods(
Operands,
false);
3599 return parseRegOrImmWithIntInputMods(
Operands,
false);
3603 auto Loc = getLoc();
3604 if (trySkipId(
"off")) {
3606 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3613 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3622unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3627 return Match_InvalidOperand;
3629 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3630 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3633 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3635 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3636 return Match_InvalidOperand;
3644 if (tryAnotherVOPDEncoding(Inst))
3645 return Match_InvalidOperand;
3647 return Match_Success;
3651 static const unsigned Variants[] = {
3660ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3661 if (isForcedDPP() && isForcedVOP3()) {
3665 if (getForcedEncodingSize() == 32) {
3670 if (isForcedVOP3()) {
3675 if (isForcedSDWA()) {
3681 if (isForcedDPP()) {
3689StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3690 if (isForcedDPP() && isForcedVOP3())
3693 if (getForcedEncodingSize() == 32)
3709AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3713 case AMDGPU::FLAT_SCR:
3715 case AMDGPU::VCC_LO:
3716 case AMDGPU::VCC_HI:
3723 return MCRegister();
3730bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3731 unsigned OpIdx)
const {
3739 const MCOperand &MO = Inst.
getOperand(OpIdx);
3788unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3794 case AMDGPU::V_LSHLREV_B64_e64:
3795 case AMDGPU::V_LSHLREV_B64_gfx10:
3796 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3797 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3798 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3799 case AMDGPU::V_LSHRREV_B64_e64:
3800 case AMDGPU::V_LSHRREV_B64_gfx10:
3801 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3802 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3803 case AMDGPU::V_ASHRREV_I64_e64:
3804 case AMDGPU::V_ASHRREV_I64_gfx10:
3805 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3806 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3807 case AMDGPU::V_LSHL_B64_e64:
3808 case AMDGPU::V_LSHR_B64_e64:
3809 case AMDGPU::V_ASHR_I64_e64:
3822 bool AddMandatoryLiterals =
false) {
3825 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3829 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3831 return {getNamedOperandIdx(Opcode, OpName::src0X),
3832 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3833 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3834 getNamedOperandIdx(Opcode, OpName::src0Y),
3835 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3836 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3841 return {getNamedOperandIdx(Opcode, OpName::src0),
3842 getNamedOperandIdx(Opcode, OpName::src1),
3843 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3846bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3847 const MCOperand &MO = Inst.
getOperand(OpIdx);
3849 return !isInlineConstant(Inst, OpIdx);
3856 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3867 const unsigned Opcode = Inst.
getOpcode();
3868 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3871 if (!LaneSelOp.
isReg())
3874 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3877bool AMDGPUAsmParser::validateConstantBusLimitations(
3879 const unsigned Opcode = Inst.
getOpcode();
3880 const MCInstrDesc &
Desc = MII.
get(Opcode);
3881 MCRegister LastSGPR;
3882 unsigned ConstantBusUseCount = 0;
3883 unsigned NumLiterals = 0;
3884 unsigned LiteralSize;
3900 SmallDenseSet<MCRegister> SGPRsUsed;
3901 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3903 SGPRsUsed.
insert(SGPRUsed);
3904 ++ConstantBusUseCount;
3909 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3911 for (
int OpIdx : OpIndices) {
3915 const MCOperand &MO = Inst.
getOperand(OpIdx);
3916 if (usesConstantBus(Inst, OpIdx)) {
3925 if (SGPRsUsed.
insert(LastSGPR).second) {
3926 ++ConstantBusUseCount;
3946 if (NumLiterals == 0) {
3949 }
else if (LiteralSize !=
Size) {
3955 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3957 "invalid operand (violates constant bus restrictions)");
3964std::optional<unsigned>
3965AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3967 const unsigned Opcode = Inst.
getOpcode();
3973 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3974 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3983 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3984 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3985 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3986 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3987 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3988 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3992 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3993 int I = getNamedOperandIdx(Opcode, OpName);
3997 int64_t
Imm =
Op.getImm();
4003 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4004 OpName::vsrc2Y, OpName::imm}) {
4005 int I = getNamedOperandIdx(Opcode, OpName);
4015 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4016 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
4018 return InvalidCompOprIdx;
4021bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
4028 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
4029 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
4030 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
4032 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
4036 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4037 if (!InvalidCompOprIdx.has_value())
4040 auto CompOprIdx = *InvalidCompOprIdx;
4043 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
4044 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4047 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
4048 if (CompOprIdx == VOPD::Component::DST) {
4050 Error(Loc,
"dst registers must be distinct");
4052 Error(Loc,
"one dst register must be even and the other odd");
4054 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4055 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
4056 " operands must use different VGPR banks");
4064bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
4066 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
4067 if (!InvalidCompOprIdx.has_value())
4071 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
4072 if (InvalidCompOprIdx.has_value()) {
4077 if (*InvalidCompOprIdx == VOPD::Component::DST)
4090bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
4091 const unsigned Opcode = Inst.
getOpcode();
4106 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4107 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4108 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4109 int I = getNamedOperandIdx(Opcode, OpName);
4116 return !tryVOPD3(Inst);
4121bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
4126 return tryVOPD(Inst);
4127 return tryVOPD3(Inst);
4130bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
4135 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
4143bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4151 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4152 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4153 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4158 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4159 hasBVHRayTracingInsts())
4162 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4163 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4168 bool IsPackedD16 =
false;
4171 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4172 IsPackedD16 = D16Idx >= 0;
4174 DataSize = (DataSize + 1) / 2;
4177 if ((VDataSize / 4) == DataSize + TFESize)
4182 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4184 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4186 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4190bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4199 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4201 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4202 AMDGPU::OpName RSrcOpName =
4204 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4205 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4206 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4210 assert(SrsrcIdx > VAddr0Idx);
4213 if (BaseOpcode->
BVH) {
4214 if (IsA16 == BaseOpcode->
A16)
4216 Error(IDLoc,
"image address size does not match a16");
4222 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4223 unsigned ActualAddrSize =
4224 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4226 unsigned ExpectedAddrSize =
4230 if (hasPartialNSAEncoding() &&
4232 int VAddrLastIdx = SrsrcIdx - 1;
4233 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4235 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4238 if (ExpectedAddrSize > 12)
4239 ExpectedAddrSize = 16;
4244 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4248 if (ActualAddrSize == ExpectedAddrSize)
4251 Error(IDLoc,
"image address size does not match dim and a16");
4255bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4262 if (!
Desc.mayLoad() || !
Desc.mayStore())
4265 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4272 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4275bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4282 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4290 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4293bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4307 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4308 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4315bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4322 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4325 if (!BaseOpcode->
MSAA)
4328 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4334 return DimInfo->
MSAA;
4339 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4340 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4341 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4351bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4359 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4362 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4370 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4374bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4379 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4382 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4385 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4393 "source operand must be either a VGPR or an inline constant");
4400bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4405 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4408 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4412 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4414 "inline constants are not allowed for this operand");
4421bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4429 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4430 if (BlgpIdx != -1) {
4431 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4432 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4442 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4444 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4449 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4451 "wrong register tuple size for blgp value " + Twine(BLGP));
4459 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4463 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4467 MCRegister Src2Reg = Src2.
getReg();
4469 if (Src2Reg == DstReg)
4474 .getSizeInBits() <= 128)
4477 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4479 "source 2 operand must not partially overlap with dst");
4486bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4490 case V_DIV_SCALE_F32_gfx6_gfx7:
4491 case V_DIV_SCALE_F32_vi:
4492 case V_DIV_SCALE_F32_gfx10:
4493 case V_DIV_SCALE_F64_gfx6_gfx7:
4494 case V_DIV_SCALE_F64_vi:
4495 case V_DIV_SCALE_F64_gfx10:
4502 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4503 AMDGPU::OpName::src2_modifiers}) {
4514bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4521 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4530bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4536 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4543 case AMDGPU::V_SUBREV_F32_e32:
4544 case AMDGPU::V_SUBREV_F32_e64:
4545 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4546 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4547 case AMDGPU::V_SUBREV_F32_e32_vi:
4548 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4549 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4550 case AMDGPU::V_SUBREV_F32_e64_vi:
4552 case AMDGPU::V_SUBREV_CO_U32_e32:
4553 case AMDGPU::V_SUBREV_CO_U32_e64:
4554 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4555 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4557 case AMDGPU::V_SUBBREV_U32_e32:
4558 case AMDGPU::V_SUBBREV_U32_e64:
4559 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4560 case AMDGPU::V_SUBBREV_U32_e32_vi:
4561 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4562 case AMDGPU::V_SUBBREV_U32_e64_vi:
4564 case AMDGPU::V_SUBREV_U32_e32:
4565 case AMDGPU::V_SUBREV_U32_e64:
4566 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4567 case AMDGPU::V_SUBREV_U32_e32_vi:
4568 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4569 case AMDGPU::V_SUBREV_U32_e64_vi:
4571 case AMDGPU::V_SUBREV_F16_e32:
4572 case AMDGPU::V_SUBREV_F16_e64:
4573 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4574 case AMDGPU::V_SUBREV_F16_e32_vi:
4575 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4576 case AMDGPU::V_SUBREV_F16_e64_vi:
4578 case AMDGPU::V_SUBREV_U16_e32:
4579 case AMDGPU::V_SUBREV_U16_e64:
4580 case AMDGPU::V_SUBREV_U16_e32_vi:
4581 case AMDGPU::V_SUBREV_U16_e64_vi:
4583 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4584 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4585 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4587 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4588 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4590 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4591 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4593 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4594 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4596 case AMDGPU::V_LSHRREV_B32_e32:
4597 case AMDGPU::V_LSHRREV_B32_e64:
4598 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4599 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4600 case AMDGPU::V_LSHRREV_B32_e32_vi:
4601 case AMDGPU::V_LSHRREV_B32_e64_vi:
4602 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4603 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4605 case AMDGPU::V_ASHRREV_I32_e32:
4606 case AMDGPU::V_ASHRREV_I32_e64:
4607 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4608 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4609 case AMDGPU::V_ASHRREV_I32_e32_vi:
4610 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4611 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4612 case AMDGPU::V_ASHRREV_I32_e64_vi:
4614 case AMDGPU::V_LSHLREV_B32_e32:
4615 case AMDGPU::V_LSHLREV_B32_e64:
4616 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4617 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4618 case AMDGPU::V_LSHLREV_B32_e32_vi:
4619 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4620 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4621 case AMDGPU::V_LSHLREV_B32_e64_vi:
4623 case AMDGPU::V_LSHLREV_B16_e32:
4624 case AMDGPU::V_LSHLREV_B16_e64:
4625 case AMDGPU::V_LSHLREV_B16_e32_vi:
4626 case AMDGPU::V_LSHLREV_B16_e64_vi:
4627 case AMDGPU::V_LSHLREV_B16_gfx10:
4629 case AMDGPU::V_LSHRREV_B16_e32:
4630 case AMDGPU::V_LSHRREV_B16_e64:
4631 case AMDGPU::V_LSHRREV_B16_e32_vi:
4632 case AMDGPU::V_LSHRREV_B16_e64_vi:
4633 case AMDGPU::V_LSHRREV_B16_gfx10:
4635 case AMDGPU::V_ASHRREV_I16_e32:
4636 case AMDGPU::V_ASHRREV_I16_e64:
4637 case AMDGPU::V_ASHRREV_I16_e32_vi:
4638 case AMDGPU::V_ASHRREV_I16_e64_vi:
4639 case AMDGPU::V_ASHRREV_I16_gfx10:
4641 case AMDGPU::V_LSHLREV_B64_e64:
4642 case AMDGPU::V_LSHLREV_B64_gfx10:
4643 case AMDGPU::V_LSHLREV_B64_vi:
4645 case AMDGPU::V_LSHRREV_B64_e64:
4646 case AMDGPU::V_LSHRREV_B64_gfx10:
4647 case AMDGPU::V_LSHRREV_B64_vi:
4649 case AMDGPU::V_ASHRREV_I64_e64:
4650 case AMDGPU::V_ASHRREV_I64_gfx10:
4651 case AMDGPU::V_ASHRREV_I64_vi:
4653 case AMDGPU::V_PK_LSHLREV_B16:
4654 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4655 case AMDGPU::V_PK_LSHLREV_B16_vi:
4657 case AMDGPU::V_PK_LSHRREV_B16:
4658 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4659 case AMDGPU::V_PK_LSHRREV_B16_vi:
4660 case AMDGPU::V_PK_ASHRREV_I16:
4661 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4662 case AMDGPU::V_PK_ASHRREV_I16_vi:
4669bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4671 const unsigned Opcode = Inst.
getOpcode();
4680 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4681 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4685 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4689 "lds_direct is not supported on this GPU");
4695 "lds_direct cannot be used with this instruction");
4699 if (SrcName != OpName::src0) {
4701 "lds_direct may be used as src0 only");
4711 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4712 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4713 if (
Op.isFlatOffset())
4714 return Op.getStartLoc();
4719bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4722 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4727 return validateFlatOffset(Inst,
Operands);
4730 return validateSMEMOffset(Inst,
Operands);
4735 const unsigned OffsetSize = 24;
4736 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4738 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4739 "-bit unsigned offset for buffer ops");
4743 const unsigned OffsetSize = 16;
4744 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4746 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4753bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4759 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4763 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4765 "flat offset modifier is not supported on this GPU");
4772 bool AllowNegative =
4774 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4776 Twine(
"expected a ") +
4777 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4778 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4787 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4788 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4789 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4790 return Op.getStartLoc();
4795bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4804 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4820 ?
"expected a 23-bit unsigned offset for buffer ops"
4821 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4822 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4823 :
"expected a 21-bit signed offset");
4828bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4831 const MCInstrDesc &
Desc = MII.
get(Opcode);
4835 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4836 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4838 const int OpIndices[] = {Src0Idx, Src1Idx};
4840 unsigned NumExprs = 0;
4841 unsigned NumLiterals = 0;
4844 for (
int OpIdx : OpIndices) {
4848 const MCOperand &MO = Inst.
getOperand(OpIdx);
4852 std::optional<int64_t>
Imm;
4855 }
else if (MO.
isExpr()) {
4864 if (!
Imm.has_value()) {
4866 }
else if (!isInlineConstant(Inst, OpIdx)) {
4870 if (NumLiterals == 0 || LiteralValue !=
Value) {
4878 if (NumLiterals + NumExprs <= 1)
4882 "only one unique literal operand is allowed");
4886bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4889 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4897 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4898 if (OpSelIdx != -1) {
4902 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4903 if (OpSelHiIdx != -1) {
4912 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4922 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4923 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4924 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4925 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4927 const MCOperand &Src0 = Inst.
getOperand(Src0Idx);
4928 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
4934 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4936 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4946 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4947 if (Src2Idx != -1) {
4948 const MCOperand &Src2 = Inst.
getOperand(Src2Idx);
4958bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4959 if (!hasTrue16Insts())
4961 const MCRegisterInfo *MRI = getMRI();
4963 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4969 if (OpSelOpValue == 0)
4971 unsigned OpCount = 0;
4972 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4973 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4974 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4981 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4982 if (OpSelOpIsHi != VGPRSuffixIsHi)
4991bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4992 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5004 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
5015 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5016 AMDGPU::OpName::src1_modifiers,
5017 AMDGPU::OpName::src2_modifiers};
5019 for (
unsigned i = 0; i < 3; ++i) {
5029bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
5032 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
5033 if (DppCtrlIdx >= 0) {
5040 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
5041 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
5042 :
"DP ALU dpp only supports row_newbcast");
5047 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
5048 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5051 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
5053 const MCOperand &Src1 = Inst.
getOperand(Src1Idx);
5057 "invalid operand for instruction");
5062 "src1 immediate operand invalid for instruction");
5072bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
5073 return (
Reg == AMDGPU::VCC && isWave64()) ||
5074 (
Reg == AMDGPU::VCC_LO && isWave32());
5078bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
5081 const MCInstrDesc &
Desc = MII.
get(Opcode);
5082 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5089 std::optional<unsigned> LiteralOpIdx;
5092 for (
int OpIdx : OpIndices) {
5096 const MCOperand &MO = Inst.
getOperand(OpIdx);
5102 std::optional<int64_t>
Imm;
5108 bool IsAnotherLiteral =
false;
5109 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5110 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5111 if (!
Imm.has_value()) {
5113 IsAnotherLiteral =
true;
5114 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5119 HasMandatoryLiteral);
5120 unsigned OpTy =
Desc.operands()[OpIdx].OperandType;
5130 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5131 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5133 "invalid operand for instruction");
5138 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5139 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5141 "invalid operand for instruction");
5145 if (IsFP64 && IsValid32Op && !IsForcedFP64)
5152 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5153 !getFeatureBits()[FeatureVOP3Literal]) {
5155 "literal operands are not supported");
5159 if (LiteralOpIdx && IsAnotherLiteral) {
5161 getOperandLoc(
Operands, *LiteralOpIdx)),
5162 "only one unique literal operand is allowed");
5166 if (IsAnotherLiteral)
5167 LiteralOpIdx = OpIdx;
5176 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5190bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5196 ? AMDGPU::OpName::data0
5197 : AMDGPU::OpName::vdata;
5199 const MCRegisterInfo *MRI = getMRI();
5200 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5204 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5205 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5209 auto FB = getFeatureBits();
5210 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5211 if (DataAreg < 0 || DstAreg < 0)
5213 return DstAreg == DataAreg;
5216 return DstAreg < 1 && DataAreg < 1;
5219bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5220 auto FB = getFeatureBits();
5221 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5225 const MCRegisterInfo *MRI = getMRI();
5228 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5231 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5235 case AMDGPU::DS_LOAD_TR6_B96:
5236 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5240 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5241 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5245 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5246 if (VAddrIdx != -1) {
5249 if ((
Sub - AMDGPU::VGPR0) & 1)
5254 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5255 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5260 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5261 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5281 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5282 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5284 return Op.getStartLoc();
5289bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5292 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5295 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5298 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5299 auto FB = getFeatureBits();
5300 bool UsesNeg =
false;
5301 if (FB[AMDGPU::FeatureGFX940Insts]) {
5303 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5304 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5305 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5306 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5311 if (IsNeg == UsesNeg)
5314 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5315 :
"invalid modifier: neg is not supported");
5320bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5326 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5327 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5328 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5329 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5332 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5335 if (
Reg == AMDGPU::SGPR_NULL)
5338 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5342bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5347 return validateGWS(Inst,
Operands);
5352 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5357 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5358 Error(S,
"gds modifier is not supported on this GPU");
5366bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5368 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5372 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5373 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5376 const MCRegisterInfo *MRI = getMRI();
5377 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5379 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5382 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5384 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5391bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5395 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5403 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5406 Error(S,
"scale_offset is not supported on this GPU");
5409 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5412 Error(S,
"nv is not supported on this GPU");
5417 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5420 Error(S,
"scale_offset is not supported for this instruction");
5424 return validateTHAndScopeBits(Inst,
Operands, CPol);
5428 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5429 Error(S,
"cache policy is not supported for SMRD instructions");
5433 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5440 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5444 "scc modifier is not supported for this instruction on this GPU");
5455 :
"instruction must use glc");
5460 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5463 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5465 :
"instruction must not use glc");
5473bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5475 const unsigned CPol) {
5480 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5487 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5492 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5500 return PrintError(
"invalid th value for SMEM instruction");
5507 return PrintError(
"scope and th combination is not valid");
5513 return PrintError(
"invalid th value for atomic instructions");
5516 return PrintError(
"invalid th value for store instructions");
5519 return PrintError(
"invalid th value for load instructions");
5525bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5529 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5531 Error(Loc,
"TFE modifier has no meaning for store instructions");
5539bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5545 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5549 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5552 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5553 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5562 "wrong register tuple size for " +
5567 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5568 !validateFmt(BFmt, AMDGPU::OpName::src1))
5572 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5573 if (AScaleIdx == -1)
5577 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5581 "invalid matrix and scale format combination");
5588bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5590 if (!validateLdsDirect(Inst,
Operands))
5592 if (!validateTrue16OpSel(Inst)) {
5594 "op_sel operand conflicts with 16-bit operand suffix");
5597 if (!validateSOPLiteral(Inst,
Operands))
5599 if (!validateVOPLiteral(Inst,
Operands)) {
5602 if (!validateConstantBusLimitations(Inst,
Operands)) {
5605 if (!validateVOPD(Inst,
Operands)) {
5608 if (!validateIntClampSupported(Inst)) {
5610 "integer clamping is not supported on this GPU");
5613 if (!validateOpSel(Inst)) {
5615 "invalid op_sel operand");
5618 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5620 "invalid neg_lo operand");
5623 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5625 "invalid neg_hi operand");
5628 if (!validateDPP(Inst,
Operands)) {
5632 if (!validateMIMGD16(Inst)) {
5634 "d16 modifier is not supported on this GPU");
5637 if (!validateMIMGDim(Inst,
Operands)) {
5638 Error(IDLoc,
"missing dim operand");
5641 if (!validateTensorR128(Inst)) {
5643 "instruction must set modifier r128=0");
5646 if (!validateMIMGMSAA(Inst)) {
5648 "invalid dim; must be MSAA type");
5651 if (!validateMIMGDataSize(Inst, IDLoc)) {
5654 if (!validateMIMGAddrSize(Inst, IDLoc))
5656 if (!validateMIMGAtomicDMask(Inst)) {
5658 "invalid atomic image dmask");
5661 if (!validateMIMGGatherDMask(Inst)) {
5663 "invalid image_gather dmask: only one bit must be set");
5666 if (!validateMovrels(Inst,
Operands)) {
5669 if (!validateOffset(Inst,
Operands)) {
5672 if (!validateMAIAccWrite(Inst,
Operands)) {
5675 if (!validateMAISrc2(Inst,
Operands)) {
5678 if (!validateMFMA(Inst,
Operands)) {
5681 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5685 if (!validateAGPRLdSt(Inst)) {
5688 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5689 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5690 :
"invalid register class: agpr loads and stores not supported on "
5694 if (!validateVGPRAlign(Inst)) {
5695 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5702 if (!validateBLGP(Inst,
Operands)) {
5706 if (!validateDivScale(Inst)) {
5707 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5710 if (!validateWaitCnt(Inst,
Operands)) {
5713 if (!validateTFE(Inst,
Operands)) {
5716 if (!validateWMMA(Inst,
Operands)) {
5725 unsigned VariantID = 0);
5729 unsigned VariantID);
5731bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5736bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5737 const FeatureBitset &FBS,
5738 ArrayRef<unsigned> Variants) {
5739 for (
auto Variant : Variants) {
5747bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5749 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5752 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5757 getParser().clearPendingErrors();
5761 StringRef VariantName = getMatchedVariantName();
5762 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5763 return Error(IDLoc, Twine(VariantName,
5764 " variant of this instruction is not supported"));
5768 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5769 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5771 FeatureBitset FeaturesWS32 = getFeatureBits();
5772 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5773 .
flip(AMDGPU::FeatureWavefrontSize32);
5774 FeatureBitset AvailableFeaturesWS32 =
5775 ComputeAvailableFeatures(FeaturesWS32);
5777 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5778 return Error(IDLoc,
"instruction requires wavesize=32");
5782 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5783 return Error(IDLoc,
"instruction not supported on this GPU (" +
5784 getSTI().
getCPU() +
")" +
": " + Mnemo);
5789 return Error(IDLoc,
"invalid instruction" + Suggestion);
5795 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5796 if (
Op.isToken() && InvalidOprIdx > 1) {
5797 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5798 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5803bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5807 bool MatchingInlineAsm) {
5810 unsigned Result = Match_Success;
5811 for (
auto Variant : getMatchedVariants()) {
5814 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5818 if (R == Match_Success || R == Match_MissingFeature ||
5819 (R == Match_InvalidOperand && Result != Match_MissingFeature) ||
5820 (R == Match_MnemonicFail && Result != Match_InvalidOperand &&
5821 Result != Match_MissingFeature)) {
5825 if (R == Match_Success)
5829 if (Result == Match_Success) {
5830 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5833 emitTargetDirective();
5841 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5848 case Match_MissingFeature:
5852 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5854 case Match_InvalidOperand: {
5855 SMLoc ErrorLoc = IDLoc;
5856 if (ErrorInfo != ~0ULL) {
5857 if (ErrorInfo >=
Operands.size()) {
5858 return Error(IDLoc,
"too few operands for instruction");
5860 ErrorLoc = ((AMDGPUOperand &)*
Operands[ErrorInfo]).getStartLoc();
5861 if (ErrorLoc == SMLoc())
5865 return Error(ErrorLoc,
"invalid VOPDY instruction");
5867 return Error(ErrorLoc,
"invalid operand for instruction");
5870 case Match_MnemonicFail:
5876bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5881 if (getParser().parseAbsoluteExpression(Tmp)) {
5884 Ret =
static_cast<uint32_t
>(Tmp);
5888bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5889 if (!getSTI().getTargetTriple().isAMDGCN())
5890 return TokError(
"directive only supported for amdgcn architecture");
5892 std::string TargetIDDirective;
5893 SMLoc TargetStart = getTok().getLoc();
5894 if (getParser().parseEscapedString(TargetIDDirective))
5897 std::optional<AMDGPU::TargetID> MaybeParsed =
5900 return getParser().Error(TargetStart,
5901 "malformed target id '" + TargetIDDirective +
"'");
5904 const Triple &
TT = getSTI().getTargetTriple();
5910 return getParser().Error(
5911 TargetStart,
"target id '" + TargetIDDirective +
5912 "' specifies a processor that is not valid for "
5914 TT.getArchName() +
"'");
5917 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5918 getTargetStreamer().getTargetID();
5921 const Triple &STITriple = getSTI().getTargetTriple();
5922 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5923 return getParser().Error(
5924 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5925 " is incompatible with " +
5926 Twine(CurrentTargetID->toString()));
5930 StringRef DirectiveProcessor =
5933 if (DirectiveISA != ISA) {
5934 return getParser().Error(TargetStart,
5935 ".amdgcn_target directive processor " +
5936 Twine(DirectiveProcessor) +
5937 " does not match the specified processor " +
5938 Twine(getSTI().
getCPU()));
5944 CurrentTargetID->getXnackSetting())) {
5946 ".amdgcn_target directive has conflicting xnack settings");
5950 CurrentTargetID->getSramEccSetting())) {
5952 ".amdgcn_target directive has conflicting sramecc settings");
5958 getTargetStreamer().getTargetID()->setXnackSetting(
5960 getTargetStreamer().getTargetID()->setSramEccSetting(
5966bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5970bool AMDGPUAsmParser::calculateGPRBlocks(
5971 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5972 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5973 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5974 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5975 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
5980 const MCExpr *
NumSGPRs = NextFreeSGPR;
5981 int64_t EvaluatedSGPRs;
5983 if (
ISA.Major >= 10)
5988 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
5989 !Features.
test(FeatureSGPRInitBug) &&
5990 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
5991 return OutOfRangeError(SGPRRange);
5993 const MCExpr *ExtraSGPRs =
5997 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
5998 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
5999 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6000 return OutOfRangeError(SGPRRange);
6002 if (Features.
test(FeatureSGPRInitBug))
6009 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6010 unsigned Granule) ->
const MCExpr * {
6014 const MCExpr *AlignToGPR =
6016 const MCExpr *DivGPR =
6022 VGPRBlocks = GetNumGPRBlocks(
6031bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6032 if (!getSTI().getTargetTriple().isAMDGCN())
6033 return TokError(
"directive only supported for amdgcn architecture");
6036 return TokError(
"directive only supported for amdhsa OS");
6038 StringRef KernelName;
6039 if (getParser().parseIdentifier(KernelName))
6046 AMDGPU::MCKernelDescriptor KD =
6056 const MCExpr *NextFreeVGPR = ZeroExpr;
6058 const MCExpr *NamedBarCnt = ZeroExpr;
6063 const MCExpr *NextFreeSGPR = ZeroExpr;
6066 unsigned ImpliedUserSGPRCount = 0;
6070 std::optional<unsigned> ExplicitUserSGPRCount;
6071 const MCExpr *ReserveVCC = OneExpr;
6072 const MCExpr *ReserveFlatScr = OneExpr;
6073 std::optional<bool> EnableWavefrontSize32;
6080 SMRange IDRange = getTok().getLocRange();
6081 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6084 if (ID ==
".end_amdhsa_kernel")
6087 if (!Seen.
insert(ID).second)
6088 return TokError(
".amdhsa_ directives cannot be repeated");
6090 SMLoc ValStart = getLoc();
6091 const MCExpr *ExprVal;
6092 if (getParser().parseExpression(ExprVal))
6094 SMLoc ValEnd = getLoc();
6095 SMRange ValRange = SMRange(ValStart, ValEnd);
6099 bool EvaluatableExpr;
6100 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6102 return OutOfRangeError(ValRange);
6106#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6107 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6108 return OutOfRangeError(RANGE); \
6109 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6114#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6116 return Error(IDRange.Start, "directive should have resolvable expression", \
6119 if (ID ==
".amdhsa_group_segment_fixed_size") {
6122 return OutOfRangeError(ValRange);
6124 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6127 return OutOfRangeError(ValRange);
6129 }
else if (ID ==
".amdhsa_kernarg_size") {
6131 return OutOfRangeError(ValRange);
6133 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6135 ExplicitUserSGPRCount = Val;
6136 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6140 "directive is not supported with architected flat scratch",
6143 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6146 ImpliedUserSGPRCount += 4;
6147 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6150 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6153 return OutOfRangeError(ValRange);
6157 ImpliedUserSGPRCount += Val;
6158 PreloadLength = Val;
6160 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6163 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6166 return OutOfRangeError(ValRange);
6170 PreloadOffset = Val;
6171 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6174 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6177 ImpliedUserSGPRCount += 2;
6178 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6181 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6184 ImpliedUserSGPRCount += 2;
6185 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6188 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6191 ImpliedUserSGPRCount += 2;
6192 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6195 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6198 ImpliedUserSGPRCount += 2;
6199 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6202 "directive is not supported with architected flat scratch",
6206 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6209 ImpliedUserSGPRCount += 2;
6210 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6213 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6216 ImpliedUserSGPRCount += 1;
6217 }
else if (ID ==
".amdhsa_wavefront_size32") {
6220 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6221 EnableWavefrontSize32 = Val;
6223 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6225 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6227 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6229 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6232 "directive is not supported with architected flat scratch",
6235 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6237 }
else if (ID ==
".amdhsa_enable_private_segment") {
6241 "directive is not supported without architected flat scratch",
6244 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6246 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6248 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6250 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6252 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6254 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6256 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6258 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6260 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6262 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6264 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6266 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6267 VGPRRange = ValRange;
6268 NextFreeVGPR = ExprVal;
6269 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6270 SGPRRange = ValRange;
6271 NextFreeSGPR = ExprVal;
6272 }
else if (ID ==
".amdhsa_accum_offset") {
6274 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6275 AccumOffset = ExprVal;
6276 }
else if (ID ==
".amdhsa_named_barrier_count") {
6278 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6279 NamedBarCnt = ExprVal;
6280 }
else if (ID ==
".amdhsa_reserve_vcc") {
6282 return OutOfRangeError(ValRange);
6283 ReserveVCC = ExprVal;
6284 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6286 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6289 "directive is not supported with architected flat scratch",
6292 return OutOfRangeError(ValRange);
6293 ReserveFlatScr = ExprVal;
6294 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6296 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6298 return OutOfRangeError(ValRange);
6299 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny() ||
6300 getSTI().hasFeature(AMDGPU::FeatureXNACK);
6301 if (Val != XnackOn) {
6302 return getParser().Error(
6304 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6306 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6308 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6310 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6312 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6314 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6316 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6318 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6320 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6322 }
else if (ID ==
".amdhsa_dx10_clamp") {
6323 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6324 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6327 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6329 }
else if (ID ==
".amdhsa_ieee_mode") {
6330 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6331 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6334 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6336 }
else if (ID ==
".amdhsa_fp16_overflow") {
6338 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6340 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6342 }
else if (ID ==
".amdhsa_tg_split") {
6344 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6347 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6350 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6352 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6354 }
else if (ID ==
".amdhsa_memory_ordered") {
6356 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6358 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6360 }
else if (ID ==
".amdhsa_forward_progress") {
6362 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6364 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6366 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6368 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6369 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6371 SharedVGPRCount = Val;
6373 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6375 }
else if (ID ==
".amdhsa_inst_pref_size") {
6377 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6378 if (
ISA.Major == 11) {
6380 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6384 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6387 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6390 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6392 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6394 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6396 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6399 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6401 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6403 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6405 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6407 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6409 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6411 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6413 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6415 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6417 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6419 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6421 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6424 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6427#undef PARSE_BITS_ENTRY
6430 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6431 return TokError(
".amdhsa_next_free_vgpr directive is required");
6433 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6434 return TokError(
".amdhsa_next_free_sgpr directive is required");
6436 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6438 return TokError(
"too many user SGPRs enabled, found " +
6439 Twine(UserSGPRCount) +
", but only " +
6445 if (PreloadLength) {
6451 const MCExpr *VGPRBlocks;
6452 const MCExpr *SGPRBlocks;
6453 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6454 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6455 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6456 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6459 int64_t EvaluatedVGPRBlocks;
6460 bool VGPRBlocksEvaluatable =
6461 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6462 if (VGPRBlocksEvaluatable &&
6464 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6465 return OutOfRangeError(VGPRRange);
6469 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6470 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6472 int64_t EvaluatedSGPRBlocks;
6473 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6475 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6476 return OutOfRangeError(SGPRRange);
6479 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6480 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6482 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6483 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6484 "enabled user SGPRs");
6490 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6491 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6496 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6497 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6502 return TokError(
"Kernarg size should be resolvable");
6504 if (PreloadLength && kernarg_size &&
6505 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6506 return TokError(
"Kernarg preload length + offset is larger than the "
6507 "kernarg segment size");
6510 if (!Seen.
contains(
".amdhsa_accum_offset"))
6511 return TokError(
".amdhsa_accum_offset directive is required");
6512 int64_t EvaluatedAccum;
6513 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6514 uint64_t UEvaluatedAccum = EvaluatedAccum;
6515 if (AccumEvaluatable &&
6516 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6517 return TokError(
"accum_offset should be in range [4..256] in "
6520 int64_t EvaluatedNumVGPR;
6521 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6525 return TokError(
"accum_offset exceeds total VGPR allocation");
6531 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6532 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6538 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6539 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6542 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6544 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6545 return TokError(
"shared_vgpr_count directive not valid on "
6546 "wavefront size 32");
6549 if (VGPRBlocksEvaluatable &&
6550 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6552 return TokError(
"shared_vgpr_count*2 + "
6553 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6558 emitTargetDirective();
6559 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6560 NextFreeVGPR, NextFreeSGPR,
6561 ReserveVCC, ReserveFlatScr);
6565bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6567 if (ParseAsAbsoluteExpression(
Version))
6570 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6571 emitTargetDirective();
6575bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6576 AMDGPUMCKernelCodeT &
C) {
6579 if (ID ==
"max_scratch_backing_memory_byte_size") {
6580 Parser.eatToEndOfStatement();
6584 SmallString<40> ErrStr;
6585 raw_svector_ostream Err(ErrStr);
6586 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6587 return TokError(Err.
str());
6591 if (ID ==
"enable_wavefront_size32") {
6594 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6596 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6599 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6603 if (ID ==
"wavefront_size") {
6604 if (
C.wavefront_size == 5) {
6606 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6608 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6609 }
else if (
C.wavefront_size == 6) {
6611 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6618bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6619 AMDGPUMCKernelCodeT KernelCode;
6629 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6632 if (ID ==
".end_amd_kernel_code_t")
6635 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6640 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6645bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6646 StringRef KernelName;
6647 if (!parseId(KernelName,
"expected symbol name"))
6650 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6657bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6658 if (!getSTI().getTargetTriple().isAMDGCN()) {
6659 return Error(getLoc(),
6660 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6664 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6666 std::optional<AMDGPU::TargetID> MaybeParsed =
6669 return Error(getParser().getTok().getLoc(),
6670 "malformed target id '" + TargetIDDirective +
"'");
6673 const Triple &
TT = getSTI().getTargetTriple();
6679 return Error(getParser().getTok().getLoc(),
6680 "target id '" + TargetIDDirective +
6681 "' specifies a processor that is not valid for subarch '" +
6682 TT.getArchName() +
"'");
6685 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6686 getTargetStreamer().getTargetID();
6689 const Triple &STITriple = getSTI().getTargetTriple();
6690 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6691 return Error(getParser().getTok().getLoc(),
6692 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6693 " is incompatible with " +
6694 Twine(CurrentTargetID->toString()));
6698 StringRef DirectiveProcessor =
6701 if (DirectiveISA != ISA) {
6702 return Error(getParser().getTok().getLoc(),
6703 ".amd_amdgpu_isa directive processor " +
6704 Twine(DirectiveProcessor) +
6705 " does not match the specified processor " +
6706 Twine(getSTI().
getCPU()));
6709 getTargetStreamer().EmitISAVersion();
6715bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6718 std::string HSAMetadataString;
6723 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6724 return Error(getLoc(),
"invalid HSA metadata");
6731bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6732 const char *AssemblerDirectiveEnd,
6733 std::string &CollectString) {
6735 raw_string_ostream CollectStream(CollectString);
6737 getLexer().setSkipSpace(
false);
6739 bool FoundEnd =
false;
6742 CollectStream << getTokenStr();
6746 if (trySkipId(AssemblerDirectiveEnd)) {
6751 CollectStream << Parser.parseStringToEndOfStatement()
6752 <<
getContext().getAsmInfo().getSeparatorString();
6754 Parser.eatToEndOfStatement();
6757 getLexer().setSkipSpace(
true);
6760 return TokError(Twine(
"expected directive ") +
6761 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6768bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6774 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6775 if (!PALMetadata->setFromString(
String))
6776 return Error(getLoc(),
"invalid PAL metadata");
6781bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6784 Twine(
" directive is "
6785 "not available on non-amdpal OSes"))
6789 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6790 PALMetadata->setLegacy();
6793 if (ParseAsAbsoluteExpression(
Key)) {
6794 return TokError(Twine(
"invalid value in ") +
6798 return TokError(Twine(
"expected an even number of values in ") +
6801 if (ParseAsAbsoluteExpression(
Value)) {
6802 return TokError(Twine(
"invalid value in ") +
6805 PALMetadata->setRegister(
Key,
Value);
6814bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6815 if (getParser().checkForValidSection())
6819 SMLoc NameLoc = getLoc();
6820 if (getParser().parseIdentifier(Name))
6821 return TokError(
"expected identifier in directive");
6824 if (getParser().parseComma())
6830 SMLoc SizeLoc = getLoc();
6831 if (getParser().parseAbsoluteExpression(
Size))
6834 return Error(SizeLoc,
"size must be non-negative");
6835 if (
Size > LocalMemorySize)
6836 return Error(SizeLoc,
"size is too large");
6840 SMLoc AlignLoc = getLoc();
6841 if (getParser().parseAbsoluteExpression(Alignment))
6844 return Error(AlignLoc,
"alignment must be a power of two");
6849 if (Alignment >= 1u << 31)
6850 return Error(AlignLoc,
"alignment is too large");
6856 Symbol->redefineIfPossible();
6857 if (!
Symbol->isUndefined())
6858 return Error(NameLoc,
"invalid symbol redefinition");
6860 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6864bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6865 if (getParser().checkForValidSection())
6869 if (getParser().parseIdentifier(FuncName))
6870 return TokError(
"expected symbol name after .amdgpu_info");
6873 AMDGPU::InfoSectionData ParsedInfoData;
6874 AMDGPU::FuncInfo FI;
6876 bool HasScalarAttrs =
false;
6883 SMLoc IDLoc = getLoc();
6884 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6887 if (ID ==
".end_amdgpu_info")
6895 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6897 if (Dir ==
"flags") {
6899 if (getParser().parseAbsoluteExpression(Val))
6902 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6904 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6906 HasScalarAttrs =
true;
6907 }
else if (Dir ==
"num_sgpr") {
6909 if (getParser().parseAbsoluteExpression(Val))
6911 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6912 HasScalarAttrs =
true;
6913 }
else if (Dir ==
"num_vgpr") {
6915 if (getParser().parseAbsoluteExpression(Val))
6918 HasScalarAttrs =
true;
6919 }
else if (Dir ==
"num_agpr") {
6921 if (getParser().parseAbsoluteExpression(Val))
6924 HasScalarAttrs =
true;
6925 }
else if (Dir ==
"private_segment_size") {
6927 if (getParser().parseAbsoluteExpression(Val))
6930 HasScalarAttrs =
true;
6931 }
else if (Dir ==
"use") {
6933 if (getParser().parseIdentifier(ResName))
6934 return TokError(
"expected resource symbol for .amdgpu_use");
6935 ParsedInfoData.
Uses.push_back(
6936 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6937 }
else if (Dir ==
"call") {
6939 if (getParser().parseIdentifier(DstName))
6940 return TokError(
"expected callee symbol for .amdgpu_call");
6941 ParsedInfoData.
Calls.push_back(
6942 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6943 }
else if (Dir ==
"indirect_call") {
6945 if (getParser().parseEscapedString(TypeId))
6946 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6947 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6948 }
else if (Dir ==
"typeid") {
6950 if (getParser().parseEscapedString(TypeId))
6951 return TokError(
"expected type ID string for .amdgpu_typeid");
6952 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6954 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6959 ParsedInfoData.
Funcs.push_back(std::move(FI));
6961 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6962 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6963 Data.Funcs.push_back(std::move(Func));
6964 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6965 Data.Uses.push_back(Use);
6966 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6968 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6971 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6972 Data.TypeIds.push_back(std::move(TypeId));
6977void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
6984void AMDGPUAsmParser::checkKernelPrologues() {
6985 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
6986 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
6987 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
6988 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
6989 if (!AMDHSAKernelSymbols.
contains(Sym))
6991 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
6992 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
6996 "' does not begin with the required prologue "
6997 "sequence: s_mov_b64 followed by v_nop and "
6998 "global_prefetch_b8");
7002 OpcodeStream.
clear();
7003 OpcodeStreamSymbols.clear();
7004 AMDHSAKernelSymbols.
clear();
7007void AMDGPUAsmParser::onEndOfFile() {
7008 emitTargetDirective();
7009 checkKernelPrologues();
7011 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7014bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7015 StringRef IDVal = DirectiveID.
getString();
7018 if (IDVal ==
".amdhsa_kernel")
7019 return ParseDirectiveAMDHSAKernel();
7021 if (IDVal ==
".amdhsa_code_object_version")
7022 return ParseDirectiveAMDHSACodeObjectVersion();
7026 return ParseDirectiveHSAMetadata();
7028 if (IDVal ==
".amd_kernel_code_t")
7029 return ParseDirectiveAMDKernelCodeT();
7031 if (IDVal ==
".amdgpu_hsa_kernel")
7032 return ParseDirectiveAMDGPUHsaKernel();
7034 if (IDVal ==
".amd_amdgpu_isa")
7035 return ParseDirectiveISAVersion();
7039 Twine(
" directive is "
7040 "not available on non-amdhsa OSes"))
7045 if (IDVal ==
".amdgcn_target")
7046 return ParseDirectiveAMDGCNTarget();
7048 if (IDVal ==
".amdgpu_lds")
7049 return ParseDirectiveAMDGPULDS();
7051 if (IDVal ==
".amdgpu_info")
7052 return ParseDirectiveAMDGPUInfo();
7055 return ParseDirectivePALMetadataBegin();
7058 return ParseDirectivePALMetadata();
7063bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7070 return hasSGPR104_SGPR105();
7073 case SRC_SHARED_BASE_LO:
7074 case SRC_SHARED_BASE:
7075 case SRC_SHARED_LIMIT_LO:
7076 case SRC_SHARED_LIMIT:
7078 case SRC_PRIVATE_BASE_LO:
7079 case SRC_PRIVATE_BASE:
7080 case SRC_PRIVATE_LIMIT_LO:
7081 case SRC_PRIVATE_LIMIT:
7083 case SRC_FLAT_SCRATCH_BASE_LO:
7084 case SRC_FLAT_SCRATCH_BASE_HI:
7085 return hasGloballyAddressableScratch();
7086 case SRC_POPS_EXITING_WAVE_ID:
7099 getTargetStreamer().getTargetID()->isXnackSupported();
7129 return hasSGPR102_SGPR103();
7137 ParseStatus Res = parseVOPD(
Operands);
7142 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7154 SMLoc LBraceLoc = getLoc();
7159 auto Loc = getLoc();
7162 Error(Loc,
"expected a register");
7166 RBraceLoc = getLoc();
7171 "expected a comma or a closing square bracket"))
7175 if (
Operands.size() - Prefix > 1) {
7177 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7178 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7187StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7189 setForcedEncodingSize(0);
7190 setForcedDPP(
false);
7191 setForcedSDWA(
false);
7193 if (
Name.consume_back(
"_e64_dpp")) {
7195 setForcedEncodingSize(64);
7198 if (
Name.consume_back(
"_e64")) {
7199 setForcedEncodingSize(64);
7202 if (
Name.consume_back(
"_e32")) {
7203 setForcedEncodingSize(32);
7206 if (
Name.consume_back(
"_dpp")) {
7210 if (
Name.consume_back(
"_sdwa")) {
7211 setForcedSDWA(
true);
7219 unsigned VariantID);
7225 Name = parseMnemonicSuffix(Name);
7231 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7233 bool IsMIMG = Name.starts_with(
"image_");
7236 OperandMode
Mode = OperandMode_Default;
7238 Mode = OperandMode_NSA;
7242 checkUnsupportedInstruction(Name, NameLoc);
7243 if (!Parser.hasPendingError()) {
7246 :
"not a valid operand.";
7266ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7269 if (!trySkipId(Name))
7272 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7276ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7285ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7287 std::function<
bool(int64_t &)> ConvertResult) {
7291 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7295 if (ConvertResult && !ConvertResult(
Value)) {
7296 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7299 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7303ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7305 bool (*ConvertResult)(int64_t &)) {
7314 const unsigned MaxSize = 4;
7318 for (
int I = 0;; ++
I) {
7320 SMLoc Loc = getLoc();
7324 if (
Op != 0 &&
Op != 1)
7325 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7332 if (
I + 1 == MaxSize)
7333 return Error(getLoc(),
"expected a closing square bracket");
7339 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7343ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7345 AMDGPUOperand::ImmTy ImmTy,
7346 bool IgnoreNegative) {
7350 if (trySkipId(Name)) {
7352 }
else if (trySkipId(
"no", Name)) {
7361 return Error(S,
"r128 modifier is not supported on this GPU");
7362 if (Name ==
"a16" && !
hasA16())
7363 return Error(S,
"a16 modifier is not supported on this GPU");
7365 if (Bit == 0 && Name ==
"gds") {
7368 return Error(S,
"nogds is not allowed");
7371 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7372 ImmTy = AMDGPUOperand::ImmTyR128A16;
7374 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7378unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7379 bool &Disabling)
const {
7380 Disabling =
Id.consume_front(
"no");
7383 return StringSwitch<unsigned>(Id)
7390 return StringSwitch<unsigned>(Id)
7400 SMLoc StringLoc = getLoc();
7402 int64_t CPolVal = 0;
7422 ResScope = parseScope(
Operands, Scope);
7435 if (trySkipId(
"nv")) {
7439 }
else if (trySkipId(
"no",
"nv")) {
7446 if (trySkipId(
"scale_offset")) {
7450 }
else if (trySkipId(
"no",
"scale_offset")) {
7463 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7464 AMDGPUOperand::ImmTyCPol));
7469 SMLoc OpLoc = getLoc();
7470 unsigned Enabled = 0, Seen = 0;
7474 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7481 return Error(S,
"dlc modifier is not supported on this GPU");
7484 return Error(S,
"scc modifier is not supported on this GPU");
7487 return Error(S,
"duplicate cache policy modifier");
7499 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7508 ParseStatus Res = parseStringOrIntWithPrefix(
7509 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7523 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7527 if (
Value ==
"TH_DEFAULT")
7529 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7530 Value ==
"TH_LOAD_NT_WB") {
7531 return Error(StringLoc,
"invalid th value");
7532 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7534 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7536 }
else if (
Value.consume_front(
"TH_STORE_")) {
7539 return Error(StringLoc,
"invalid th value");
7542 if (
Value ==
"BYPASS")
7547 TH |= StringSwitch<int64_t>(
Value)
7557 .Default(0xffffffff);
7559 TH |= StringSwitch<int64_t>(
Value)
7570 .Default(0xffffffff);
7573 if (TH == 0xffffffff)
7574 return Error(StringLoc,
"invalid th value");
7581 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7582 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7583 std::optional<unsigned> InsertAt = std::nullopt) {
7584 auto i = OptionalIdx.find(ImmT);
7585 if (i != OptionalIdx.end()) {
7586 unsigned Idx = i->second;
7587 const AMDGPUOperand &
Op =
7588 static_cast<const AMDGPUOperand &
>(*
Operands[Idx]);
7592 Op.addImmOperands(Inst, 1);
7594 if (InsertAt.has_value())
7601ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7607 StringLoc = getLoc();
7612ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7618 SMLoc StringLoc = getLoc();
7622 Value = getTokenStr();
7626 if (
Value == Ids[IntVal])
7631 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7632 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7637ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7639 AMDGPUOperand::ImmTy
Type) {
7643 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7645 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7654bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7657 SMLoc Loc = getLoc();
7659 auto Res = parseIntWithPrefix(Pref, Val);
7665 if (Val < 0 || Val > MaxVal) {
7666 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7675 AMDGPUOperand::ImmTy ImmTy) {
7676 const char *Pref =
"index_key";
7678 SMLoc Loc = getLoc();
7679 auto Res = parseIntWithPrefix(Pref, ImmVal);
7683 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7684 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7685 (ImmVal < 0 || ImmVal > 1))
7686 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7688 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7689 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7691 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7696 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7700 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7704 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7709 AMDGPUOperand::ImmTy
Type) {
7715 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7716 AMDGPUOperand::ImmTyMatrixAFMT);
7720 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7721 AMDGPUOperand::ImmTyMatrixBFMT);
7726 AMDGPUOperand::ImmTy
Type) {
7732 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7733 AMDGPUOperand::ImmTyMatrixAScale);
7737 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7738 AMDGPUOperand::ImmTyMatrixBScale);
7743 AMDGPUOperand::ImmTy
Type) {
7749 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7750 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7754 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7755 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7760ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7761 using namespace llvm::AMDGPU::MTBUFFormat;
7767 for (
int I = 0;
I < 2; ++
I) {
7768 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7771 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7776 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7782 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7785 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7786 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7792ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7793 using namespace llvm::AMDGPU::MTBUFFormat;
7797 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7800 if (Fmt == UFMT_UNDEF)
7807bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7808 StringRef FormatStr, SMLoc Loc) {
7809 using namespace llvm::AMDGPU::MTBUFFormat;
7813 if (
Format != DFMT_UNDEF) {
7819 if (
Format != NFMT_UNDEF) {
7824 Error(Loc,
"unsupported format");
7828ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7831 using namespace llvm::AMDGPU::MTBUFFormat;
7835 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7840 SMLoc Loc = getLoc();
7841 if (!parseId(Str,
"expected a format string") ||
7842 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7844 if (Dfmt == DFMT_UNDEF)
7845 return Error(Loc,
"duplicate numeric format");
7846 if (Nfmt == NFMT_UNDEF)
7847 return Error(Loc,
"duplicate data format");
7850 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7851 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7855 if (Ufmt == UFMT_UNDEF)
7856 return Error(FormatLoc,
"unsupported format");
7865ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7868 using namespace llvm::AMDGPU::MTBUFFormat;
7871 if (Id == UFMT_UNDEF)
7875 return Error(Loc,
"unified format is not supported on this GPU");
7881ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7882 using namespace llvm::AMDGPU::MTBUFFormat;
7883 SMLoc Loc = getLoc();
7888 return Error(Loc,
"out of range format");
7893ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7894 using namespace llvm::AMDGPU::MTBUFFormat;
7900 StringRef FormatStr;
7901 SMLoc Loc = getLoc();
7902 if (!parseId(FormatStr,
"expected a format string"))
7905 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7907 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7917 return parseNumericFormat(
Format);
7921 using namespace llvm::AMDGPU::MTBUFFormat;
7925 SMLoc Loc = getLoc();
7935 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7954 Res = parseSymbolicOrNumericFormat(
Format);
7959 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7960 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7967 return Error(getLoc(),
"duplicate format");
7973 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7975 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7976 AMDGPUOperand::ImmTyInstOffset);
7983 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
7985 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
7991 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
7994 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8004 OptionalImmIndexMap OptionalIdx;
8006 unsigned OperandIdx[4];
8007 unsigned EnMask = 0;
8010 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8011 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8016 OperandIdx[SrcIdx] = Inst.
size();
8017 Op.addRegOperands(Inst, 1);
8024 OperandIdx[SrcIdx] = Inst.
size();
8030 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8031 Op.addImmOperands(Inst, 1);
8035 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8039 OptionalIdx[
Op.getImmTy()] = i;
8045 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8052 for (
auto i = 0; i < SrcIdx; ++i) {
8054 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8060 AMDGPUOperand::ImmTyExpCompr);
8070 int64_t CntVal,
bool Saturate,
8076 IntVal =
encode(ISA, IntVal, CntVal);
8077 if (CntVal !=
decode(ISA, IntVal)) {
8079 IntVal =
encode(ISA, IntVal, -1);
8087bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8089 SMLoc CntLoc = getLoc();
8090 StringRef CntName = getTokenStr();
8097 SMLoc ValLoc = getLoc();
8104 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8106 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8108 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8111 Error(CntLoc,
"invalid counter name " + CntName);
8116 Error(ValLoc,
"too large value for " + CntName);
8125 Error(getLoc(),
"expected a counter name");
8139 if (!parseCnt(Waitcnt))
8147 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8151bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8152 SMLoc FieldLoc = getLoc();
8153 StringRef FieldName = getTokenStr();
8158 SMLoc ValueLoc = getLoc();
8165 if (FieldName ==
"instid0") {
8167 }
else if (FieldName ==
"instskip") {
8169 }
else if (FieldName ==
"instid1") {
8172 Error(FieldLoc,
"invalid field name " + FieldName);
8191 .Case(
"VALU_DEP_1", 1)
8192 .Case(
"VALU_DEP_2", 2)
8193 .Case(
"VALU_DEP_3", 3)
8194 .Case(
"VALU_DEP_4", 4)
8195 .Case(
"TRANS32_DEP_1", 5)
8196 .Case(
"TRANS32_DEP_2", 6)
8197 .Case(
"TRANS32_DEP_3", 7)
8198 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8199 .Case(
"SALU_CYCLE_1", 9)
8200 .Case(
"SALU_CYCLE_2", 10)
8201 .Case(
"SALU_CYCLE_3", 11)
8209 Delay |=
Value << Shift;
8219 if (!parseDelay(Delay))
8227 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8231bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8233bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8239void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8240 StringRef DepCtrName) {
8243 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8246 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8249 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8252 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8259bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8261 using namespace llvm::AMDGPU::DepCtr;
8263 SMLoc DepCtrLoc = getLoc();
8264 StringRef DepCtrName = getTokenStr();
8274 unsigned PrevOprMask = UsedOprMask;
8275 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8278 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8287 Error(getLoc(),
"expected a counter name");
8292 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8293 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8298 using namespace llvm::AMDGPU::DepCtr;
8301 SMLoc Loc = getLoc();
8304 unsigned UsedOprMask = 0;
8306 if (!parseDepCtr(DepCtr, UsedOprMask))
8314 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8318bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8324ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8326 OperandInfoTy &Width) {
8327 using namespace llvm::AMDGPU::Hwreg;
8333 HwReg.Loc = getLoc();
8336 HwReg.IsSymbolic =
true;
8338 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8346 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8356 Width.Loc = getLoc();
8365 using namespace llvm::AMDGPU::Hwreg;
8368 SMLoc Loc = getLoc();
8370 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8372 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8373 HwregOffset::Default);
8374 struct : StructuredOpField {
8375 using StructuredOpField::StructuredOpField;
8376 bool validate(AMDGPUAsmParser &Parser)
const override {
8378 return Error(Parser,
"only values from 1 to 32 are legal");
8381 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8382 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8385 Res = parseHwregFunc(HwReg,
Offset, Width);
8388 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8390 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8394 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8401 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8403 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8407bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8413bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8414 OperandInfoTy &Stream) {
8415 using namespace llvm::AMDGPU::SendMsg;
8420 Msg.IsSymbolic =
true;
8427 Op.IsDefined =
true;
8433 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8438 Stream.IsDefined =
true;
8439 Stream.Loc = getLoc();
8448bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8449 const OperandInfoTy &
Op,
8450 const OperandInfoTy &Stream) {
8451 using namespace llvm::AMDGPU::SendMsg;
8460 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8465 Error(
Msg.Loc,
"invalid message id");
8471 Error(
Op.Loc,
"message does not support operations");
8473 Error(
Msg.Loc,
"missing message operation");
8479 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8481 Error(
Op.Loc,
"invalid operation id");
8486 Error(Stream.Loc,
"message operation does not support streams");
8490 Error(Stream.Loc,
"invalid message stream id");
8497 using namespace llvm::AMDGPU::SendMsg;
8500 SMLoc Loc = getLoc();
8504 OperandInfoTy
Op(OP_NONE_);
8505 OperandInfoTy Stream(STREAM_ID_NONE_);
8506 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8511 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8513 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8519 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8523bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8526 using namespace llvm::AMDGPU::WaitEvent;
8528 SMLoc Loc = getLoc();
8531 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8533 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8535 StructuredOpField *TargetBitfield =
8536 isGFX11() ? &DontWaitExportReady : &ExportReady;
8538 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8542 if (!validateStructuredOpFields({TargetBitfield}))
8544 ImmVal = TargetBitfield->Val;
8551 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8553 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8554 AMDGPUOperand::ImmTyWaitEvent));
8558bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8571 int Slot = StringSwitch<int>(Str)
8578 return Error(S,
"invalid interpolation slot");
8581 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8592 if (!Str.starts_with(
"attr"))
8593 return Error(S,
"invalid interpolation attribute");
8595 StringRef Chan = Str.take_back(2);
8596 int AttrChan = StringSwitch<int>(Chan)
8603 return Error(S,
"invalid or missing interpolation attribute channel");
8605 Str = Str.drop_back(2).drop_front(4);
8608 if (Str.getAsInteger(10, Attr))
8609 return Error(S,
"invalid or missing interpolation attribute number");
8612 return Error(S,
"out of bounds interpolation attribute number");
8617 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8618 Operands.push_back(AMDGPUOperand::CreateImm(
8619 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8628 using namespace llvm::AMDGPU::Exp;
8638 return Error(S, (Id == ET_INVALID)
8639 ?
"invalid exp target"
8640 :
"exp target is not supported on this GPU");
8643 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8651bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8655bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8660 return getTokenKind() ==
Kind;
8663StringRef AMDGPUAsmParser::getId()
const {
8667bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8675bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8677 StringRef Tok = getTokenStr();
8686bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8688 if (isId(Id) && peekToken().is(Kind)) {
8697 if (isToken(Kind)) {
8705 const StringRef ErrMsg) {
8706 if (!trySkipToken(Kind)) {
8707 Error(getLoc(), ErrMsg);
8713bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8717 if (Parser.parseExpression(Expr))
8720 if (Expr->evaluateAsAbsolute(
Imm))
8723 if (Expected.empty()) {
8724 Error(S,
"expected absolute expression");
8727 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8736 if (Parser.parseExpression(Expr))
8740 if (Expr->evaluateAsAbsolute(IntVal)) {
8741 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8743 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8748bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8750 Val =
getToken().getStringContents();
8754 Error(getLoc(), ErrMsg);
8758bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8760 Val = getTokenStr();
8764 if (!ErrMsg.
empty())
8765 Error(getLoc(), ErrMsg);
8769AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8771AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8774 : getLexer().peekTok(ShouldSkipSpace);
8778 auto TokCount = getLexer().peekTokens(Tokens);
8780 for (
auto Idx = TokCount; Idx < Tokens.
size(); ++Idx)
8785 return getLexer().getKind();
8788SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8790StringRef AMDGPUAsmParser::getTokenStr()
const {
8794void AMDGPUAsmParser::lex() { Parser.Lex(); }
8796const AMDGPUOperand &
8798 int MCOpIdx)
const {
8800 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8801 if (TargetOp.getMCOpIdx() == MCOpIdx)
8808 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8812SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8817 int MCOpIdx)
const {
8818 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8821SMLoc AMDGPUAsmParser::getOperandLoc(
8822 std::function<
bool(
const AMDGPUOperand &)>
Test,
8824 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8825 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8827 return Op.getStartLoc();
8832SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8834 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8849 StringRef
Id = getTokenStr();
8850 SMLoc IdLoc = getLoc();
8856 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8857 if (
I == Fields.
end())
8858 return Error(IdLoc,
"unknown field");
8859 if ((*I)->IsDefined)
8860 return Error(IdLoc,
"duplicate field");
8863 (*I)->Loc = getLoc();
8866 (*I)->IsDefined =
true;
8873bool AMDGPUAsmParser::validateStructuredOpFields(
8875 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8876 return F->validate(*
this);
8886 const unsigned XorMask) {
8893bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8894 const unsigned MaxVal,
8895 const Twine &ErrMsg, SMLoc &Loc) {
8911bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8912 const unsigned MinVal,
8913 const unsigned MaxVal,
8914 const StringRef ErrMsg) {
8916 for (
unsigned i = 0; i < OpNum; ++i) {
8917 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8924bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8925 using namespace llvm::AMDGPU::Swizzle;
8928 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8929 "expected a 2-bit lane id")) {
8939bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8940 using namespace llvm::AMDGPU::Swizzle;
8946 if (!parseSwizzleOperand(GroupSize, 2, 32,
8947 "group size must be in the interval [2,32]", Loc)) {
8951 Error(Loc,
"group size must be a power of two");
8954 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8955 "lane id must be in the interval [0,group size - 1]",
8963bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8964 using namespace llvm::AMDGPU::Swizzle;
8969 if (!parseSwizzleOperand(GroupSize, 2, 32,
8970 "group size must be in the interval [2,32]", Loc)) {
8974 Error(Loc,
"group size must be a power of two");
8982bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
8983 using namespace llvm::AMDGPU::Swizzle;
8988 if (!parseSwizzleOperand(GroupSize, 1, 16,
8989 "group size must be in the interval [1,16]", Loc)) {
8993 Error(Loc,
"group size must be a power of two");
9001bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9002 using namespace llvm::AMDGPU::Swizzle;
9009 SMLoc StrLoc = getLoc();
9010 if (!parseString(Ctl)) {
9013 if (Ctl.
size() != BITMASK_WIDTH) {
9014 Error(StrLoc,
"expected a 5-character mask");
9018 unsigned AndMask = 0;
9019 unsigned OrMask = 0;
9020 unsigned XorMask = 0;
9022 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9026 Error(StrLoc,
"invalid mask");
9047bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9048 using namespace llvm::AMDGPU::Swizzle;
9051 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9057 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9058 "FFT swizzle must be in the interval [0," +
9059 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9067bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9068 using namespace llvm::AMDGPU::Swizzle;
9071 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9078 if (!parseSwizzleOperand(
Direction, 0, 1,
9079 "direction must be 0 (left) or 1 (right)", Loc))
9083 if (!parseSwizzleOperand(
9084 RotateSize, 0, ROTATE_MAX_SIZE,
9085 "number of threads to rotate must be in the interval [0," +
9086 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9091 (RotateSize << ROTATE_SIZE_SHIFT);
9095bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9097 SMLoc OffsetLoc = getLoc();
9103 Error(OffsetLoc,
"expected a 16-bit offset");
9109bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9110 using namespace llvm::AMDGPU::Swizzle;
9114 SMLoc ModeLoc = getLoc();
9117 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9118 Ok = parseSwizzleQuadPerm(
Imm);
9119 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9120 Ok = parseSwizzleBitmaskPerm(
Imm);
9121 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9122 Ok = parseSwizzleBroadcast(
Imm);
9123 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9124 Ok = parseSwizzleSwap(
Imm);
9125 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9126 Ok = parseSwizzleReverse(
Imm);
9127 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9128 Ok = parseSwizzleFFT(
Imm);
9129 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9130 Ok = parseSwizzleRotate(
Imm);
9132 Error(ModeLoc,
"expected a swizzle mode");
9135 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9145 if (trySkipId(
"offset")) {
9149 if (trySkipId(
"swizzle")) {
9150 Ok = parseSwizzleMacro(
Imm);
9152 Ok = parseSwizzleOffset(
Imm);
9157 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9164bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9170int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9172 using namespace llvm::AMDGPU::VGPRIndexMode;
9184 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9185 if (trySkipId(IdSymbolic[ModeId])) {
9193 ?
"expected a VGPR index mode or a closing parenthesis"
9194 :
"expected a VGPR index mode");
9199 Error(S,
"duplicate VGPR index mode");
9207 "expected a comma or a closing parenthesis"))
9216 using namespace llvm::AMDGPU::VGPRIndexMode;
9222 Imm = parseGPRIdxMacro();
9226 if (getParser().parseAbsoluteExpression(
Imm))
9229 return Error(S,
"invalid immediate: only 4-bit values are legal");
9233 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9237bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9248 if (isRegister() || isModifier())
9255 assert(Opr.isImm() || Opr.isExpr());
9256 SMLoc Loc = Opr.getStartLoc();
9260 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9261 Error(Loc,
"expected an absolute expression or a label");
9262 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9263 Error(Loc,
"expected a 16-bit signed jump offset");
9283 OptionalImmIndexMap OptionalIdx;
9284 unsigned FirstOperandIdx = 1;
9285 bool IsAtomicReturn =
false;
9291 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9292 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9296 Op.addRegOperands(Inst, 1);
9300 if (IsAtomicReturn && i == FirstOperandIdx)
9301 Op.addRegOperands(Inst, 1);
9306 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9307 Op.addImmOperands(Inst, 1);
9319 OptionalIdx[
Op.getImmTy()] = i;
9323 AMDGPUOperand::ImmTyOffset);
9335bool AMDGPUOperand::isSMRDOffset8()
const {
9339bool AMDGPUOperand::isSMEMOffset()
const {
9341 return isImmLiteral();
9344bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9379bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9380 if (BoundCtrl == 0 || BoundCtrl == 1) {
9388void AMDGPUAsmParser::onBeginOfFile() {
9389 if (!getParser().getStreamer().getTargetStreamer())
9392 if (!getTargetStreamer().getTargetID())
9393 getTargetStreamer().initializeTargetID(getSTI(),
9397void AMDGPUAsmParser::emitTargetDirective() {
9398 if (TargetDirectiveEmitted)
9400 TargetDirectiveEmitted =
true;
9402 if (!getParser().getStreamer().getTargetStreamer() ||
9407 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9416bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9420 StringRef TokenId = getTokenStr();
9421 AGVK VK = StringSwitch<AGVK>(TokenId)
9422 .Case(
"max", AGVK::AGVK_Max)
9423 .Case(
"min", AGVK::AGVK_Min)
9424 .Case(
"or", AGVK::AGVK_Or)
9425 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9426 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9427 .Case(
"alignto", AGVK::AGVK_AlignTo)
9428 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9429 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9430 .Default(AGVK::AGVK_None);
9439 if (Exprs.
empty()) {
9441 "empty " + Twine(TokenId) +
" expression");
9444 if (CommaCount + 1 != Exprs.
size()) {
9446 "mismatch of commas in " + Twine(TokenId) +
" expression");
9450 Expected && Exprs.
size() != Expected) {
9451 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9452 Twine(Expected) +
" operands");
9459 if (getParser().parseExpression(Expr, EndLoc))
9463 if (LastTokenWasComma)
9467 "unexpected token in " + Twine(TokenId) +
" expression");
9473 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9477 StringRef
Name = getTokenStr();
9478 if (Name ==
"mul") {
9479 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9483 if (Name ==
"div") {
9484 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9495 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9500 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9501 AMDGPU::OpName::src2};
9509 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9514 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9516 if (
DstOp.isReg() &&
9521 if ((OpSel & (1 << SrcNum)) != 0)
9527void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9534 OptionalImmIndexMap &OptionalIdx) {
9535 cvtVOP3P(Inst,
Operands, OptionalIdx);
9544 &&
Desc.NumOperands > (OpNum + 1)
9546 &&
Desc.operands()[OpNum + 1].RegClass != -1
9548 &&
Desc.getOperandConstraint(OpNum + 1,
9552void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9554 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9555 AMDGPU::OpName::src2};
9556 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9557 AMDGPU::OpName::src1_modifiers,
9558 AMDGPU::OpName::src2_modifiers};
9559 for (
int J = 0; J < 3; ++J) {
9560 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9566 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9569 if ((OpSel & (1 << J)) != 0)
9572 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9579void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9581 OptionalImmIndexMap OptionalIdx;
9586 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9587 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9591 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9593 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9594 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9595 Op.isInterpAttrChan()) {
9597 }
else if (
Op.isImmModifier()) {
9598 OptionalIdx[
Op.getImmTy()] =
I;
9606 AMDGPUOperand::ImmTyHigh);
9610 AMDGPUOperand::ImmTyClamp);
9614 AMDGPUOperand::ImmTyOModSI);
9619 AMDGPUOperand::ImmTyOpSel);
9620 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9623 cvtOpSelHelper(Inst, OpSel);
9628 OptionalImmIndexMap OptionalIdx;
9633 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9634 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9638 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9640 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9641 }
else if (
Op.isImmModifier()) {
9642 OptionalIdx[
Op.getImmTy()] =
I;
9650 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9653 AMDGPUOperand::ImmTyOpSel);
9656 AMDGPUOperand::ImmTyWaitEXP);
9662 cvtOpSelHelper(Inst, OpSel);
9665void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9667 OptionalImmIndexMap OptionalIdx;
9670 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9674 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9675 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9678 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9683 if (NumOperands == CbszOpIdx) {
9688 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9689 }
else if (
Op.isImmModifier()) {
9690 OptionalIdx[
Op.getImmTy()] =
I;
9692 Op.addRegOrImmOperands(Inst, 1);
9697 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9698 if (CbszIdx != OptionalIdx.end()) {
9699 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9703 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9704 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9705 if (BlgpIdx != OptionalIdx.end()) {
9706 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9717 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9718 if (OpselIdx != OptionalIdx.end()) {
9719 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9723 unsigned OpSelHi = 0;
9724 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9725 if (OpselHiIdx != OptionalIdx.end()) {
9726 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9729 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9730 AMDGPU::OpName::src1_modifiers};
9732 for (
unsigned J = 0; J < 2; ++J) {
9733 unsigned ModVal = 0;
9734 if (OpSel & (1 << J))
9736 if (OpSelHi & (1 << J))
9739 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9745 OptionalImmIndexMap &OptionalIdx) {
9750 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9751 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9755 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9757 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9758 }
else if (
Op.isImmModifier()) {
9759 OptionalIdx[
Op.getImmTy()] =
I;
9761 Op.addRegOrImmOperands(Inst, 1);
9767 AMDGPUOperand::ImmTyScaleSel);
9771 AMDGPUOperand::ImmTyClamp);
9777 AMDGPUOperand::ImmTyByteSel);
9782 AMDGPUOperand::ImmTyOModSI);
9789 auto *it = Inst.
begin();
9791 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9800 OptionalImmIndexMap OptionalIdx;
9801 cvtVOP3(Inst,
Operands, OptionalIdx);
9805 OptionalImmIndexMap &OptIdx) {
9810 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9811 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9812 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9813 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9814 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9815 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9816 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9817 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9818 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9819 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9828 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9829 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9832 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9833 if (BitOp3Idx != -1) {
9840 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9841 if (OpSelIdx != -1) {
9845 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9846 if (OpSelHiIdx != -1) {
9847 int DefaultVal =
IsPacked ? -1 : 0;
9853 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9854 if (MatrixAFMTIdx != -1) {
9856 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9860 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9861 if (MatrixBFMTIdx != -1) {
9863 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9866 int MatrixAScaleIdx =
9867 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9868 if (MatrixAScaleIdx != -1) {
9870 AMDGPUOperand::ImmTyMatrixAScale, 0);
9873 int MatrixBScaleIdx =
9874 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9875 if (MatrixBScaleIdx != -1) {
9877 AMDGPUOperand::ImmTyMatrixBScale, 0);
9880 int MatrixAScaleFmtIdx =
9881 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9882 if (MatrixAScaleFmtIdx != -1) {
9884 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9887 int MatrixBScaleFmtIdx =
9888 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9889 if (MatrixBScaleFmtIdx != -1) {
9891 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9896 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9900 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9902 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9906 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9910 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9911 AMDGPU::OpName::src2};
9912 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9913 AMDGPU::OpName::src1_modifiers,
9914 AMDGPU::OpName::src2_modifiers};
9917 unsigned OpSelHi = 0;
9924 if (OpSelHiIdx != -1)
9933 for (
int J = 0; J < 3; ++J) {
9934 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9938 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9948 uint32_t ModVal = 0;
9950 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9951 if (SrcOp.
isReg() && getMRI()
9958 if ((OpSel & (1 << J)) != 0)
9962 if ((OpSelHi & (1 << J)) != 0)
9965 if ((NegLo & (1 << J)) != 0)
9968 if ((NegHi & (1 << J)) != 0)
9976 OptionalImmIndexMap OptIdx;
9982 unsigned i,
unsigned Opc,
9984 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
9985 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
9987 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
9993 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9996 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
9997 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
9999 OptionalImmIndexMap OptIdx;
10000 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10001 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10002 OptIdx[
Op.getImmTy()] = i;
10007 AMDGPUOperand::ImmTyIndexKey8bit);
10011 AMDGPUOperand::ImmTyIndexKey16bit);
10015 AMDGPUOperand::ImmTyIndexKey32bit);
10032 SMLoc S = getLoc();
10035 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10036 SMLoc OpYLoc = getLoc();
10039 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10042 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10051 auto addOp = [&](uint16_t ParsedOprIdx) {
10052 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10054 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10058 Op.addRegOperands(Inst, 1);
10062 Op.addImmOperands(Inst, 1);
10074 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10078 const auto &CInfo = InstInfo[CompIdx];
10079 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10080 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10081 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10082 if (CInfo.hasSrc2Acc())
10083 addOp(CInfo.getIndexOfDstInParsedOperands());
10087 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10088 if (BitOp3Idx != -1) {
10089 OptionalImmIndexMap OptIdx;
10090 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10092 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10102bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10104bool AMDGPUOperand::isDPPCtrl()
const {
10105 using namespace AMDGPU::DPP;
10107 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10110 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10111 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10112 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10113 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10114 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10115 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10116 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10117 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10118 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10119 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10120 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10121 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10130bool AMDGPUOperand::isBLGP()
const {
10134bool AMDGPUOperand::isS16Imm()
const {
10138bool AMDGPUOperand::isU16Imm()
const {
10146bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10151 SMLoc Loc =
getToken().getEndLoc();
10152 Token = std::string(getTokenStr());
10154 if (getLoc() != Loc)
10159 if (!parseId(Suffix))
10163 StringRef DimId = Token;
10178 SMLoc S = getLoc();
10184 SMLoc Loc = getLoc();
10185 if (!parseDimId(Encoding))
10186 return Error(Loc,
"invalid dim value");
10189 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10198 SMLoc S = getLoc();
10207 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10210 for (
size_t i = 0; i < 8; ++i) {
10214 SMLoc Loc = getLoc();
10215 if (getParser().parseAbsoluteExpression(Sels[i]))
10217 if (0 > Sels[i] || 7 < Sels[i])
10218 return Error(Loc,
"expected a 3-bit value");
10221 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10225 for (
size_t i = 0; i < 8; ++i)
10226 DPP8 |= (Sels[i] << (i * 3));
10229 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10233bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10235 if (Ctrl ==
"row_newbcast")
10238 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10241 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10242 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10245 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10246 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10250int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10253 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10257 for (
int i = 0; i < 4; ++i) {
10262 SMLoc Loc = getLoc();
10263 if (getParser().parseAbsoluteExpression(Temp))
10265 if (Temp < 0 || Temp > 3) {
10266 Error(Loc,
"expected a 2-bit value");
10270 Val += (Temp << i * 2);
10273 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10279int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10280 using namespace AMDGPU::DPP;
10285 SMLoc Loc = getLoc();
10287 if (getParser().parseAbsoluteExpression(Val))
10290 struct DppCtrlCheck {
10296 DppCtrlCheck
Check =
10297 StringSwitch<DppCtrlCheck>(Ctrl)
10298 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10299 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10300 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10301 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10302 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10303 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10304 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10305 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10306 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10307 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10311 if (
Check.Ctrl == -1) {
10312 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10320 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10328 using namespace AMDGPU::DPP;
10331 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10334 SMLoc S = getLoc();
10340 if (Ctrl ==
"row_mirror") {
10341 Val = DppCtrl::ROW_MIRROR;
10342 }
else if (Ctrl ==
"row_half_mirror") {
10343 Val = DppCtrl::ROW_HALF_MIRROR;
10346 if (Ctrl ==
"quad_perm") {
10347 Val = parseDPPCtrlPerm();
10349 Val = parseDPPCtrlSel(Ctrl);
10358 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10364 OptionalImmIndexMap OptionalIdx;
10371 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10373 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10374 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10378 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10379 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10383 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10384 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10385 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10386 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10387 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10388 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10389 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10390 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10391 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10397 if (OldIdx == NumOperands) {
10399 constexpr int DST_IDX = 0;
10401 }
else if (Src2ModIdx == NumOperands) {
10411 if (IsVOP3CvtSrDpp) {
10420 if (TiedTo != -1) {
10425 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10427 if (IsDPP8 &&
Op.isDppFI()) {
10430 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10431 }
else if (
Op.isReg()) {
10432 Op.addRegOperands(Inst, 1);
10433 }
else if (
Op.isImm() &&
10435 Op.addImmOperands(Inst, 1);
10436 }
else if (
Op.isImm()) {
10437 OptionalIdx[
Op.getImmTy()] =
I;
10445 AMDGPUOperand::ImmTyClamp);
10451 AMDGPUOperand::ImmTyByteSel);
10456 AMDGPUOperand::ImmTyOModSI);
10459 cvtVOP3P(Inst,
Operands, OptionalIdx);
10461 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10464 AMDGPUOperand::ImmTyOpSel);
10469 AMDGPUOperand::ImmTyDPP8);
10470 using namespace llvm::AMDGPU::DPP;
10474 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10476 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10478 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10480 AMDGPUOperand::ImmTyDppBoundCtrl);
10484 AMDGPUOperand::ImmTyDppFI);
10490 OptionalImmIndexMap OptionalIdx;
10494 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10495 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10502 if (TiedTo != -1) {
10507 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10509 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10517 Op.addImmOperands(Inst, 1);
10519 Op.addRegWithFPInputModsOperands(Inst, 2);
10520 }
else if (
Op.isDppFI()) {
10522 }
else if (
Op.isReg()) {
10523 Op.addRegOperands(Inst, 1);
10529 Op.addRegWithFPInputModsOperands(Inst, 2);
10530 }
else if (
Op.isReg()) {
10531 Op.addRegOperands(Inst, 1);
10532 }
else if (
Op.isDPPCtrl()) {
10533 Op.addImmOperands(Inst, 1);