Skip to content

Commit 0b995ed

Browse files
arsenmpravinjagtap
arsenm
authored and
pravinjagtap
committed
AMDGPU: Add v_smfmac_f32_16x16x128_bf8_bf8 for gfx950 (llvm#117232)
1 parent 6afb53e commit 0b995ed

File tree

13 files changed

+307
-1
lines changed

13 files changed

+307
-1
lines changed

‎clang/include/clang/Basic/BuiltinsAMDGPU.def

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_16x16x64_bf16, "V4fV8yV16yV4fiIiIi",
450450
TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_32x32x32_bf16, "V16fV8yV16yV16fiIiIi", "nc", "gfx950-insts")
451451
TARGET_BUILTIN(__builtin_amdgcn_smfmac_i32_16x16x128_i8, "V4iV4iV8iV4iiIiIi", "nc", "gfx950-insts")
452452
TARGET_BUILTIN(__builtin_amdgcn_smfmac_i32_32x32x64_i8, "V16iV4iV8iV16iiIiIi", "nc", "gfx950-insts")
453+
TARGET_BUILTIN(__builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8, "V4fV4iV8iV4fiIiIi", "nc", "gfx950-insts")
453454

454455
//===----------------------------------------------------------------------===//
455456
// GFX12+ only builtins.

‎clang/test/CodeGenOpenCL/builtins-amdgcn-mfma.cl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,4 +510,11 @@ void test_smfmac_i32_32x32x64_i8(global v16i* out, v4i a, v8i b, v16i c, int idx
510510
*out = __builtin_amdgcn_smfmac_i32_32x32x64_i8(a, b, c, idx, 0, 0);
511511
}
512512

513+
// CHECK-GFX950-LABEL: @test_smfmac_f32_16x16x128_bf8_bf8
514+
// CHECK-GFX950: call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %a, <8 x i32> %b, <4 x float> %c, i32 %idx, i32 0, i32 0)
515+
void test_smfmac_f32_16x16x128_bf8_bf8(global v4f* out, v4i a, v8i b, v4f c, int idx)
516+
{
517+
*out = __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8(a, b, c, idx, 0, 0);
518+
}
519+
513520
#endif

‎clang/test/SemaOpenCL/builtins-amdgcn-error-gfx950-param.cl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,9 @@ void test_smfmac_i32_32x32x64_i8(global int16* out, int4 a, int8 b, int16 c, int
100100
*out = __builtin_amdgcn_smfmac_i32_32x32x64_i8(a, b, c, idx, d, 0); // expected-error{{argument to '__builtin_amdgcn_smfmac_i32_32x32x64_i8' must be a constant integer}}
101101
*out = __builtin_amdgcn_smfmac_i32_32x32x64_i8(a, b, c, idx, 0, d); // expected-error{{argument to '__builtin_amdgcn_smfmac_i32_32x32x64_i8' must be a constant integer}}
102102
}
103+
104+
void test_smfmac_f32_16x16x128_bf8_bf8(global float4* out, int4 a, int8 b, float4 c, int idx, int d)
105+
{
106+
*out = __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8(a, b, c, idx, d, 0); // expected-error{{argument to '__builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8' must be a constant integer}}
107+
*out = __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8(a, b, c, idx, 0, d); // expected-error{{argument to '__builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8' must be a constant integer}}
108+
}

‎clang/test/SemaOpenCL/builtins-amdgcn-error-gfx950.cl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ void test(__global float4* out0, half8 a0, half8 b0, float4 c0,
4040
*out9 = __builtin_amdgcn_smfmac_f32_32x32x32_bf16(a9, b9, c9, 0, 0, 0); // expected-error{{'__builtin_amdgcn_smfmac_f32_32x32x32_bf16' needs target feature gfx950-insts}}
4141
*out10 = __builtin_amdgcn_smfmac_i32_16x16x128_i8(a10, b10, c10, 0, 0, 0); // expected-error{{'__builtin_amdgcn_smfmac_i32_16x16x128_i8' needs target feature gfx950-insts}}
4242
*out11 = __builtin_amdgcn_smfmac_i32_32x32x64_i8(a11, b11, c11, 0, 0, 0); // expected-error{{'__builtin_amdgcn_smfmac_i32_32x32x64_i8' needs target feature gfx950-insts}}
43+
*out12 = __builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8(a12, b12, c12, 0, 0, 0); // expected-error{{'__builtin_amdgcn_smfmac_f32_16x16x128_bf8_bf8' needs target feature gfx950-insts}}
4344
*out14 = __builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4(a14, b14, c14, 0, 0, 0, d14, 0, e14); // expected-error{{'__builtin_amdgcn_mfma_scale_f32_16x16x128_f8f6f4' needs target feature gfx950-insts}}
4445
*out15 = __builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4(a15, b15, c15, 0, 0, 0, d15, 0, e15); // expected-error{{'__builtin_amdgcn_mfma_scale_f32_32x32x64_f8f6f4' needs target feature gfx950-insts}}
4546
}

‎llvm/include/llvm/IR/IntrinsicsAMDGPU.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3185,6 +3185,7 @@ def int_amdgcn_smfmac_f32_16x16x64_bf16 : AMDGPUMSmfmacIntrinsic<llvm_v4f32_ty,
31853185
def int_amdgcn_smfmac_f32_32x32x32_bf16 : AMDGPUMSmfmacIntrinsic<llvm_v16f32_ty, llvm_v8bf16_ty, llvm_v16bf16_ty>;
31863186
def int_amdgcn_smfmac_i32_16x16x128_i8 : AMDGPUMSmfmacIntrinsic<llvm_v4i32_ty, llvm_v4i32_ty, llvm_v8i32_ty>;
31873187
def int_amdgcn_smfmac_i32_32x32x64_i8 : AMDGPUMSmfmacIntrinsic<llvm_v16i32_ty, llvm_v4i32_ty, llvm_v8i32_ty>;
3188+
def int_amdgcn_smfmac_f32_16x16x128_bf8_bf8 : AMDGPUMSmfmacIntrinsic<llvm_v4f32_ty, llvm_v4i32_ty, llvm_v8i32_ty>;
31883189
}
31893190

31903191
//===----------------------------------------------------------------------===//

‎llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1081,6 +1081,7 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
10811081
case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
10821082
case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
10831083
case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1084+
case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
10841085
return selectSMFMACIntrin(I);
10851086
default:
10861087
return selectImpl(I, *CoverageInfo);
@@ -3517,6 +3518,9 @@ bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
35173518
case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
35183519
Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
35193520
break;
3521+
case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3522+
Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3523+
break;
35203524
default:
35213525
llvm_unreachable("unhandled smfmac intrinsic");
35223526
}

‎llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4787,7 +4787,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
47874787
case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
47884788
case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
47894789
case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
4790-
case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8: {
4790+
case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
4791+
case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8: {
47914792
// vdst, srcA, srcB, srcC, idx
47924793
OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
47934794
OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);

‎llvm/lib/Target/AMDGPU/SIInstrInfo.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2690,6 +2690,7 @@ def VOP_V4F32_V2I32_V4I32_I32 : VOPProfile <[v4f32, v2i32, v4i32, i32]>;
26902690
def VOP_V16F32_V2I32_V4I32_I32 : VOPProfile <[v16f32, v2i32, v4i32, i32]>;
26912691
def VOP_V4I32_V4I32_V8I32_I32 : VOPProfile <[v4i32, v4i32, v8i32, i32]>;
26922692
def VOP_V16I32_V4I32_V8I32_I32 : VOPProfile <[v16i32, v4i32, v8i32, i32]>;
2693+
def VOP_V4F32_V4I32_V8I32_I32 : VOPProfile <[v4f32, v4i32, v8i32, i32]>;
26932694

26942695
def VOP_V4F32_V8F16_V8F16_V4F32 : VOPProfile <[v4f32, v8f16, v8f16, v4f32]>;
26952696
def VOP_V16F32_V8F16_V8F16_V16F32 : VOPProfile <[v16f32, v8f16, v8f16, v16f32]>;

‎llvm/lib/Target/AMDGPU/VOP3PInstructions.td

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,8 @@ def VOPProfileSMFMAC_F32_32X32X32_F8 : VOPProfileSMFMAC<VOP_V16F32_V2I32_V4I32_
647647
def VOPProfileSMFMAC_I32_16X16X128_I8 : VOPProfileSMFMAC<VOP_V4I32_V4I32_V8I32_I32, AVDst_128, AVSrc_128, AVSrc_256>;
648648
def VOPProfileSMFMAC_I32_32X32X64_I8 : VOPProfileSMFMAC<VOP_V16I32_V4I32_V8I32_I32, AVDst_512, AVSrc_128, AVSrc_256>;
649649

650+
def VOPProfileSMFMAC_F32_16X16X128_F8 : VOPProfileSMFMAC<VOP_V4F32_V4I32_V8I32_I32, AVDst_128, AVSrc_128, AVSrc_256>;
651+
650652
def VOPProfileMAI_F32_V8F16_X32 : VOPProfileMAI<VOP_V4F32_V8F16_V8F16_V4F32, AISrc_128_f32, ADst_128, AVSrc_128>;
651653
def VOPProfileMAI_F32_V8F16_X32_VCD : VOPProfileMAI<VOP_V4F32_V8F16_V8F16_V4F32, VISrc_128_f32, VDst_128, AVSrc_128>;
652654
def VOPProfileMAI_F32_V8F16_X16 : VOPProfileMAI<VOP_V16F32_V8F16_V8F16_V16F32, AISrc_512_f32, ADst_512, AVSrc_128>;
@@ -1063,6 +1065,7 @@ defm V_SMFMAC_F32_16X16X64_BF16 : SMFMACInst<"v_smfmac_f32_16x16x64_bf16",
10631065
defm V_SMFMAC_F32_32X32X32_BF16 : SMFMACInst<"v_smfmac_f32_32x32x32_bf16", "F32_32X32X32_BF16", int_amdgcn_smfmac_f32_32x32x32_bf16>;
10641066
defm V_SMFMAC_I32_16X16X128_I8 : SMFMACInst<"v_smfmac_i32_16x16x128_i8", "I32_16X16X128_I8", int_amdgcn_smfmac_i32_16x16x128_i8>;
10651067
defm V_SMFMAC_I32_32X32X64_I8 : SMFMACInst<"v_smfmac_i32_32x32x64_i8", "I32_32X32X64_I8", int_amdgcn_smfmac_i32_32x32x64_i8>;
1068+
defm V_SMFMAC_F32_16X16X128_BF8_BF8 : SMFMACInst<"v_smfmac_f32_16x16x128_bf8_bf8", "F32_16X16X128_F8", int_amdgcn_smfmac_f32_16x16x128_bf8_bf8>;
10661069
}
10671070

10681071
def MAIInstInfoTable : GenericTable {
@@ -2168,6 +2171,8 @@ defm V_SMFMAC_F32_32X32X32_BF16 : VOP3P_Real_SMFMAC <0x46, "v_smfmac_f32_32x3
21682171
defm V_SMFMAC_I32_16X16X128_I8 : VOP3P_Real_SMFMAC <0x3a, "v_smfmac_i32_16x16x128i8">;
21692172
defm V_SMFMAC_I32_32X32X64_I8 : VOP3P_Real_SMFMAC <0x47, "v_smfmac_i32_32x32x64i8">;
21702173

2174+
defm V_SMFMAC_F32_16X16X128_BF8_BF8 : VOP3P_Real_SMFMAC <0x3b, "v_smfmac_f32_16x16x128bf8bf8">;
2175+
21712176
defm V_PK_FMA_F32 : VOP3P_Real_vi <0x30>;
21722177
defm V_PK_MUL_F32 : VOP3P_Real_vi <0x31>;
21732178
defm V_PK_ADD_F32 : VOP3P_Real_vi <0x32>;

‎llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,14 @@ define amdgpu_kernel void @smfmac_i32_16x16x128_i8(<4 x i32> %arg0, <8 x i32> %a
368368
ret void
369369
}
370370

371+
declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32, i32)
372+
373+
; CHECK: DIVERGENT: %result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 1, i32 2)
374+
define amdgpu_kernel void @smfmac_f32_16x16x128_bf8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, ptr addrspace(1) %out) {
375+
%result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 1, i32 2)
376+
store <4 x float> %result, ptr addrspace(1) %out
377+
ret void
378+
}
371379

372380
declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #1
373381
declare i32 @llvm.amdgcn.permlane16.i32(i32, i32, i32, i32, i1, i1) #1

‎llvm/test/CodeGen/AMDGPU/llvm.amdgcn.smfmac.gfx950.ll

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2024,4 +2024,217 @@ define <16 x i32> @test_smfmac_i32_32x32x64_i8__sgpr(<4 x i32> inreg %arg0, <8 x
20242024
ret <16 x i32> %result
20252025
}
20262026

2027+
; --------------------------------------------------------------------
2028+
; llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8
2029+
; --------------------------------------------------------------------
2030+
2031+
declare <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32>, <8 x i32>, <4 x float>, i32, i32 immarg, i32 immarg)
2032+
2033+
define amdgpu_kernel void @test_smfmac_f32_16x16x128_bf8_bf8__vgpr(ptr addrspace(1) %arg, <4 x i32> %a, <8 x i32> %b, i32 %idx) #0 {
2034+
; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
2035+
; SDAG: ; %bb.0: ; %bb
2036+
; SDAG-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
2037+
; SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0
2038+
; SDAG-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34
2039+
; SDAG-NEXT: v_mov_b32_e32 v16, 0
2040+
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
2041+
; SDAG-NEXT: global_load_dwordx4 v[8:11], v0, s[2:3]
2042+
; SDAG-NEXT: s_load_dword s16, s[0:1], 0x64
2043+
; SDAG-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54
2044+
; SDAG-NEXT: v_mov_b32_e32 v12, s4
2045+
; SDAG-NEXT: v_mov_b32_e32 v13, s5
2046+
; SDAG-NEXT: v_mov_b32_e32 v14, s6
2047+
; SDAG-NEXT: v_mov_b32_e32 v15, s7
2048+
; SDAG-NEXT: v_mov_b32_e32 v0, s8
2049+
; SDAG-NEXT: v_mov_b32_e32 v1, s9
2050+
; SDAG-NEXT: v_mov_b32_e32 v2, s10
2051+
; SDAG-NEXT: v_mov_b32_e32 v3, s11
2052+
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
2053+
; SDAG-NEXT: v_mov_b32_e32 v4, s12
2054+
; SDAG-NEXT: v_mov_b32_e32 v5, s13
2055+
; SDAG-NEXT: v_mov_b32_e32 v6, s14
2056+
; SDAG-NEXT: v_mov_b32_e32 v7, s15
2057+
; SDAG-NEXT: v_mov_b32_e32 v17, s16
2058+
; SDAG-NEXT: s_waitcnt vmcnt(0)
2059+
; SDAG-NEXT: s_nop 0
2060+
; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v17 cbsz:1 abid:2
2061+
; SDAG-NEXT: s_nop 6
2062+
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3]
2063+
; SDAG-NEXT: s_endpgm
2064+
;
2065+
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__vgpr:
2066+
; GISEL: ; %bb.0: ; %bb
2067+
; GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
2068+
; GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
2069+
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
2070+
; GISEL-NEXT: global_load_dwordx4 v[8:11], v0, s[2:3]
2071+
; GISEL-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34
2072+
; GISEL-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54
2073+
; GISEL-NEXT: s_load_dword s16, s[0:1], 0x64
2074+
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
2075+
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[6:7]
2076+
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
2077+
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
2078+
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
2079+
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
2080+
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
2081+
; GISEL-NEXT: v_mov_b32_e32 v16, s16
2082+
; GISEL-NEXT: s_waitcnt vmcnt(0)
2083+
; GISEL-NEXT: s_nop 0
2084+
; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[8:11], v[12:15], v[0:7], v16 cbsz:1 abid:2
2085+
; GISEL-NEXT: v_mov_b32_e32 v0, 0
2086+
; GISEL-NEXT: s_nop 5
2087+
; GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[2:3]
2088+
; GISEL-NEXT: s_endpgm
2089+
bb:
2090+
%id = call i32 @llvm.amdgcn.workitem.id.x()
2091+
%gep = getelementptr <4 x float>, ptr addrspace(1) %arg, i32 %id
2092+
%in.1 = load <4 x float>, ptr addrspace(1) %gep
2093+
%mai.1 = tail call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %a, <8 x i32> %b, <4 x float> %in.1, i32 %idx, i32 1, i32 2)
2094+
store <4 x float> %mai.1, ptr addrspace(1) %arg
2095+
ret void
2096+
}
2097+
2098+
define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {
2099+
; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8:
2100+
; SDAG: ; %bb.0:
2101+
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2102+
; SDAG-NEXT: v_accvgpr_write_b32 a0, v12
2103+
; SDAG-NEXT: v_accvgpr_write_b32 a1, v13
2104+
; SDAG-NEXT: v_accvgpr_write_b32 a2, v14
2105+
; SDAG-NEXT: v_accvgpr_write_b32 a3, v15
2106+
; SDAG-NEXT: s_nop 1
2107+
; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16
2108+
; SDAG-NEXT: s_nop 6
2109+
; SDAG-NEXT: v_accvgpr_read_b32 v0, a0
2110+
; SDAG-NEXT: v_accvgpr_read_b32 v1, a1
2111+
; SDAG-NEXT: v_accvgpr_read_b32 v2, a2
2112+
; SDAG-NEXT: v_accvgpr_read_b32 v3, a3
2113+
; SDAG-NEXT: s_setpc_b64 s[30:31]
2114+
;
2115+
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8:
2116+
; GISEL: ; %bb.0:
2117+
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2118+
; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16
2119+
; GISEL-NEXT: s_nop 6
2120+
; GISEL-NEXT: v_mov_b32_e32 v0, v12
2121+
; GISEL-NEXT: v_mov_b32_e32 v1, v13
2122+
; GISEL-NEXT: v_mov_b32_e32 v2, v14
2123+
; GISEL-NEXT: v_mov_b32_e32 v3, v15
2124+
; GISEL-NEXT: s_setpc_b64 s[30:31]
2125+
%result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
2126+
ret <4 x float> %result
2127+
}
2128+
2129+
define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags0(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {
2130+
; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags0:
2131+
; SDAG: ; %bb.0:
2132+
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2133+
; SDAG-NEXT: v_accvgpr_write_b32 a0, v12
2134+
; SDAG-NEXT: v_accvgpr_write_b32 a1, v13
2135+
; SDAG-NEXT: v_accvgpr_write_b32 a2, v14
2136+
; SDAG-NEXT: v_accvgpr_write_b32 a3, v15
2137+
; SDAG-NEXT: s_nop 1
2138+
; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:1 abid:3
2139+
; SDAG-NEXT: s_nop 6
2140+
; SDAG-NEXT: v_accvgpr_read_b32 v0, a0
2141+
; SDAG-NEXT: v_accvgpr_read_b32 v1, a1
2142+
; SDAG-NEXT: v_accvgpr_read_b32 v2, a2
2143+
; SDAG-NEXT: v_accvgpr_read_b32 v3, a3
2144+
; SDAG-NEXT: s_setpc_b64 s[30:31]
2145+
;
2146+
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags0:
2147+
; GISEL: ; %bb.0:
2148+
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2149+
; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:1 abid:3
2150+
; GISEL-NEXT: s_nop 6
2151+
; GISEL-NEXT: v_mov_b32_e32 v0, v12
2152+
; GISEL-NEXT: v_mov_b32_e32 v1, v13
2153+
; GISEL-NEXT: v_mov_b32_e32 v2, v14
2154+
; GISEL-NEXT: v_mov_b32_e32 v3, v15
2155+
; GISEL-NEXT: s_setpc_b64 s[30:31]
2156+
%result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 1, i32 immarg 3)
2157+
ret <4 x float> %result
2158+
}
2159+
2160+
define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__flags1(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3) {
2161+
; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags1:
2162+
; SDAG: ; %bb.0:
2163+
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2164+
; SDAG-NEXT: v_accvgpr_write_b32 a0, v12
2165+
; SDAG-NEXT: v_accvgpr_write_b32 a1, v13
2166+
; SDAG-NEXT: v_accvgpr_write_b32 a2, v14
2167+
; SDAG-NEXT: v_accvgpr_write_b32 a3, v15
2168+
; SDAG-NEXT: s_nop 1
2169+
; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[0:3], v[4:11], v16 cbsz:3 abid:1
2170+
; SDAG-NEXT: s_nop 6
2171+
; SDAG-NEXT: v_accvgpr_read_b32 v0, a0
2172+
; SDAG-NEXT: v_accvgpr_read_b32 v1, a1
2173+
; SDAG-NEXT: v_accvgpr_read_b32 v2, a2
2174+
; SDAG-NEXT: v_accvgpr_read_b32 v3, a3
2175+
; SDAG-NEXT: s_setpc_b64 s[30:31]
2176+
;
2177+
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__flags1:
2178+
; GISEL: ; %bb.0:
2179+
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2180+
; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[12:15], v[0:3], v[4:11], v16 cbsz:3 abid:1
2181+
; GISEL-NEXT: s_nop 6
2182+
; GISEL-NEXT: v_mov_b32_e32 v0, v12
2183+
; GISEL-NEXT: v_mov_b32_e32 v1, v13
2184+
; GISEL-NEXT: v_mov_b32_e32 v2, v14
2185+
; GISEL-NEXT: v_mov_b32_e32 v3, v15
2186+
; GISEL-NEXT: s_setpc_b64 s[30:31]
2187+
%result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 3, i32 immarg 1)
2188+
ret <4 x float> %result
2189+
}
2190+
2191+
define <4 x float> @test_smfmac_f32_16x16x128_bf8_bf8__sgpr(<4 x i32> inreg %arg0, <8 x i32> inreg %arg1, <4 x float> inreg %arg2, i32 inreg %arg3) {
2192+
; SDAG-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__sgpr:
2193+
; SDAG: ; %bb.0:
2194+
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2195+
; SDAG-NEXT: v_mov_b32_e32 v8, s0
2196+
; SDAG-NEXT: v_mov_b32_e32 v9, s1
2197+
; SDAG-NEXT: v_mov_b32_e32 v10, s2
2198+
; SDAG-NEXT: v_mov_b32_e32 v11, s3
2199+
; SDAG-NEXT: v_mov_b32_e32 v0, s4
2200+
; SDAG-NEXT: v_mov_b32_e32 v1, s5
2201+
; SDAG-NEXT: v_mov_b32_e32 v2, s6
2202+
; SDAG-NEXT: v_mov_b32_e32 v3, s7
2203+
; SDAG-NEXT: v_mov_b32_e32 v4, s8
2204+
; SDAG-NEXT: v_mov_b32_e32 v5, s9
2205+
; SDAG-NEXT: v_mov_b32_e32 v6, s10
2206+
; SDAG-NEXT: v_mov_b32_e32 v7, s11
2207+
; SDAG-NEXT: v_accvgpr_write_b32 a0, s12
2208+
; SDAG-NEXT: v_accvgpr_write_b32 a1, s13
2209+
; SDAG-NEXT: v_accvgpr_write_b32 a2, s14
2210+
; SDAG-NEXT: v_accvgpr_write_b32 a3, s15
2211+
; SDAG-NEXT: v_mov_b32_e32 v12, s16
2212+
; SDAG-NEXT: s_nop 1
2213+
; SDAG-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 a[0:3], v[8:11], v[0:7], v12
2214+
; SDAG-NEXT: s_nop 6
2215+
; SDAG-NEXT: v_accvgpr_read_b32 v0, a0
2216+
; SDAG-NEXT: v_accvgpr_read_b32 v1, a1
2217+
; SDAG-NEXT: v_accvgpr_read_b32 v2, a2
2218+
; SDAG-NEXT: v_accvgpr_read_b32 v3, a3
2219+
; SDAG-NEXT: s_setpc_b64 s[30:31]
2220+
;
2221+
; GISEL-LABEL: test_smfmac_f32_16x16x128_bf8_bf8__sgpr:
2222+
; GISEL: ; %bb.0:
2223+
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2224+
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[2:3]
2225+
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
2226+
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
2227+
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
2228+
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
2229+
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
2230+
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
2231+
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
2232+
; GISEL-NEXT: v_mov_b32_e32 v16, s16
2233+
; GISEL-NEXT: s_nop 1
2234+
; GISEL-NEXT: v_smfmac_f32_16x16x128_bf8_bf8 v[0:3], v[12:15], v[4:11], v16
2235+
; GISEL-NEXT: s_setpc_b64 s[30:31]
2236+
%result = call <4 x float> @llvm.amdgcn.smfmac.f32.16x16x128.bf8.bf8(<4 x i32> %arg0, <8 x i32> %arg1, <4 x float> %arg2, i32 %arg3, i32 immarg 0, i32 immarg 0)
2237+
ret <4 x float> %result
2238+
}
2239+
20272240
attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }

0 commit comments

Comments
 (0)