used isXNACKEnabled subtarget option in the PatFrag to optimize the selection.

Author: cdevadas

llvm/lib/Target/AMDGPU/SMInstructions.td

@@ -867,103 +867,60 @@ def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;
 def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">;
 
 class SMRDAlignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{
+  // Ignore the alignment check if XNACK support is disabled.
+  if (!Subtarget->isXNACKEnabled())
+    return true;
+
   // Returns true if it is a naturally aligned multi-dword load.
   LoadSDNode *Ld = cast<LoadSDNode>(N);
   unsigned Size = Ld->getMemoryVT().getStoreSize();
-  return (Size <= 4) || (Ld->getAlign().value() >= PowerOf2Ceil(Size));
+  return Size <= 4 || Ld->getAlign().value() >= Size;
 }]> {
   let GISelPredicateCode = [{
-    auto &Ld = cast<GLoad>(MI);
-    TypeSize Size = Ld.getMMO().getSize().getValue();
-    return (Size <= 4) || (Ld.getMMO().getAlign().value() >= PowerOf2Ceil(Size));
+  if (!Subtarget->isXNACKEnabled())
+    return true;
+
+  auto &Ld = cast<GLoad>(MI);
+  TypeSize Size = Ld.getMMO().getSize().getValue();
+  return Size <= 4 || Ld.getMMO().getAlign().value() >= Size;
   }];
 }
 
 class SMRDUnalignedLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{
+  // Do the alignment check if XNACK support is enabled.
+  if (!Subtarget->isXNACKEnabled())
+    return false;
+
   // Returns true if it is an under aligned multi-dword load.
   LoadSDNode *Ld = cast<LoadSDNode>(N);
   unsigned Size = Ld->getMemoryVT().getStoreSize();
-  return (Size > 4) && (Ld->getAlign().value() < PowerOf2Ceil(Size));
+  return Size > 4 && (Ld->getAlign().value() < Size);
 }]> {
   let GISelPredicateCode = [{
-    auto &Ld = cast<GLoad>(MI);
-    TypeSize Size = Ld.getMMO().getSize().getValue();
-    return (Size > 4) && (Ld.getMMO().getAlign().value() < PowerOf2Ceil(Size));
+  if (!Subtarget->isXNACKEnabled())
+    return false;
+
+  auto &Ld = cast<GLoad>(MI);
+  TypeSize Size = Ld.getMMO().getSize().getValue();
+  return Size > 4 && (Ld.getMMO().getAlign().value() < Size);
   }];
 }
 
-def alignedmultidwordload : SMRDAlignedLoadPat<smrd_load>;
-def unalignedmultidwordload : SMRDUnalignedLoadPat<smrd_load>;
-
-multiclass SMRD_Align_Pattern <string Instr, ValueType vt> {
-
-  // 1. IMM offset
-  def : GCNPat <
-    (alignedmultidwordload (SMRDImm i64:$sbase, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))> {
-    let OtherPredicates = [isGFX8Plus];
-  }
-  def : GCNPat <
-    (unalignedmultidwordload (SMRDImm i64:$sbase, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") $sbase, $offset, 0))> {
-    let OtherPredicates = [isGFX8Plus];
-  }
-
-  // 2. SGPR offset
-  def : GCNPat <
-    (alignedmultidwordload (SMRDSgpr i64:$sbase, i32:$soffset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))> {
-    let OtherPredicates = [isGFX8Only];
-  }
-  def : GCNPat <
-    (unalignedmultidwordload (SMRDSgpr i64:$sbase, i32:$soffset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_ec") $sbase, $soffset, 0))> {
-    let OtherPredicates = [isGFX8Only];
-  }
-  def : GCNPat <
-    (alignedmultidwordload (SMRDSgpr i64:$sbase, i32:$soffset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))> {
-    let OtherPredicates = [isGFX9Plus];
-  }
-  def : GCNPat <
-    (unalignedmultidwordload (SMRDSgpr i64:$sbase, i32:$soffset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, 0, 0))> {
-    let OtherPredicates = [isGFX9Plus];
-  }
-
-  // 3. SGPR+IMM offset
-  def : GCNPat <
-    (alignedmultidwordload (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> {
-    let OtherPredicates = [isGFX9Plus];
-  }
-  def : GCNPat <
-    (unalignedmultidwordload (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, $offset, 0))> {
-    let OtherPredicates = [isGFX9Plus];
-  }
-
-  // 4. No offset
-  def : GCNPat <
-    (vt (alignedmultidwordload (i64 SReg_64:$sbase))),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))> {
-    let OtherPredicates = [isGFX8Plus];
-  }
-  def : GCNPat <
-    (vt (unalignedmultidwordload (i64 SReg_64:$sbase))),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") i64:$sbase, 0, 0))> {
-    let OtherPredicates = [isGFX8Plus];
-  }
-}
+def aligned_smrd_load : SMRDAlignedLoadPat<smrd_load>;
+def unaligned_smrd_load : SMRDUnalignedLoadPat<smrd_load>;
 
 multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> {
 
   // 1. IMM offset
   def : GCNPat <
-    (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))> {
-    let OtherPredicates = [isGFX6GFX7];
-  }
+    (aligned_smrd_load (SMRDImm i64:$sbase, i32:$offset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
+  >;
+  if !gt(vt.Size, 32) then
+    def : GCNPat <
+      (unaligned_smrd_load (SMRDImm i64:$sbase, i32:$offset)),
+      (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") $sbase, $offset, 0))
+    >;
 
   // 2. 32-bit IMM offset on CI
   if immci then def : GCNPat <
@@ -974,19 +931,49 @@ multiclass SMRD_Pattern <string Instr, ValueType vt, bit immci = true> {
 
   // 3. SGPR offset
   def : GCNPat <
-    (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
+    (aligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
     (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))> {
-    let OtherPredicates = [isGFX6GFX7];
+    let OtherPredicates = [isNotGFX9Plus];
   }
-
-  // 4. No offset
   def : GCNPat <
-    (vt (smrd_load (i64 SReg_64:$sbase))),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))> {
-    let OtherPredicates = [isGFX6GFX7];
+    (aligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))> {
+    let OtherPredicates = [isGFX9Plus];
+  }
+  if !gt(vt.Size, 32) then {
+    def : GCNPat <
+      (unaligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
+      (vt (!cast<SM_Pseudo>(Instr#"_SGPR_ec") $sbase, $soffset, 0))> {
+      let OtherPredicates = [isNotGFX9Plus];
+    }
+    def : GCNPat <
+      (unaligned_smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
+      (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, 0, 0))> {
+      let OtherPredicates = [isGFX9Plus];
+    }
   }
 
-  defm : SMRD_Align_Pattern<Instr, vt>;
+  // 4. SGPR+IMM offset
+  def : GCNPat <
+    (aligned_smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> {
+    let OtherPredicates = [isGFX9Plus];
+  }
+  if !gt(vt.Size, 32) then
+    def : GCNPat <
+      (unaligned_smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
+      (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM_ec") $sbase, $soffset, $offset, 0))> {
+      let OtherPredicates = [isGFX9Plus];
+    }
+
+  // 5. No offset
+  def : GCNPat <
+    (vt (aligned_smrd_load (i64 SReg_64:$sbase))),
+    (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))>;
+  if !gt(vt.Size, 32) then
+    def : GCNPat <
+      (vt (unaligned_smrd_load (i64 SReg_64:$sbase))),
+      (vt (!cast<SM_Pseudo>(Instr#"_IMM_ec") i64:$sbase, 0, 0))>;
 }
 
 multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {

llvm/test/CodeGen/AMDGPU/GlobalISel/addsubu64.ll

@@ -7,11 +7,11 @@ define amdgpu_kernel void @s_add_u64(ptr addrspace(1) %out, i64 %a, i64 %b) {
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
-; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x34
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_add_u32 s0, s6, s2
-; GFX11-NEXT:    s_addc_u32 s1, s7, s3
+; GFX11-NEXT:    s_add_u32 s0, s6, s0
+; GFX11-NEXT:    s_addc_u32 s1, s7, s1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
@@ -23,10 +23,10 @@ define amdgpu_kernel void @s_add_u64(ptr addrspace(1) %out, i64 %a, i64 %b) {
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
-; GFX12-NEXT:    s_load_b64 s[2:3], s[0:1], 0x34
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[6:7], s[2:3]
+; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[6:7], s[0:1]
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
@@ -59,11 +59,11 @@ define amdgpu_kernel void @s_sub_u64(ptr addrspace(1) %out, i64 %a, i64 %b) {
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
-; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x34
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_sub_u32 s0, s6, s2
-; GFX11-NEXT:    s_subb_u32 s1, s7, s3
+; GFX11-NEXT:    s_sub_u32 s0, s6, s0
+; GFX11-NEXT:    s_subb_u32 s1, s7, s1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
@@ -75,10 +75,10 @@ define amdgpu_kernel void @s_sub_u64(ptr addrspace(1) %out, i64 %a, i64 %b) {
 ; GFX12:       ; %bb.0: ; %entry
 ; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
-; GFX12-NEXT:    s_load_b64 s[2:3], s[0:1], 0x34
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
 ; GFX12-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_sub_nc_u64 s[0:1], s[6:7], s[2:3]
+; GFX12-NEXT:    s_sub_nc_u64 s[0:1], s[6:7], s[0:1]
 ; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[4:5]

llvm/test/CodeGen/AMDGPU/GlobalISel/bool-legalization.ll

@@ -131,9 +131,9 @@ define amdgpu_kernel void @brcond_sgpr_trunc_and(i32 %cond0, i32 %cond1) {
 ;
 ; WAVE32-LABEL: brcond_sgpr_trunc_and:
 ; WAVE32:       ; %bb.0: ; %entry
-; WAVE32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
+; WAVE32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; WAVE32-NEXT:    s_waitcnt lgkmcnt(0)
-; WAVE32-NEXT:    s_and_b32 s0, s2, s3
+; WAVE32-NEXT:    s_and_b32 s0, s0, s1
 ; WAVE32-NEXT:    s_xor_b32 s0, s0, 1
 ; WAVE32-NEXT:    s_and_b32 s0, s0, 1
 ; WAVE32-NEXT:    s_cmp_lg_u32 s0, 0

llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll

@@ -1401,20 +1401,20 @@ define amdgpu_kernel void @cvt_ubyte0_or_multiuse(ptr addrspace(1) %in, ptr addr
 ;
 ; VI-LABEL: cvt_ubyte0_or_multiuse:
 ; VI:       ; %bb.0: ; %bb
-; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s4
-; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_mov_b32_e32 v0, s0
+; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, v0, v2
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[0:1]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_or_b32_e32 v0, 0x80000001, v0
 ; VI-NEXT:    v_cvt_f32_ubyte0_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
 ; VI-NEXT:    v_add_f32_e32 v2, v0, v1
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 bb: