[X86] For minsize memset/memcpy, use byte or double-word accesses

AZero13 · AZero13 · commit bca34564253b · 2024-10-03T12:18:03.000-04:00
repstosb and repstosd are the same size, but stosd is only done for 0 because the process of multiplying the constant so that it is copied across the bytes of the 32-bit number adds extra instructions that cause the size to increase. For 0, repstosb and repstosd are the same size, but stosd is only done for 0 because the process of multiplying the constant so that it is copied across the bytes of the 32-bit number adds extra instructions that cause the size to increase. For 0, we do not need to do that at all.

For memcpy, the same goes, and as a result the minsize check was moved ahead because a jmp to memcpy encoded takes more bytes than repmovsb.
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -98,6 +98,14 @@ static SDValue emitRepstos(const X86Subtarget &Subtarget, SelectionDAG &DAG,
   return DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
 }
 
+/// Emit a single REP STOSB instruction for a particular constant size.
+static SDValue emitRepstosB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
+                            const SDLoc &dl, SDValue Chain, SDValue Dst,
+                            SDValue Val, uint64_t Size) {
+  return emitRepstos(Subtarget, DAG, dl, Chain, Dst, Val,
+                     DAG.getIntPtrConstant(Size, dl), MVT::i8);
+}
+
 /// Returns a REP STOS instruction, possibly with a few load/stores to implement
 /// a constant size memory set. In some cases where we know REP MOVS is
 /// inefficient we return an empty SDValue so the calling code can either
@@ -109,6 +117,26 @@ static SDValue emitConstantSizeRepstos(SelectionDAG &DAG,
                                        EVT SizeVT, Align Alignment,
                                        bool isVolatile, bool AlwaysInline,
                                        MachinePointerInfo DstPtrInfo) {
+  /// In case we optimize for size, we use repstosb even if it's less efficient
+  /// so we can save the loads/stores of the leftover.
+  if (DAG.getMachineFunction().getFunction().hasMinSize()) {
+    if (auto *ValC = dyn_cast<ConstantSDNode>(Val)) {
+      // Special case 0 because otherwise we get large literals,
+      // which causes larger encoding.
+      if ((Size & 31) == 0 && (ValC->getZExtValue() & 255) == 0) {
+        MVT BlockType = MVT::i32;
+        const uint64_t BlockBits = BlockType.getSizeInBits();
+        const uint64_t BlockBytes = BlockBits / 8;
+        const uint64_t BlockCount = Size / BlockBytes;
+
+        Val = DAG.getConstant(0, dl, BlockType);
+        // repstosd is same size as repstosb
+        return emitRepstos(Subtarget, DAG, dl, Chain, Dst, Val,
+                           DAG.getIntPtrConstant(BlockCount, dl), BlockType);
+      }
+    }
+    return emitRepstosB(Subtarget, DAG, dl, Chain, Dst, Val, Size);
+  }
 
   if (Size > Subtarget.getMaxInlineSizeThreshold())
     return SDValue();
@@ -230,6 +258,10 @@ static SDValue emitConstantSizeRepmov(
     SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
     Align Alignment, bool isVolatile, bool AlwaysInline,
     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) {
+  /// In case we optimize for size, we use repmovsb even if it's less efficient
+  /// so we can save the loads/stores of the leftover.
+  if (DAG.getMachineFunction().getFunction().hasMinSize())
+    return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
 
   /// TODO: Revisit next line: big copy with ERMSB on march >= haswell are very
   /// efficient.
@@ -260,11 +292,6 @@ static SDValue emitConstantSizeRepmov(
 
   assert(BytesLeft && "We have leftover at this point");
 
-  /// In case we optimize for size, we use repmovsb even if it's less efficient
-  /// so we can save the loads/stores of the leftover.
-  if (DAG.getMachineFunction().getFunction().hasMinSize())
-    return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
-
   // Handle the last 1 - 7 bytes.
   SmallVector<SDValue, 4> Results;
   Results.push_back(RepMovs);
diff --git a/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll b/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll
@@ -78,9 +78,9 @@ define void @test2(ptr nocapture %x) nounwind minsize {
 ; NOFAST32-NEXT:    pushl %esi
 ; NOFAST32-NEXT:    subl $4100, %esp # imm = 0x1004
 ; NOFAST32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; NOFAST32-NEXT:    movl $1024, %ecx # imm = 0x400
+; NOFAST32-NEXT:    movl $4096, %ecx # imm = 0x1000
 ; NOFAST32-NEXT:    movl %esp, %edi
-; NOFAST32-NEXT:    rep;movsl (%esi), %es:(%edi)
+; NOFAST32-NEXT:    rep;movsb (%esi), %es:(%edi)
 ; NOFAST32-NEXT:    calll foo@PLT
 ; NOFAST32-NEXT:    addl $4100, %esp # imm = 0x1004
 ; NOFAST32-NEXT:    popl %esi
@@ -106,9 +106,9 @@ define void @test2(ptr nocapture %x) nounwind minsize {
 ; NOFAST:       # %bb.0:
 ; NOFAST-NEXT:    subq $4104, %rsp # imm = 0x1008
 ; NOFAST-NEXT:    movq %rdi, %rsi
-; NOFAST-NEXT:    movl $512, %ecx # imm = 0x200
+; NOFAST-NEXT:    movl $4096, %ecx # imm = 0x1000
 ; NOFAST-NEXT:    movq %rsp, %rdi
-; NOFAST-NEXT:    rep;movsq (%rsi), %es:(%rdi)
+; NOFAST-NEXT:    rep;movsb (%rsi), %es:(%rdi)
 ; NOFAST-NEXT:    callq foo@PLT
 ; NOFAST-NEXT:    addq $4104, %rsp # imm = 0x1008
 ; NOFAST-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/memcpy.ll b/llvm/test/CodeGen/X86/memcpy.ll
@@ -202,14 +202,16 @@ define void @test3_minsize(ptr nocapture %A, ptr nocapture %B) nounwind minsize
 ; DARWIN-LABEL: test3_minsize:
 ; DARWIN:       ## %bb.0:
 ; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; DARWIN-NEXT:    popq %rcx
+; DARWIN-NEXT:    rep;movsb (%rsi), %es:(%rdi)
+; DARWIN-NEXT:    retq
 ;
 ; LINUX-LABEL: test3_minsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
-; LINUX-NEXT:    popq %rdx
-; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
+; LINUX-NEXT:    popq %rcx
+; LINUX-NEXT:    rep;movsb (%rsi), %es:(%rdi)
+; LINUX-NEXT:    retq
 ;
 ; LINUX-SKL-LABEL: test3_minsize:
 ; LINUX-SKL:       # %bb.0:
@@ -249,14 +251,16 @@ define void @test3_minsize_optsize(ptr nocapture %A, ptr nocapture %B) nounwind
 ; DARWIN-LABEL: test3_minsize_optsize:
 ; DARWIN:       ## %bb.0:
 ; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; DARWIN-NEXT:    popq %rcx
+; DARWIN-NEXT:    rep;movsb (%rsi), %es:(%rdi)
+; DARWIN-NEXT:    retq
 ;
 ; LINUX-LABEL: test3_minsize_optsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
-; LINUX-NEXT:    popq %rdx
-; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
+; LINUX-NEXT:    popq %rcx
+; LINUX-NEXT:    rep;movsb (%rsi), %es:(%rdi)
+; LINUX-NEXT:    retq
 ;
 ; LINUX-SKL-LABEL: test3_minsize_optsize:
 ; LINUX-SKL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/memset-minsize.ll b/llvm/test/CodeGen/X86/memset-minsize.ll
@@ -27,11 +27,9 @@ entry:
 define void @medium_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: medium_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $512, %edx # imm = 0x200
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movl $128, %ecx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 512, i1 false)
@@ -41,11 +39,9 @@ entry:
 define void @large_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: large_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $4096, %edx # imm = 0x1000
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movl $1024, %ecx # imm = 0x400
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 4096, i1 false)
@@ -55,11 +51,9 @@ entry:
 define void @huge_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: huge_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $8192, %edx # imm = 0x2000
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movl $2048, %ecx # imm = 0x800
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 8192, i1 false)
@@ -69,11 +63,9 @@ entry:
 define void @odd_length_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: odd_length_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $255, %edx
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    movl $255, %ecx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosb %al, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 255, i1 false)
@@ -83,11 +75,10 @@ entry:
 define void @align_1_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: align_1_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $256, %edx # imm = 0x100
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    pushq $64
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 1 %ptr, i8 0, i32 256, i1 false)
@@ -97,11 +88,10 @@ entry:
 define void @align_2_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: align_2_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $256, %edx # imm = 0x100
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    pushq $64
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 2 %ptr, i8 0, i32 256, i1 false)
@@ -111,11 +101,10 @@ entry:
 define void @align_4_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: align_4_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $256, %edx # imm = 0x100
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    pushq $64
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 256, i1 false)
@@ -125,11 +114,10 @@ entry:
 define void @align_8_memset_to_rep_stos(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: align_8_memset_to_rep_stos:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movl $256, %edx # imm = 0x100
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    callq memset@PLT
-; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    pushq $64
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i32(ptr align 8 %ptr, i8 0, i32 256, i1 false)
@@ -139,10 +127,10 @@ entry:
 define void @small_memset_to_rep_stos_64(ptr %ptr) minsize nounwind {
 ; CHECK-LABEL: small_memset_to_rep_stos_64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq $16
+; CHECK-NEXT:    pushq $32
 ; CHECK-NEXT:    popq %rcx
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    rep;stosq %rax, %es:(%rdi)
+; CHECK-NEXT:    rep;stosl %eax, %es:(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   call void @llvm.memset.p0.i64(ptr align 8 %ptr, i8 0, i64 128, i1 false)
diff --git a/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll b/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
