[AArch64] Fix SVE scalar fcopysign lowering without neon. #129787

davemgreen · 2025-03-04T22:22:10Z

Without this we can try to generate invalid instructions or create illegal types. This patch generates a SVE fcopysign instead and use its lowering. BF16 is left out of the moment as it doesn't lower successfully (but could use the same code as fp16).

llvmbot · 2025-03-04T22:22:47Z

@llvm/pr-subscribers-backend-aarch64

Author: David Green (davemgreen)

Changes

Without this we can try to generate invalid instructions or create illegal types. Generate a SVE fcopysign instead and use it's lowering. BF16 is left out of the moment as it doesn't lower successfully (but could use the same code as fp16).

Full diff: https://github.com/llvm/llvm-project/pull/129787.diff

2 Files Affected:

(modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+29)
(modified) llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll (+55-84)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2dca8c0da4756..13fa505d609bd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -10686,6 +10686,35 @@ SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
     return convertFromScalableVector(DAG, VT, Res);
   }
 
+  // With SVE, but without Neon, extend the scalars to scalable vectors and use
+  // a SVE FCOPYSIGN.
+  if (!VT.isVector() && VT.isSimple() && !Subtarget->isNeonAvailable()) {
+    EVT SVT;
+    switch (VT.getSimpleVT().SimpleTy) {
+    case MVT::f16:
+      SVT = MVT::nxv8f16;
+      break;
+    case MVT::f32:
+      SVT = MVT::nxv4f32;
+      break;
+    case MVT::f64:
+      SVT = MVT::nxv2f64;
+      break;
+    default:
+      return SDValue();
+    }
+
+    SDValue Ins1 =
+        DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, SVT, DAG.getUNDEF(SVT), In1,
+                    DAG.getConstant(0, DL, MVT::i64));
+    SDValue Ins2 =
+        DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, SVT, DAG.getUNDEF(SVT), In2,
+                    DAG.getConstant(0, DL, MVT::i64));
+    SDValue FCS = DAG.getNode(ISD::FCOPYSIGN, DL, SVT, Ins1, Ins2);
+    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, FCS,
+                       DAG.getConstant(0, DL, MVT::i64));
+  }
+
   auto BitCast = [this](EVT VT, SDValue Op, SelectionDAG &DAG) {
     if (VT.isScalableVector())
       return getSVESafeBitCast(VT, Op, DAG);
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
index 238c124b7cb06..79921e25caf53 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll
@@ -11,32 +11,21 @@ target triple = "aarch64-unknown-linux-gnu"
 define void @test_copysign_f16(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_f16:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    adrp x8, .LCPI0_0
+; SVE-NEXT:    ldr h0, [x1]
 ; SVE-NEXT:    ldr h1, [x0]
-; SVE-NEXT:    ldr h2, [x1]
-; SVE-NEXT:    ldr q0, [x8, :lo12:.LCPI0_0]
-; SVE-NEXT:    adrp x8, .LCPI0_1
-; SVE-NEXT:    ldr q4, [x8, :lo12:.LCPI0_1]
-; SVE-NEXT:    mov z3.d, z0.d
-; SVE-NEXT:    fmov s0, s1
-; SVE-NEXT:    fmov s3, s2
-; SVE-NEXT:    bif v0.16b, v3.16b, v4.16b
+; SVE-NEXT:    and z0.h, z0.h, #0x8000
+; SVE-NEXT:    and z1.h, z1.h, #0x7fff
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str h0, [x0]
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_f16:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    adrp x8, .LCPI0_0
-; SVE2-NEXT:    ldr h1, [x0]
-; SVE2-NEXT:    ldr h2, [x1]
-; SVE2-NEXT:    ldr q0, [x8, :lo12:.LCPI0_0]
-; SVE2-NEXT:    adrp x8, .LCPI0_1
-; SVE2-NEXT:    ldr q4, [x8, :lo12:.LCPI0_1]
-; SVE2-NEXT:    mov z3.d, z0.d
-; SVE2-NEXT:    fmov s0, s1
-; SVE2-NEXT:    fmov s3, s2
-; SVE2-NEXT:    bif v0.16b, v3.16b, v4.16b
-; SVE2-NEXT:    str h0, [x0]
+; SVE2-NEXT:    mov z0.h, #32767 // =0x7fff
+; SVE2-NEXT:    ldr h1, [x1]
+; SVE2-NEXT:    ldr h2, [x0]
+; SVE2-NEXT:    bsl z2.d, z2.d, z1.d, z0.d
+; SVE2-NEXT:    str h2, [x0]
 ; SVE2-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test_copysign_f16:
@@ -66,32 +55,40 @@ define void @test_copysign_f16(ptr %ap, ptr %bp) {
 define void @test_copysign_bf16(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_bf16:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    adrp x8, .LCPI1_0
-; SVE-NEXT:    ldr h1, [x0]
-; SVE-NEXT:    ldr h2, [x1]
-; SVE-NEXT:    ldr q0, [x8, :lo12:.LCPI1_0]
-; SVE-NEXT:    adrp x8, .LCPI1_1
-; SVE-NEXT:    ldr q4, [x8, :lo12:.LCPI1_1]
-; SVE-NEXT:    mov z3.d, z0.d
-; SVE-NEXT:    fmov s0, s1
-; SVE-NEXT:    fmov s3, s2
-; SVE-NEXT:    bif v0.16b, v3.16b, v4.16b
+; SVE-NEXT:    sub sp, sp, #16
+; SVE-NEXT:    .cfi_def_cfa_offset 16
+; SVE-NEXT:    ldr h0, [x0]
+; SVE-NEXT:    ldr h1, [x1]
+; SVE-NEXT:    fmov w8, s0
+; SVE-NEXT:    str h1, [sp, #12]
+; SVE-NEXT:    ldrb w9, [sp, #13]
+; SVE-NEXT:    and w8, w8, #0x7fff
+; SVE-NEXT:    tst w9, #0x80
+; SVE-NEXT:    fmov s0, w8
+; SVE-NEXT:    eor w8, w8, #0x8000
+; SVE-NEXT:    fmov s1, w8
+; SVE-NEXT:    fcsel h0, h1, h0, ne
 ; SVE-NEXT:    str h0, [x0]
+; SVE-NEXT:    add sp, sp, #16
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_bf16:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    adrp x8, .LCPI1_0
-; SVE2-NEXT:    ldr h1, [x0]
-; SVE2-NEXT:    ldr h2, [x1]
-; SVE2-NEXT:    ldr q0, [x8, :lo12:.LCPI1_0]
-; SVE2-NEXT:    adrp x8, .LCPI1_1
-; SVE2-NEXT:    ldr q4, [x8, :lo12:.LCPI1_1]
-; SVE2-NEXT:    mov z3.d, z0.d
-; SVE2-NEXT:    fmov s0, s1
-; SVE2-NEXT:    fmov s3, s2
-; SVE2-NEXT:    bif v0.16b, v3.16b, v4.16b
+; SVE2-NEXT:    sub sp, sp, #16
+; SVE2-NEXT:    .cfi_def_cfa_offset 16
+; SVE2-NEXT:    ldr h0, [x0]
+; SVE2-NEXT:    ldr h1, [x1]
+; SVE2-NEXT:    fmov w8, s0
+; SVE2-NEXT:    str h1, [sp, #12]
+; SVE2-NEXT:    ldrb w9, [sp, #13]
+; SVE2-NEXT:    and w8, w8, #0x7fff
+; SVE2-NEXT:    tst w9, #0x80
+; SVE2-NEXT:    fmov s0, w8
+; SVE2-NEXT:    eor w8, w8, #0x8000
+; SVE2-NEXT:    fmov s1, w8
+; SVE2-NEXT:    fcsel h0, h1, h0, ne
 ; SVE2-NEXT:    str h0, [x0]
+; SVE2-NEXT:    add sp, sp, #16
 ; SVE2-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test_copysign_bf16:
@@ -139,32 +136,21 @@ define void @test_copysign_bf16(ptr %ap, ptr %bp) {
 define void @test_copysign_f32(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_f32:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    adrp x8, .LCPI2_0
+; SVE-NEXT:    ldr s0, [x1]
 ; SVE-NEXT:    ldr s1, [x0]
-; SVE-NEXT:    ldr s2, [x1]
-; SVE-NEXT:    ldr q0, [x8, :lo12:.LCPI2_0]
-; SVE-NEXT:    adrp x8, .LCPI2_1
-; SVE-NEXT:    ldr q4, [x8, :lo12:.LCPI2_1]
-; SVE-NEXT:    mov z3.d, z0.d
-; SVE-NEXT:    fmov s0, s1
-; SVE-NEXT:    fmov s3, s2
-; SVE-NEXT:    bif v0.16b, v3.16b, v4.16b
+; SVE-NEXT:    and z0.s, z0.s, #0x80000000
+; SVE-NEXT:    and z1.s, z1.s, #0x7fffffff
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str s0, [x0]
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_f32:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    adrp x8, .LCPI2_0
-; SVE2-NEXT:    ldr s1, [x0]
-; SVE2-NEXT:    ldr s2, [x1]
-; SVE2-NEXT:    ldr q0, [x8, :lo12:.LCPI2_0]
-; SVE2-NEXT:    adrp x8, .LCPI2_1
-; SVE2-NEXT:    ldr q4, [x8, :lo12:.LCPI2_1]
-; SVE2-NEXT:    mov z3.d, z0.d
-; SVE2-NEXT:    fmov s0, s1
-; SVE2-NEXT:    fmov s3, s2
-; SVE2-NEXT:    bif v0.16b, v3.16b, v4.16b
-; SVE2-NEXT:    str s0, [x0]
+; SVE2-NEXT:    mov z0.s, #0x7fffffff
+; SVE2-NEXT:    ldr s1, [x1]
+; SVE2-NEXT:    ldr s2, [x0]
+; SVE2-NEXT:    bsl z2.d, z2.d, z1.d, z0.d
+; SVE2-NEXT:    str s2, [x0]
 ; SVE2-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test_copysign_f32:
@@ -187,36 +173,21 @@ define void @test_copysign_f32(ptr %ap, ptr %bp) {
 define void @test_copysign_f64(ptr %ap, ptr %bp) {
 ; SVE-LABEL: test_copysign_f64:
 ; SVE:       // %bb.0:
-; SVE-NEXT:    adrp x8, .LCPI3_1
-; SVE-NEXT:    ptrue p0.d, vl2
-; SVE-NEXT:    ldr d2, [x0]
-; SVE-NEXT:    ldr q0, [x8, :lo12:.LCPI3_1]
-; SVE-NEXT:    adrp x8, .LCPI3_0
-; SVE-NEXT:    ldr d3, [x1]
-; SVE-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
-; SVE-NEXT:    fneg z0.d, p0/m, z0.d
-; SVE-NEXT:    mov z4.d, z1.d
-; SVE-NEXT:    fmov d1, d2
-; SVE-NEXT:    fmov d4, d3
-; SVE-NEXT:    bsl v0.16b, v1.16b, v4.16b
+; SVE-NEXT:    ldr d0, [x1]
+; SVE-NEXT:    ldr d1, [x0]
+; SVE-NEXT:    and z0.d, z0.d, #0x8000000000000000
+; SVE-NEXT:    and z1.d, z1.d, #0x7fffffffffffffff
+; SVE-NEXT:    orr z0.d, z1.d, z0.d
 ; SVE-NEXT:    str d0, [x0]
 ; SVE-NEXT:    ret
 ;
 ; SVE2-LABEL: test_copysign_f64:
 ; SVE2:       // %bb.0:
-; SVE2-NEXT:    adrp x8, .LCPI3_1
-; SVE2-NEXT:    ptrue p0.d, vl2
+; SVE2-NEXT:    mov z0.d, #0x7fffffffffffffff
+; SVE2-NEXT:    ldr d1, [x1]
 ; SVE2-NEXT:    ldr d2, [x0]
-; SVE2-NEXT:    ldr q0, [x8, :lo12:.LCPI3_1]
-; SVE2-NEXT:    adrp x8, .LCPI3_0
-; SVE2-NEXT:    ldr d3, [x1]
-; SVE2-NEXT:    ldr q1, [x8, :lo12:.LCPI3_0]
-; SVE2-NEXT:    fneg z0.d, p0/m, z0.d
-; SVE2-NEXT:    mov z4.d, z1.d
-; SVE2-NEXT:    fmov d1, d2
-; SVE2-NEXT:    fmov d4, d3
-; SVE2-NEXT:    bsl v0.16b, v1.16b, v4.16b
-; SVE2-NEXT:    str d0, [x0]
+; SVE2-NEXT:    bsl z2.d, z2.d, z1.d, z0.d
+; SVE2-NEXT:    str d2, [x0]
 ; SVE2-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test_copysign_f64:

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

david-arm · 2025-03-05T09:12:06Z

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll

+; SVE-NEXT:    fmov s0, w8
+; SVE-NEXT:    eor w8, w8, #0x8000
+; SVE-NEXT:    fmov s1, w8
+; SVE-NEXT:    fcsel h0, h1, h0, ne


The bf16 copysign seems to generate legal code for streaming compatible functions. Is this why you didn't need to add the bf16 to the new code in LowerFCOPYSIGN? I just want to make sure there isn't still a bug for bf16.

The combo of SVE and bf16 is not handled very well yet - it was failed to legalize when I tried using scalable vectors. The scalar version should always be safe, but it would be more efficient to use SVE instruction if they were available. The codegen should be be identical to the fp16 version, but I figured it wasn't worth bitcasting the types to fp16 and back, it sounded like a bit of a bodge. We can get the improved codegen once SVE bf16 is doing better.

paulwalker-arm · 2025-03-05T11:26:36Z

Just a FYI but I have recently refactored LowerFCOPYSIGN to simplify it somewhat whilst at the same time making it work better for bfloat types. Nothing to hold up this PR, but you can leave the bfloat side of things to me.

Without this we can try to generate invalid instructions or create illegal types. Generate a SVE fcopysign instead and use its lowering. BF16 is left out of the moment as it doesn't lower successfully (but could use the same code as fp16).

david-arm

LGTM!

davemgreen · 2025-03-05T20:17:47Z

/cherry-pick d4ab3df

llvmbot · 2025-03-05T20:23:15Z

Failed to cherry-pick: d4ab3df

https://github.com/llvm/llvm-project/actions/runs/13684753931

Please manually backport the fix and push it to your github fork. Once this is done, please create a pull request

davemgreen · 2025-03-06T06:42:07Z

/cherry-pick 4c2d1b4 d4ab3df

llvmbot · 2025-03-06T06:47:43Z

/pull-request #129997

Without this we can try to generate invalid instructions or create illegal types. This patch generates a SVE fcopysign instead and use its lowering. BF16 is left out of the moment as it doesn't lower successfully (but could use the same code as fp16).

Without this we can try to generate invalid instructions or create illegal types. This patch generates a SVE fcopysign instead and use its lowering. BF16 is left out of the moment as it doesn't lower successfully (but could use the same code as fp16). (cherry picked from commit d4ab3df)

davemgreen requested review from pawosm-arm, sdesmalen-arm, kmclaughlin-arm and david-arm March 4, 2025 22:22

llvmbot added the backend:AArch64 label Mar 4, 2025

pawosm-arm added this to the LLVM 20.X Release milestone Mar 4, 2025

github-project-automation bot added this to LLVM Release Status Mar 4, 2025

github-project-automation bot moved this to Needs Triage in LLVM Release Status Mar 4, 2025

david-arm reviewed Mar 5, 2025

View reviewed changes

davemgreen force-pushed the gh-a64-svecopysign branch from 11be9b6 to c736e41 Compare March 5, 2025 13:21

david-arm approved these changes Mar 5, 2025

View reviewed changes

github-project-automation bot moved this from Needs Triage to Needs Merge in LLVM Release Status Mar 5, 2025

davemgreen merged commit d4ab3df into llvm:main Mar 5, 2025
11 checks passed

github-project-automation bot moved this from Needs Merge to Done in LLVM Release Status Mar 5, 2025

davemgreen deleted the gh-a64-svecopysign branch March 5, 2025 17:18

llvmbot added the release:cherry-pick-failed label Mar 5, 2025

llvmbot removed the release:cherry-pick-failed label Mar 6, 2025

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[AArch64] Fix SVE scalar fcopysign lowering without neon. #129787

[AArch64] Fix SVE scalar fcopysign lowering without neon. #129787

Uh oh!

davemgreen commented Mar 4, 2025 •

edited

Loading

Uh oh!

llvmbot commented Mar 4, 2025

Uh oh!

Uh oh!

david-arm Mar 5, 2025

Uh oh!

davemgreen Mar 5, 2025

Uh oh!

paulwalker-arm commented Mar 5, 2025 •

edited

Loading

Uh oh!

david-arm left a comment

Uh oh!

Uh oh!

davemgreen commented Mar 5, 2025

Uh oh!

llvmbot commented Mar 5, 2025

Uh oh!

davemgreen commented Mar 6, 2025

Uh oh!

llvmbot commented Mar 6, 2025

Uh oh!

Uh oh!

[AArch64] Fix SVE scalar fcopysign lowering without neon. #129787

[AArch64] Fix SVE scalar fcopysign lowering without neon. #129787

Uh oh!

Conversation

davemgreen commented Mar 4, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented Mar 4, 2025

Uh oh!

Uh oh!

david-arm Mar 5, 2025

Choose a reason for hiding this comment

Uh oh!

davemgreen Mar 5, 2025

Choose a reason for hiding this comment

Uh oh!

paulwalker-arm commented Mar 5, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

david-arm left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

davemgreen commented Mar 5, 2025

Uh oh!

llvmbot commented Mar 5, 2025

Uh oh!

davemgreen commented Mar 6, 2025

Uh oh!

llvmbot commented Mar 6, 2025

Uh oh!

Uh oh!

davemgreen commented Mar 4, 2025 •

edited

Loading

paulwalker-arm commented Mar 5, 2025 •

edited

Loading