[SDAG] Check fminnum/fmaxnum for non-zero operand.

sparker-arm · sparker-arm · commit 7bff37783f72 · 2023-02-07T10:54:23.000Z
Currently, in TargetLowering, if the target does not support fminnum, we lower to fminimum if neither operand could be a NaN. But this isn't quite correct because fminnum and fminimum treat +/-0 differently; so, we need to prove that one of the operands isn't a zero, or we don't have signed zeros. Differential Revision: https://reviews.llvm.org/D143256
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4918,9 +4918,21 @@ bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
          "Floating point type expected");
 
   // If the value is a constant, we can obviously see if it is a zero or not.
-  // TODO: Add BuildVector support.
   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
     return !C->isZero();
+
+  // Return false if we find any zero in a vector.
+  if (Op->getOpcode() == ISD::BUILD_VECTOR ||
+      Op->getOpcode() == ISD::SPLAT_VECTOR) {
+    for (const SDValue &OpVal : Op->op_values()) {
+      if (OpVal.isUndef())
+        return false;
+      if (auto *C = dyn_cast<ConstantFPSDNode>(OpVal))
+        if (C->isZero())
+          return false;
+    }
+    return true;
+  }
   return false;
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -7985,14 +7985,17 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
   }
 
   // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
-  // instead if there are no NaNs.
-  if (Node->getFlags().hasNoNaNs()) {
+  // instead if there are no NaNs and there can't be an incompatiable zero
+  // compare: at least one operand isn't +/-0, or there are no signed-zeros.
+  if (Node->getFlags().hasNoNaNs() &&
+      (Node->getFlags().hasNoSignedZeros() ||
+       DAG.isKnownNeverZeroFloat(Node->getOperand(0)) ||
+       DAG.isKnownNeverZeroFloat(Node->getOperand(1)))) {
     unsigned IEEE2018Op =
         Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
-    if (isOperationLegalOrCustom(IEEE2018Op, VT)) {
+    if (isOperationLegalOrCustom(IEEE2018Op, VT))
       return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
                          Node->getOperand(1), Node->getFlags());
-    }
   }
 
   if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
diff --git a/llvm/test/CodeGen/ARM/lower-vmax.ll b/llvm/test/CodeGen/ARM/lower-vmax.ll
@@ -1,22 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=arm-eabihf -mattr=+neon < %s | FileCheck -check-prefixes=CHECK-NO_NEON %s
 ; RUN: llc -mtriple=arm-eabihf -mattr=+neon,+neonfp < %s | FileCheck -check-prefixes=CHECK-NEON %s
 
 define float @max_f32(float, float) {
-;CHECK-NEON: vmax.f32
-;CHECK-NO_NEON: vcmp.f32
-;CHECK-NO_NEON: vmrs
-;CHECK-NO_NEON: vmovgt.f32
+; CHECK-NO_NEON-LABEL: max_f32:
+; CHECK-NO_NEON:       @ %bb.0:
+; CHECK-NO_NEON-NEXT:    vcmp.f32 s1, s0
+; CHECK-NO_NEON-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NO_NEON-NEXT:    vmovgt.f32 s0, s1
+; CHECK-NO_NEON-NEXT:    mov pc, lr
+;
+; CHECK-NEON-LABEL: max_f32:
+; CHECK-NEON:       @ %bb.0:
+; CHECK-NEON-NEXT:    vcmp.f32 s1, s0
+; CHECK-NEON-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEON-NEXT:    vmovgt.f32 s0, s1
+; CHECK-NEON-NEXT:    mov pc, lr
   %3 = call nnan float @llvm.maxnum.f32(float %1, float %0)
   ret float %3
 }
 
 declare float @llvm.maxnum.f32(float, float) #1
 
 define float @min_f32(float, float) {
-;CHECK-NEON: vmin.f32
-;CHECK-NO_NEON: vcmp.f32
-;CHECK-NO_NEON: vmrs
-;CHECK-NO_NEON: vmovlt.f32
+; CHECK-NO_NEON-LABEL: min_f32:
+; CHECK-NO_NEON:       @ %bb.0:
+; CHECK-NO_NEON-NEXT:    vcmp.f32 s1, s0
+; CHECK-NO_NEON-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NO_NEON-NEXT:    vmovlt.f32 s0, s1
+; CHECK-NO_NEON-NEXT:    mov pc, lr
+;
+; CHECK-NEON-LABEL: min_f32:
+; CHECK-NEON:       @ %bb.0:
+; CHECK-NEON-NEXT:    vcmp.f32 s1, s0
+; CHECK-NEON-NEXT:    vmrs APSR_nzcv, fpscr
+; CHECK-NEON-NEXT:    vmovlt.f32 s0, s1
+; CHECK-NEON-NEXT:    mov pc, lr
   %3 = call nnan float @llvm.minnum.f32(float %1, float %0)
   ret float %3
 }
diff --git a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
@@ -502,13 +502,24 @@ define double @fmaxnum64_non_zero_intrinsic(double %x) {
 define <4 x float> @fminnumv432_intrinsic(<4 x float> %x, <4 x float> %y) {
 ; ARMV7-LABEL: fminnumv432_intrinsic:
 ; ARMV7:       @ %bb.0:
-; ARMV7-NEXT:    vmov d17, r2, r3
-; ARMV7-NEXT:    vmov d16, r0, r1
-; ARMV7-NEXT:    mov r0, sp
-; ARMV7-NEXT:    vld1.64 {d18, d19}, [r0]
-; ARMV7-NEXT:    vmin.f32 q8, q8, q9
-; ARMV7-NEXT:    vmov r0, r1, d16
-; ARMV7-NEXT:    vmov r2, r3, d17
+; ARMV7-NEXT:    mov r12, sp
+; ARMV7-NEXT:    vld1.64 {d0, d1}, [r12]
+; ARMV7-NEXT:    vmov d3, r2, r3
+; ARMV7-NEXT:    vmov d2, r0, r1
+; ARMV7-NEXT:    vcmp.f32 s7, s3
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s6, s2
+; ARMV7-NEXT:    vmovlt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vmovlt.f32 s2, s6
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s4, s0
+; ARMV7-NEXT:    vmovlt.f32 s1, s5
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmovlt.f32 s0, s4
+; ARMV7-NEXT:    vmov r2, r3, d1
+; ARMV7-NEXT:    vmov r0, r1, d0
 ; ARMV7-NEXT:    bx lr
 ;
 ; ARMV8-LABEL: fminnumv432_intrinsic:
@@ -635,21 +646,31 @@ define <4 x float> @fminnumv432_non_zero_intrinsic(<4 x float> %x) {
 define <4 x float> @fminnumv432_one_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-LABEL: fminnumv432_one_zero_intrinsic:
 ; ARMV7:       @ %bb.0:
-; ARMV7-NEXT:    vmov d17, r2, r3
-; ARMV7-NEXT:    vmov d16, r0, r1
-; ARMV7-NEXT:    adr r0, .LCPI18_0
-; ARMV7-NEXT:    vld1.64 {d18, d19}, [r0:128]
-; ARMV7-NEXT:    vmin.f32 q8, q8, q9
-; ARMV7-NEXT:    vmov r0, r1, d16
-; ARMV7-NEXT:    vmov r2, r3, d17
+; ARMV7-NEXT:    vmov d3, r2, r3
+; ARMV7-NEXT:    vmov d2, r0, r1
+; ARMV7-NEXT:    vmov.f32 s0, #-1.000000e+00
+; ARMV7-NEXT:    vcmp.f32 s5, #0
+; ARMV7-NEXT:    vldr s1, .LCPI18_0
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s7, s0
+; ARMV7-NEXT:    vmovlt.f32 s1, s5
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s3, s0
+; ARMV7-NEXT:    vcmp.f32 s6, s0
+; ARMV7-NEXT:    vmovlt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s2, s0
+; ARMV7-NEXT:    vcmp.f32 s4, s0
+; ARMV7-NEXT:    vmovlt.f32 s2, s6
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmovlt.f32 s0, s4
+; ARMV7-NEXT:    vmov r2, r3, d1
+; ARMV7-NEXT:    vmov r0, r1, d0
 ; ARMV7-NEXT:    bx lr
-; ARMV7-NEXT:    .p2align 4
+; ARMV7-NEXT:    .p2align 2
 ; ARMV7-NEXT:  @ %bb.1:
 ; ARMV7-NEXT:  .LCPI18_0:
-; ARMV7-NEXT:    .long 0xbf800000 @ float -1
 ; ARMV7-NEXT:    .long 0x00000000 @ float 0
-; ARMV7-NEXT:    .long 0xbf800000 @ float -1
-; ARMV7-NEXT:    .long 0xbf800000 @ float -1
 ;
 ; ARMV8-LABEL: fminnumv432_one_zero_intrinsic:
 ; ARMV8:       @ %bb.0:
@@ -697,13 +718,24 @@ define <4 x float> @fminnumv432_one_zero_intrinsic(<4 x float> %x) {
 define <4 x float> @fmaxnumv432_intrinsic(<4 x float> %x, <4 x float> %y) {
 ; ARMV7-LABEL: fmaxnumv432_intrinsic:
 ; ARMV7:       @ %bb.0:
-; ARMV7-NEXT:    vmov d17, r2, r3
-; ARMV7-NEXT:    vmov d16, r0, r1
-; ARMV7-NEXT:    mov r0, sp
-; ARMV7-NEXT:    vld1.64 {d18, d19}, [r0]
-; ARMV7-NEXT:    vmax.f32 q8, q8, q9
-; ARMV7-NEXT:    vmov r0, r1, d16
-; ARMV7-NEXT:    vmov r2, r3, d17
+; ARMV7-NEXT:    mov r12, sp
+; ARMV7-NEXT:    vld1.64 {d0, d1}, [r12]
+; ARMV7-NEXT:    vmov d3, r2, r3
+; ARMV7-NEXT:    vmov d2, r0, r1
+; ARMV7-NEXT:    vcmp.f32 s7, s3
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s6, s2
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s5, s1
+; ARMV7-NEXT:    vmovgt.f32 s2, s6
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vcmp.f32 s4, s0
+; ARMV7-NEXT:    vmovgt.f32 s1, s5
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmovgt.f32 s0, s4
+; ARMV7-NEXT:    vmov r2, r3, d1
+; ARMV7-NEXT:    vmov r0, r1, d0
 ; ARMV7-NEXT:    bx lr
 ;
 ; ARMV8-LABEL: fmaxnumv432_intrinsic:
@@ -789,13 +821,31 @@ define <4 x float> @fmaxnumv432_nsz_intrinsic(<4 x float> %x, <4 x float> %y) {
 define <4 x float> @fmaxnumv432_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-LABEL: fmaxnumv432_zero_intrinsic:
 ; ARMV7:       @ %bb.0:
-; ARMV7-NEXT:    vmov d19, r2, r3
-; ARMV7-NEXT:    vmov.i32 q8, #0x0
-; ARMV7-NEXT:    vmov d18, r0, r1
-; ARMV7-NEXT:    vmax.f32 q8, q9, q8
-; ARMV7-NEXT:    vmov r0, r1, d16
-; ARMV7-NEXT:    vmov r2, r3, d17
+; ARMV7-NEXT:    vmov d3, r2, r3
+; ARMV7-NEXT:    vldr s0, .LCPI21_0
+; ARMV7-NEXT:    vmov d2, r0, r1
+; ARMV7-NEXT:    vcmp.f32 s7, #0
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s3, s0
+; ARMV7-NEXT:    vcmp.f32 s6, #0
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s2, s0
+; ARMV7-NEXT:    vcmp.f32 s5, #0
+; ARMV7-NEXT:    vmovgt.f32 s2, s6
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s1, s0
+; ARMV7-NEXT:    vcmp.f32 s4, #0
+; ARMV7-NEXT:    vmovgt.f32 s1, s5
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmovgt.f32 s0, s4
+; ARMV7-NEXT:    vmov r2, r3, d1
+; ARMV7-NEXT:    vmov r0, r1, d0
 ; ARMV7-NEXT:    bx lr
+; ARMV7-NEXT:    .p2align 2
+; ARMV7-NEXT:  @ %bb.1:
+; ARMV7-NEXT:  .LCPI21_0:
+; ARMV7-NEXT:    .long 0x00000000 @ float 0
 ;
 ; ARMV8-LABEL: fmaxnumv432_zero_intrinsic:
 ; ARMV8:       @ %bb.0:
@@ -834,13 +884,31 @@ define <4 x float> @fmaxnumv432_zero_intrinsic(<4 x float> %x) {
 define <4 x float> @fmaxnumv432_minus_zero_intrinsic(<4 x float> %x) {
 ; ARMV7-LABEL: fmaxnumv432_minus_zero_intrinsic:
 ; ARMV7:       @ %bb.0:
-; ARMV7-NEXT:    vmov d19, r2, r3
-; ARMV7-NEXT:    vmov.i32 q8, #0x80000000
-; ARMV7-NEXT:    vmov d18, r0, r1
-; ARMV7-NEXT:    vmax.f32 q8, q9, q8
-; ARMV7-NEXT:    vmov r0, r1, d16
-; ARMV7-NEXT:    vmov r2, r3, d17
+; ARMV7-NEXT:    vldr s0, .LCPI22_0
+; ARMV7-NEXT:    vmov d3, r2, r3
+; ARMV7-NEXT:    vmov d2, r0, r1
+; ARMV7-NEXT:    vcmp.f32 s7, s0
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s3, s0
+; ARMV7-NEXT:    vcmp.f32 s6, s0
+; ARMV7-NEXT:    vmovgt.f32 s3, s7
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s2, s0
+; ARMV7-NEXT:    vcmp.f32 s5, s0
+; ARMV7-NEXT:    vmovgt.f32 s2, s6
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmov.f32 s1, s0
+; ARMV7-NEXT:    vcmp.f32 s4, s0
+; ARMV7-NEXT:    vmovgt.f32 s1, s5
+; ARMV7-NEXT:    vmrs APSR_nzcv, fpscr
+; ARMV7-NEXT:    vmovgt.f32 s0, s4
+; ARMV7-NEXT:    vmov r2, r3, d1
+; ARMV7-NEXT:    vmov r0, r1, d0
 ; ARMV7-NEXT:    bx lr
+; ARMV7-NEXT:    .p2align 2
+; ARMV7-NEXT:  @ %bb.1:
+; ARMV7-NEXT:  .LCPI22_0:
+; ARMV7-NEXT:    .long 0x80000000 @ float -0
 ;
 ; ARMV8-LABEL: fmaxnumv432_minus_zero_intrinsic:
 ; ARMV8:       @ %bb.0:
diff --git a/llvm/test/CodeGen/WebAssembly/f32.ll b/llvm/test/CodeGen/WebAssembly/f32.ll
@@ -217,10 +217,13 @@ define float @fminnum32_intrinsic(float %x, float %y) {
 ; CHECK-LABEL: fminnum32_intrinsic:
 ; CHECK:         .functype fminnum32_intrinsic (f32, f32) -> (f32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get $push2=, 0
-; CHECK-NEXT:    local.get $push1=, 1
-; CHECK-NEXT:    f32.min $push0=, $pop2, $pop1
-; CHECK-NEXT:    return $pop0
+; CHECK-NEXT:    local.get $push5=, 0
+; CHECK-NEXT:    local.get $push4=, 1
+; CHECK-NEXT:    local.get $push3=, 0
+; CHECK-NEXT:    local.get $push2=, 1
+; CHECK-NEXT:    f32.lt $push0=, $pop3, $pop2
+; CHECK-NEXT:    f32.select $push1=, $pop5, $pop4, $pop0
+; CHECK-NEXT:    return $pop1
   %a = call nnan float @llvm.minnum.f32(float %x, float %y)
   ret float %a
 }
@@ -267,10 +270,13 @@ define float @fmaxnum32_intrinsic(float %x, float %y) {
 ; CHECK-LABEL: fmaxnum32_intrinsic:
 ; CHECK:         .functype fmaxnum32_intrinsic (f32, f32) -> (f32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get $push2=, 0
-; CHECK-NEXT:    local.get $push1=, 1
-; CHECK-NEXT:    f32.max $push0=, $pop2, $pop1
-; CHECK-NEXT:    return $pop0
+; CHECK-NEXT:    local.get $push5=, 0
+; CHECK-NEXT:    local.get $push4=, 1
+; CHECK-NEXT:    local.get $push3=, 0
+; CHECK-NEXT:    local.get $push2=, 1
+; CHECK-NEXT:    f32.gt $push0=, $pop3, $pop2
+; CHECK-NEXT:    f32.select $push1=, $pop5, $pop4, $pop0
+; CHECK-NEXT:    return $pop1
   %a = call nnan float @llvm.maxnum.f32(float %x, float %y)
   ret float %a
 }
@@ -291,10 +297,13 @@ define float @fmaxnum32_zero_intrinsic(float %x) {
 ; CHECK-LABEL: fmaxnum32_zero_intrinsic:
 ; CHECK:         .functype fmaxnum32_zero_intrinsic (f32) -> (f32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    local.get $push2=, 0
+; CHECK-NEXT:    local.get $push5=, 0
 ; CHECK-NEXT:    f32.const $push0=, 0x0p0
-; CHECK-NEXT:    f32.max $push1=, $pop2, $pop0
-; CHECK-NEXT:    return $pop1
+; CHECK-NEXT:    local.get $push4=, 0
+; CHECK-NEXT:    f32.const $push3=, 0x0p0
+; CHECK-NEXT:    f32.gt $push1=, $pop4, $pop3
+; CHECK-NEXT:    f32.select $push2=, $pop5, $pop0, $pop1
+; CHECK-NEXT:    return $pop2
   %a = call nnan float @llvm.maxnum.f32(float %x, float 0.0)
   ret float %a
 }
diff --git a/llvm/test/CodeGen/WebAssembly/f64.ll b/llvm/test/CodeGen/WebAssembly/f64.ll
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
