APFloat/x87: Fix string conversion for "unnormal" values (pr35860)

Summary:
Unnormal values are a feature of some very old x87 processors. We handle
them correctly for the most part -- the only exception was an unnormal
value whose significand happened to be zero. In this case the APFloat
was still initialized as normal number (category = fcNormal), but a
subsequent toString operation would assert because the math would
produce nonsensical values for the zero significand.

During review, it was decided that the correct way to fix this is to
treat all unnormal values as NaNs (as that is what any >=386 processor
will do).

The issue was discovered because LLDB would crash when trying to print
some "long double" values.

Reviewers: skatkov, scanon, gottesmm

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D41868

llvm-svn: 331884

Author: labath

llvm/lib/Support/APFloat.cpp

@@ -3032,27 +3032,29 @@ double IEEEFloat::convertToDouble() const {
 /// does not support these bit patterns:
 ///  exponent = all 1's, integer bit 0, significand 0 ("pseudoinfinity")
 ///  exponent = all 1's, integer bit 0, significand nonzero ("pseudoNaN")
-///  exponent = 0, integer bit 1 ("pseudodenormal")
 ///  exponent!=0 nor all 1's, integer bit 0 ("unnormal")
-/// At the moment, the first two are treated as NaNs, the second two as Normal.
+///  exponent = 0, integer bit 1 ("pseudodenormal")
+/// At the moment, the first three are treated as NaNs, the last one as Normal.
 void IEEEFloat::initFromF80LongDoubleAPInt(const APInt &api) {
   assert(api.getBitWidth()==80);
   uint64_t i1 = api.getRawData()[0];
   uint64_t i2 = api.getRawData()[1];
   uint64_t myexponent = (i2 & 0x7fff);
   uint64_t mysignificand = i1;
+  uint8_t myintegerbit = mysignificand >> 63;
 
   initialize(&semX87DoubleExtended);
   assert(partCount()==2);
 
   sign = static_cast<unsigned int>(i2>>15);
-  if (myexponent==0 && mysignificand==0) {
+  if (myexponent == 0 && mysignificand == 0) {
     // exponent, significand meaningless
     category = fcZero;
   } else if (myexponent==0x7fff && mysignificand==0x8000000000000000ULL) {
     // exponent, significand meaningless
     category = fcInfinity;
-  } else if (myexponent==0x7fff && mysignificand!=0x8000000000000000ULL) {
+  } else if ((myexponent == 0x7fff && mysignificand != 0x8000000000000000ULL) ||
+             (myexponent != 0x7fff && myexponent != 0 && myintegerbit == 0)) {
     // exponent meaningless
     category = fcNaN;
     significandParts()[0] = mysignificand;

llvm/unittests/ADT/APFloatTest.cpp

@@ -983,6 +983,13 @@ TEST(APFloatTest, toString) {
   ASSERT_EQ("8.73183400000000010e+02", convertToString(873.1834, 0, 0, false));
   ASSERT_EQ("1.79769313486231570e+308",
             convertToString(1.7976931348623157E+308, 0, 0, false));
+
+  {
+    SmallString<64> Str;
+    APFloat UnnormalZero(APFloat::x87DoubleExtended(), APInt(80, {0, 1}));
+    UnnormalZero.toString(Str);
+    ASSERT_EQ("NaN", Str);
+  }
 }
 
 TEST(APFloatTest, toInteger) {