[libc][math][c23] Fix impl and tests for X86_80 canonicalize function. (#87103)

In continuation to: #87097

3 files changed, 54 insertions, 55 deletions

diff --git a/libc/src/__support/FPUtil/BasicOperations.h b/libc/src/__support/FPUtil/BasicOperations.h
index a47931b..6e41564 100644
--- a/libc/src/__support/FPUtil/BasicOperations.h
+++ b/libc/src/__support/FPUtil/BasicOperations.h
@@ -185,27 +185,28 @@ LIBC_INLINE int canonicalize(T &cx, const T &x) {
     // More precisely :
     // Exponent   |       Significand      | Meaning
     //            | Bits 63-62 | Bits 61-0 |
-    // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = Infinty
+    // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = SNaN
     // All Ones   |     00     |  Non-Zero | Pseudo NaN, Value = SNaN
     // All Ones   |     01     | Anything  | Pseudo NaN, Value = SNaN
     //            |   Bit 63   | Bits 62-0 |
     // All zeroes |   One      | Anything  | Pseudo Denormal, Value =
     //            |            |           | (−1)**s × m × 2**−16382
-    // All Other  |   Zero     | Anything  | Unnormal, Value =
-    //  Values    |            |           | (−1)**s × m × 2**−16382
+    // All Other  |   Zero     | Anything  | Unnormal, Value = SNaN
+    //  Values    |            |           |
     bool bit63 = sx.get_implicit_bit();
     UInt128 mantissa = sx.get_explicit_mantissa();
     bool bit62 = static_cast<bool>((mantissa & (1ULL << 62)) >> 62);
     int exponent = sx.get_biased_exponent();
     if (exponent == 0x7FFF) {
       if (!bit63 && !bit62) {
-        if (mantissa == 0)
-          cx = FPBits<T>::inf(sx.sign()).get_val();
-        else {
+        if (mantissa == 0) {
           cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
           raise_except_if_required(FE_INVALID);
           return 1;
         }
+        cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
+        raise_except_if_required(FE_INVALID);
+        return 1;
       } else if (!bit63 && bit62) {
         cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
         raise_except_if_required(FE_INVALID);
@@ -219,9 +220,11 @@ LIBC_INLINE int canonicalize(T &cx, const T &x) {
         cx = x;
     } else if (exponent == 0 && bit63)
       cx = FPBits<T>::make_value(mantissa, 0).get_val();
-    else if (exponent != 0 && !bit63)
-      cx = FPBits<T>::make_value(mantissa, 0).get_val();
-    else if (LIBC_UNLIKELY(sx.is_signaling_nan())) {
+    else if (exponent != 0 && !bit63) {
+      cx = FPBits<T>::quiet_nan(sx.sign(), mantissa).get_val();
+      raise_except_if_required(FE_INVALID);
+      return 1;
+    } else if (LIBC_UNLIKELY(sx.is_signaling_nan())) {
       cx =
           FPBits<T>::quiet_nan(sx.sign(), sx.get_explicit_mantissa()).get_val();
       raise_except_if_required(FE_INVALID);
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 3b75612..ae2cbad 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -178,6 +178,7 @@ add_fp_unittest(
     libc.src.math.canonicalize
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.integer_literals
 )
 
 add_fp_unittest(
@@ -193,6 +194,7 @@ add_fp_unittest(
     libc.src.math.canonicalizef
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.integer_literals
 )
 
 add_fp_unittest(
@@ -208,6 +210,7 @@ add_fp_unittest(
     libc.src.math.canonicalizef128
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.integer_literals
 )
 
 add_fp_unittest(
@@ -223,6 +226,7 @@ add_fp_unittest(
     libc.src.math.canonicalizel
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.integer_literals
 )
 
 add_fp_unittest(
diff --git a/libc/test/src/math/smoke/CanonicalizeTest.h b/libc/test/src/math/smoke/CanonicalizeTest.h
index c8af83f..4361f7d 100644
--- a/libc/test/src/math/smoke/CanonicalizeTest.h
+++ b/libc/test/src/math/smoke/CanonicalizeTest.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_TEST_SRC_MATH_SMOKE_CANONICALIZETEST_H
 
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/integer_literals.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
 
@@ -22,6 +23,8 @@
 
 #define TEST_REGULAR(x, y, expected) TEST_SPECIAL(x, y, expected, 0)
 
+using LIBC_NAMESPACE::operator""_u128;
+
 template <typename T>
 class CanonicalizeTest : public LIBC_NAMESPACE::testing::Test {
 
@@ -55,33 +58,31 @@ public:
       T cx;
       // Exponent   |       Significand      | Meaning
       //            | Bits 63-62 | Bits 61-0 |
-      // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = Infinty
-
-      FPBits test1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000000));
+      // All Ones   |     00     |    Zero   | Pseudo Infinity, Value = SNaN
+      FPBits test1(0x00000000'00007FFF'00000000'00000000_u128);
       const T test1_val = test1.get_val();
-      TEST_SPECIAL(cx, test1_val, 0, 0);
-      EXPECT_FP_EQ(cx, inf);
+      TEST_SPECIAL(cx, test1_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
       // Exponent   |       Significand      | Meaning
       //            | Bits 63-62 | Bits 61-0 |
       // All Ones   |     00     |  Non-Zero | Pseudo NaN, Value = SNaN
-
-      FPBits test2_1((UInt128(0x7FFF) << 64) + UInt128(0x0000000000000001));
+      FPBits test2_1(0x00000000'00007FFF'00000000'00000001_u128);
       const T test2_1_val = test2_1.get_val();
       TEST_SPECIAL(cx, test2_1_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test2_2((UInt128(0x7FFF) << 64) + UInt128(0x0000004270000001));
+      FPBits test2_2(0x00000000'00007FFF'00000042'70000001_u128);
       const T test2_2_val = test2_2.get_val();
       TEST_SPECIAL(cx, test2_2_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test2_3((UInt128(0x7FFF) << 64) + UInt128(0x0000000008261001));
+      FPBits test2_3(0x00000000'00007FFF'00000000'08261001_u128);
       const T test2_3_val = test2_3.get_val();
       TEST_SPECIAL(cx, test2_3_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test2_4((UInt128(0x7FFF) << 64) + UInt128(0x0000780008261001));
+      FPBits test2_4(0x00000000'00007FFF'00007800'08261001_u128);
       const T test2_4_val = test2_4.get_val();
       TEST_SPECIAL(cx, test2_4_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
@@ -89,23 +90,22 @@ public:
       // Exponent   |       Significand      | Meaning
       //            | Bits 63-62 | Bits 61-0 |
       // All Ones   |     01     | Anything  | Pseudo NaN, Value = SNaN
-
-      FPBits test3_1((UInt128(0x7FFF) << 64) + UInt128(0x4000000000000000));
+      FPBits test3_1(0x00000000'00007FFF'40000000'00000000_u128);
       const T test3_1_val = test3_1.get_val();
       TEST_SPECIAL(cx, test3_1_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test3_2((UInt128(0x7FFF) << 64) + UInt128(0x4000004270000001));
+      FPBits test3_2(0x00000000'00007FFF'40000042'70000001_u128);
       const T test3_2_val = test3_2.get_val();
       TEST_SPECIAL(cx, test3_2_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test3_3((UInt128(0x7FFF) << 64) + UInt128(0x4000000008261001));
+      FPBits test3_3(0x00000000'00007FFF'40000000'08261001_u128);
       const T test3_3_val = test3_3.get_val();
       TEST_SPECIAL(cx, test3_3_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test3_4((UInt128(0x7FFF) << 64) + UInt128(0x4007800008261001));
+      FPBits test3_4(0x00000000'00007FFF'40007800'08261001_u128);
       const T test3_4_val = test3_4.get_val();
       TEST_SPECIAL(cx, test3_4_val, 1, FE_INVALID);
       EXPECT_FP_EQ(cx, aNaN);
@@ -114,20 +114,19 @@ public:
       //            |   Bit 63   | Bits 62-0 |
       // All zeroes |   One      | Anything  | Pseudo Denormal, Value =
       //            |            |           | (−1)**s × m × 2**−16382
-
-      FPBits test4_1((UInt128(0x0000) << 64) + UInt128(0x8000000000000000));
+      FPBits test4_1(0x00000000'00000000'80000000'00000000_u128);
       const T test4_1_val = test4_1.get_val();
       TEST_SPECIAL(cx, test4_1_val, 0, 0);
       EXPECT_FP_EQ(
           cx, FPBits::make_value(test4_1.get_explicit_mantissa(), 0).get_val());
 
-      FPBits test4_2((UInt128(0x0000) << 64) + UInt128(0x8000004270000001));
+      FPBits test4_2(0x00000000'00000000'80000042'70000001_u128);
       const T test4_2_val = test4_2.get_val();
       TEST_SPECIAL(cx, test4_2_val, 0, 0);
       EXPECT_FP_EQ(
           cx, FPBits::make_value(test4_2.get_explicit_mantissa(), 0).get_val());
 
-      FPBits test4_3((UInt128(0x0000) << 64) + UInt128(0x8000000008261001));
+      FPBits test4_3(0x00000000'00000000'80000000'08261001_u128);
       const T test4_3_val = test4_3.get_val();
       TEST_SPECIAL(cx, test4_3_val, 0, 0);
       EXPECT_FP_EQ(
@@ -135,44 +134,37 @@ public:
 
       // Exponent   |       Significand      | Meaning
       //            |   Bit 63   | Bits 62-0 |
-      // All Other  |   Zero     | Anything  | Unnormal, Value =
-      //  Values    |            |           | (−1)**s × m × 2**−16382
-
-      FPBits test5_1(UInt128(0x0000000000000001));
+      // All Other  |   Zero     | Anything  | Unnormal, Value = SNaN
+      //  Values    |            |           |
+      FPBits test5_1(0x00000000'00000040'00000000'00000001_u128);
       const T test5_1_val = test5_1.get_val();
-      TEST_SPECIAL(cx, test5_1_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_1.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_1_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test5_2(UInt128(0x0000004270000001));
+      FPBits test5_2(0x00000000'00000230'00000042'70000001_u128);
       const T test5_2_val = test5_2.get_val();
-      TEST_SPECIAL(cx, test5_2_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_2.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_2_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test5_3(UInt128(0x0000000008261001));
+      FPBits test5_3(0x00000000'00000560'00000000'08261001_u128);
       const T test5_3_val = test5_3.get_val();
-      TEST_SPECIAL(cx, test5_3_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_3.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_3_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test5_4(UInt128(0x0000002816000000));
+      FPBits test5_4(0x00000000'00000780'00000028'16000000_u128);
       const T test5_4_val = test5_4.get_val();
-      TEST_SPECIAL(cx, test5_4_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_4.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_4_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test5_5(UInt128(0x0000004270000001));
+      FPBits test5_5(0x00000000'00000900'00000042'70000001_u128);
       const T test5_5_val = test5_5.get_val();
-      TEST_SPECIAL(cx, test5_5_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_5.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_5_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
 
-      FPBits test5_6(UInt128(0x0000000008261001));
+      FPBits test5_6(0x00000000'00000AB0'00000000'08261001_u128);
       const T test5_6_val = test5_6.get_val();
-      TEST_SPECIAL(cx, test5_6_val, 0, 0);
-      EXPECT_FP_EQ(
-          cx, FPBits::make_value(test5_6.get_explicit_mantissa(), 0).get_val());
+      TEST_SPECIAL(cx, test5_6_val, 1, FE_INVALID);
+      EXPECT_FP_EQ(cx, aNaN);
     }
   }