[libc] adjust strtofloat precision for subnormals

Subnormal floating point numbers have a lower effective precision than
normal floating point numbers. This can cause issues for the fuzz test
since the MPFR floats have a constant precision regardless of the
exponent, and the precision must match exactly or else create rounding
errors. To solve this problem, the precision of the MPFR floats is
dynamically calculated.

Reviewed By: lntue

Differential Revision: https://reviews.llvm.org/D154909

1 files changed, 26 insertions, 5 deletions

diff --git a/libc/fuzzing/stdlib/strtofloat_fuzz.cpp b/libc/fuzzing/stdlib/strtofloat_fuzz.cpp
index 496ab06..1170b41 100644
--- a/libc/fuzzing/stdlib/strtofloat_fuzz.cpp
+++ b/libc/fuzzing/stdlib/strtofloat_fuzz.cpp
@@ -24,6 +24,25 @@
 
 using __llvm_libc::fputil::FloatProperties;
 
+// This function calculates the effective precision for a given float type and
+// exponent. Subnormals have a lower effective precision since they don't
+// necessarily use all of the bits of the mantissa.
+template <typename F> inline int effective_precision(int exponent) {
+  int full_precision = FloatProperties<F>::MANTISSA_PRECISION;
+
+  // This is intended to be 0 when the exponent is the lowest normal and
+  // increase as the exponent's magnitude increases.
+  int bits_below_normal = (-exponent) - (FloatProperties<F>::EXPONENT_BIAS - 1);
+
+  // This comparison is optimized out by the compiler.
+  if (bits_below_normal >= 0 && bits_below_normal < full_precision - 1) {
+    // The precision should be the normal, full precision, minus the bits lost
+    // by this being a subnormal, minus one for the implicit leading one.
+    return full_precision - bits_below_normal - 1;
+  }
+  return full_precision;
+}
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   uint8_t *container = new uint8_t[size + 1];
   if (!container)
@@ -46,7 +65,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   size_t base = 0;
 
-  // This is just used to determine the base.
+  // This is just used to determine the base and precision.
   mpfr_t result;
   mpfr_init2(result, 256);
   mpfr_t bin_result;
@@ -69,6 +88,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     base = 10;
   }
 
+  auto result_exp = mpfr_get_exp(result);
+
   mpfr_clear(result);
   mpfr_clear(bin_result);
 
@@ -79,15 +100,15 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   // correct double is instead 66336650, which when converted to float is
   // rounded down to 66336648. This means we have to compare against the correct
   // precision to get the correct result.
+
   mpfr_t mpfr_float;
-  mpfr_init2(mpfr_float, FloatProperties<float>::MANTISSA_PRECISION);
+  mpfr_init2(mpfr_float, effective_precision<float>(result_exp));
 
   mpfr_t mpfr_double;
-  mpfr_init2(mpfr_double, FloatProperties<double>::MANTISSA_PRECISION);
+  mpfr_init2(mpfr_double, effective_precision<double>(result_exp));
 
   mpfr_t mpfr_long_double;
-  mpfr_init2(mpfr_long_double,
-             FloatProperties<long double>::MANTISSA_PRECISION);
+  mpfr_init2(mpfr_long_double, effective_precision<long double>(result_exp));
 
   // TODO: Add support for other rounding modes.
   mpfr_strtofr(mpfr_float, str_ptr, &out_ptr, base, MPFR_RNDN);