analyzer: fix ICE when canonicalizing NaN (PR 93451)

PR analyzer/93451 reports an ICE when canonicalizing the constants
in a region_model, with a failed qsort_chk when attempting to sort
the constants within the region_model.

The svalues in the model were:
  sv0: {poisoned: uninit}
  sv1: {type: ‘double’, ‘0.0’}
  sv2: {type: ‘double’, ‘1.0e+0’}
  sv3: {type: ‘double’, ‘ Nan’}

The qsort_chk of the 3 constants fails due to tree_cmp using the
LT_EXPR ordering of the REAL_CSTs, which doesn't work for NaN.

This patch adjusts tree_cmp to impose an arbitrary ordering during
canonicalization for UNORDERED_EXPR cases w/o relying on the LT_EXPR
ordering, fixing the ICE.

gcc/analyzer/ChangeLog:
	PR analyzer/93451
	* region-model.cc (tree_cmp): For the REAL_CST case, impose an
	arbitrary order on NaNs relative to other NaNs and to non-NaNs;
	const-correctness tweak.
	(ana::selftests::build_real_cst_from_string): New function.
	(ana::selftests::append_interesting_constants): New function.
	(ana::selftests::test_tree_cmp_on_constants): New test.
	(ana::selftests::test_canonicalization_4): New test.
	(ana::selftests::analyzer_region_model_cc_tests): Call the new
	tests.

gcc/testsuite/ChangeLog:
	PR analyzer/93451
	* gcc.dg/analyzer/torture/pr93451.c: New test.

4 files changed, 119 insertions, 3 deletions

diff --git a/gcc/analyzer/ChangeLog b/gcc/analyzer/ChangeLog
index 3a2d179..9ba6adc 100644
--- a/gcc/analyzer/ChangeLog
+++ b/gcc/analyzer/ChangeLog
@@ -1,5 +1,18 @@
 2020-01-27  David Malcolm  <dmalcolm@redhat.com>
 
+	PR analyzer/93451
+	* region-model.cc (tree_cmp): For the REAL_CST case, impose an
+	arbitrary order on NaNs relative to other NaNs and to non-NaNs;
+	const-correctness tweak.
+	(ana::selftests::build_real_cst_from_string): New function.
+	(ana::selftests::append_interesting_constants): New function.
+	(ana::selftests::test_tree_cmp_on_constants): New test.
+	(ana::selftests::test_canonicalization_4): New test.
+	(ana::selftests::analyzer_region_model_cc_tests): Call the new
+	tests.
+
+2020-01-27  David Malcolm  <dmalcolm@redhat.com>
+
 	PR analyzer/93349
 	* engine.cc (run_checkers): Save and restore input_location.
 
diff --git a/gcc/analyzer/region-model.cc b/gcc/analyzer/region-model.cc
index a986549..62c96a6 100644
--- a/gcc/analyzer/region-model.cc
+++ b/gcc/analyzer/region-model.cc
@@ -1811,11 +1811,22 @@ tree_cmp (const_tree t1, const_tree t2)
 
     case REAL_CST:
       {
-	real_value *rv1 = TREE_REAL_CST_PTR (t1);
-	real_value *rv2 = TREE_REAL_CST_PTR (t2);
+	const real_value *rv1 = TREE_REAL_CST_PTR (t1);
+	const real_value *rv2 = TREE_REAL_CST_PTR (t2);
+	if (real_compare (UNORDERED_EXPR, rv1, rv2))
+	  {
+	    /* Impose an arbitrary order on NaNs relative to other NaNs
+	       and to non-NaNs.  */
+	    if (int cmp_isnan = real_isnan (rv1) - real_isnan (rv2))
+	      return cmp_isnan;
+	    if (int cmp_issignaling_nan
+		  = real_issignaling_nan (rv1) - real_issignaling_nan (rv2))
+	      return cmp_issignaling_nan;
+	    return real_isneg (rv1) - real_isneg (rv2);
+	  }
 	if (real_compare (LT_EXPR, rv1, rv2))
 	  return -1;
-	if (real_compare (LT_EXPR, rv2, rv1))
+	if (real_compare (GT_EXPR, rv1, rv2))
 	  return 1;
 	return 0;
       }
@@ -6927,6 +6938,58 @@ namespace ana {
 
 namespace selftest {
 
+/* Build a constant tree of the given type from STR.  */
+
+static tree
+build_real_cst_from_string (tree type, const char *str)
+{
+  REAL_VALUE_TYPE real;
+  real_from_string (&real, str);
+  return build_real (type, real);
+}
+
+/* Append various "interesting" constants to OUT (e.g. NaN).  */
+
+static void
+append_interesting_constants (auto_vec<tree> *out)
+{
+  out->safe_push (build_int_cst (integer_type_node, 0));
+  out->safe_push (build_int_cst (integer_type_node, 42));
+  out->safe_push (build_int_cst (unsigned_type_node, 0));
+  out->safe_push (build_int_cst (unsigned_type_node, 42));
+  out->safe_push (build_real_cst_from_string (float_type_node, "QNaN"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "-QNaN"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "SNaN"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "-SNaN"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "0.0"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "-0.0"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "Inf"));
+  out->safe_push (build_real_cst_from_string (float_type_node, "-Inf"));
+}
+
+/* Verify that tree_cmp is a well-behaved comparator for qsort, even
+   if the underlying constants aren't comparable.  */
+
+static void
+test_tree_cmp_on_constants ()
+{
+  auto_vec<tree> csts;
+  append_interesting_constants (&csts);
+
+  /* Try sorting every triple. */
+  const unsigned num = csts.length ();
+  for (unsigned i = 0; i < num; i++)
+    for (unsigned j = 0; j < num; j++)
+      for (unsigned k = 0; k < num; k++)
+	{
+	  auto_vec<tree> v (3);
+	  v.quick_push (csts[i]);
+	  v.quick_push (csts[j]);
+	  v.quick_push (csts[k]);
+	  v.qsort (tree_cmp);
+	}
+}
+
 /* Implementation detail of the ASSERT_CONDITION_* macros.  */
 
 void
@@ -7577,6 +7640,25 @@ test_canonicalization_3 ()
   ASSERT_EQ (model0, model1);
 }
 
+/* Verify that we can canonicalize a model containing NaN and other real
+   constants.  */
+
+static void
+test_canonicalization_4 ()
+{
+  auto_vec<tree> csts;
+  append_interesting_constants (&csts);
+
+  region_model model;
+
+  unsigned i;
+  tree cst;
+  FOR_EACH_VEC_ELT (csts, i, cst)
+    model.get_rvalue (cst, NULL);
+
+  model.canonicalize (NULL);
+}
+
 /* Assert that if we have two region_model instances
    with values VAL_A and VAL_B for EXPR that they are
    mergable.  Write the merged model to *OUT_MERGED_MODEL,
@@ -7957,6 +8039,7 @@ test_constraint_merging ()
 void
 analyzer_region_model_cc_tests ()
 {
+  test_tree_cmp_on_constants ();
   test_dump ();
   test_unique_constants ();
   test_svalue_equality ();
@@ -7969,6 +8052,7 @@ analyzer_region_model_cc_tests ()
   test_canonicalization_1 ();
   test_canonicalization_2 ();
   test_canonicalization_3 ();
+  test_canonicalization_4 ();
   test_state_merging ();
   test_constraint_merging ();
 }
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index f81caf5..f83f18b 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,8 @@
+2020-01-27  David Malcolm  <dmalcolm@redhat.com>
+
+	PR analyzer/93451
+	* gcc.dg/analyzer/torture/pr93451.c: New test.
+
 2020-01-27  Stam Markianos-Wright  <stam.markianos-wright@arm.com>
 
 	* gcc.target/arm/armv8_2-fp16-move-1.c: Update following load/store
diff --git a/gcc/testsuite/gcc.dg/analyzer/torture/pr93451.c b/gcc/testsuite/gcc.dg/analyzer/torture/pr93451.c
new file mode 100644
index 0000000..5908bc4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/torture/pr93451.c
@@ -0,0 +1,14 @@
+void
+mt (double);
+
+void
+nm (void)
+{
+  double ao = 0.0;
+  long int es = -1;
+
+  mt (ao);
+  ++ao;
+  mt (ao);
+  mt (*(double *) &es);
+}