PR tree-optimization/86114 - ICE in gimple_fold_builtin_strlen with an invalid call to strnlen

gcc/testsuite/ChangeLog:

	PR tree-optimization/86114
	* gcc.dg/pr86114.c: New test.

gcc/ChangeLog:

	PR tree-optimization/86114
	* gimple-fold.c (gimple_fold_builtin_strlen): Only handle LHS
	of integer types.
	* tree-ssa-strlen.c (maybe_set_strlen_range): Same.

From-SVN: r261567

5 files changed, 64 insertions, 8 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 8531f11..7826119 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,10 @@
+2018-06-13  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/86114
+	* gimple-fold.c (gimple_fold_builtin_strlen): Only handle LHS
+	of integer types.
+	* tree-ssa-strlen.c (maybe_set_strlen_range): Same.
+
 2018-06-13  Richard Biener  <rguenther@suse.de>
 
 	* tree-vect-patterns.c (vect_recog_vector_vector_shift_pattern):
@@ -10,7 +17,7 @@
 	* config/rl78/rl78.c (move_elim_pass): Use TDF_NONE rather than
 	integer 0 for argument to print_rtl_with_bb.
 	(rl78_reorg): Likewise.
- 
+
 2018-06-13  David Malcolm  <dmalcolm@redhat.com>
 
 	* config/arc/arc.c (hwloop_optimize): Strengthen local "end_label"
diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c
index c1d8442..a01bce7 100644
--- a/gcc/gimple-fold.c
+++ b/gcc/gimple-fold.c
@@ -3538,9 +3538,10 @@ gimple_fold_builtin_strlen (gimple_stmt_iterator *gsi)
       return true;
     }
 
-  tree lhs = gimple_call_lhs (stmt);
-  if (lhs && TREE_CODE (lhs) == SSA_NAME)
-    set_range_info (lhs, VR_RANGE, minlen, maxlen);
+  if (tree lhs = gimple_call_lhs (stmt))
+    if (TREE_CODE (lhs) == SSA_NAME
+	&& INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
+      set_range_info (lhs, VR_RANGE, minlen, maxlen);
 
   return false;
 }
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 0797df7..32d158f 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,8 @@
+2018-06-13  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/86114
+	* gcc.dg/pr86114.c: New test.
+
 2018-06-13  Steven G. Kargl  <kargl@gcc.gnu.org>
 
 	PR fortran/86110
diff --git a/gcc/testsuite/gcc.dg/pr86114.c b/gcc/testsuite/gcc.dg/pr86114.c
new file mode 100644
index 0000000..f829f00
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr86114.c
@@ -0,0 +1,42 @@
+/* PR tree-optimization/86114 - ICE in gimple_fold_builtin_strlen with
+   an invalid call to strnlen
+   { dg-do compile }
+   { dg-options "-O2" }  */
+
+typedef __SIZE_TYPE__ size_t;
+
+extern char* strcmp (const char*, const char*);
+extern char* strncmp (const char*, const char*, size_t);
+extern char* strlen (const char*);
+extern char* strnlen (const char*, size_t);
+extern char* strcspn (const char*, const char*);
+extern char* strspn (const char*, const char*);
+extern char* strxfrm (const char*, const char*, size_t);
+
+char** q;
+
+void test_array (const char *s)
+{
+  extern char a[8];
+
+  q[0] = strcmp (a, s);
+  q[1] = strncmp (a, s, 7);
+  q[2] = strlen (a);
+  q[3] = strnlen (a, 7);
+  q[4] = strcspn (a, s);
+  q[5] = strspn (a, s);
+  q[6] = strxfrm (a, s, 7);
+}
+
+void test_pointer (const char *s, const char *t)
+{
+  q[0] = strcmp (s, t);
+  q[1] = strncmp (s, t, 7);
+  q[2] = strlen (s);
+  q[3] = strnlen (s, 7);
+  q[4] = strcspn (s, t);
+  q[5] = strspn (s, t);
+  q[6] = strxfrm (s, s, 7);
+}
+
+/* { dg-prune-output "-Wbuiltin-declaration-mismatch" } */
diff --git a/gcc/tree-ssa-strlen.c b/gcc/tree-ssa-strlen.c
index 8794cc2..a4064a5 100644
--- a/gcc/tree-ssa-strlen.c
+++ b/gcc/tree-ssa-strlen.c
@@ -1124,14 +1124,15 @@ adjust_last_stmt (strinfo *si, gimple *stmt, bool is_strcat)
   update_stmt (last.stmt);
 }
 
-/* For an LHS that is an SSA_NAME and for strlen() argument SRC, set
-   LHS range info to [0, N] if SRC refers to a character array A[N]
-   with unknown length bounded by N.  */
+/* For an LHS that is an SSA_NAME with integer type and for strlen()
+   argument SRC, set LHS range info to [0, N] if SRC refers to
+   a character array A[N] with unknown length bounded by N.  */
 
 static void
 maybe_set_strlen_range (tree lhs, tree src)
 {
-  if (TREE_CODE (lhs) != SSA_NAME)
+  if (TREE_CODE (lhs) != SSA_NAME
+      || !INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
     return;
 
   if (TREE_CODE (src) == SSA_NAME)