middle-end/117801 - failed register coalescing due to GIMPLE schedule

For a TSVC testcase we see failed register coalescing due to a
different schedule of GIMPLE .FMA and stores fed by it.  This
can be mitigated by making direct internal functions participate
in TER - given we're using more and more of such functions to
expose target capabilities it seems to be a natural thing to not
exempt those.

Unfortunately the internal function expanding API doesn't match
what we usually have - passing in a target and returning an RTX
but instead the LHS of the call is expanded and written to.  This
makes the TER expansion of a call SSA def a bit unwieldly.

Bootstrapped and tested on x86_64-unknown-linux-gnu.

The ccmp changes have likely not seen any coverage, the debug stmt
changes might not be optimal, we might end up losing on replaceable
calls.

	PR middle-end/117801
	* tree-outof-ssa.cc (ssa_is_replaceable_p): Make
	direct internal function calls replaceable.
	* expr.cc (get_def_for_expr): Handle replacements with calls.
	(get_def_for_expr_class): Likewise.
	(optimize_bitfield_assignment_op): Likewise.
	(expand_expr_real_1): Likewise.  Properly expand direct
	internal function defs.
	* cfgexpand.cc (expand_call_stmt): Handle replacements with calls.
	(avoid_deep_ter_for_debug): Likewise, always create a debug temp
	for calls.
	(expand_debug_expr): Likewise, give up for calls.
	(expand_gimple_basic_block): Likewise.
	* ccmp.cc (ccmp_candidate_p): Likewise.
	(get_compare_parts): Likewise.

4 files changed, 39 insertions, 13 deletions

diff --git a/gcc/ccmp.cc b/gcc/ccmp.cc
index 45629ab..4f739df 100644
--- a/gcc/ccmp.cc
+++ b/gcc/ccmp.cc
@@ -100,7 +100,7 @@ ccmp_candidate_p (gimple *g, bool outer = false)
   tree_code tcode;
   basic_block bb;
 
-  if (!g)
+  if (!g || !is_gimple_assign (g))
     return false;
 
   tcode = gimple_assign_rhs_code (g);
@@ -138,7 +138,7 @@ get_compare_parts (tree t, int *up, rtx_code *rcode,
 {
   tree_code code;
   gimple *g = get_gimple_for_ssa_name (t);
-  if (g)
+  if (g && is_gimple_assign (g))
     {
       *up = TYPE_UNSIGNED (TREE_TYPE (gimple_assign_rhs1 (g)));
       code = gimple_assign_rhs_code (g);
diff --git a/gcc/cfgexpand.cc b/gcc/cfgexpand.cc
index 58d68ec..ea08810 100644
--- a/gcc/cfgexpand.cc
+++ b/gcc/cfgexpand.cc
@@ -2848,6 +2848,7 @@ expand_call_stmt (gcall *stmt)
       if (builtin_p
 	  && TREE_CODE (arg) == SSA_NAME
 	  && (def = get_gimple_for_ssa_name (arg))
+	  && is_gimple_assign (def)
 	  && gimple_assign_rhs_code (def) == ADDR_EXPR)
 	arg = gimple_assign_rhs1 (def);
       CALL_EXPR_ARG (exp, i) = arg;
@@ -4414,7 +4415,7 @@ avoid_deep_ter_for_debug (gimple *stmt, int depth)
       gimple *g = get_gimple_for_ssa_name (use);
       if (g == NULL)
 	continue;
-      if (depth > 6 && !stmt_ends_bb_p (g))
+      if ((depth > 6 || !is_gimple_assign (g)) && !stmt_ends_bb_p (g))
 	{
 	  if (deep_ter_debug_map == NULL)
 	    deep_ter_debug_map = new hash_map<tree, tree>;
@@ -5388,7 +5389,13 @@ expand_debug_expr (tree exp)
 		  t = *slot;
 	      }
 	    if (t == NULL_TREE)
-	      t = gimple_assign_rhs_to_tree (g);
+	      {
+		if (is_gimple_assign (g))
+		  t = gimple_assign_rhs_to_tree (g);
+		else
+		  /* expand_debug_expr doesn't handle CALL_EXPR right now.  */
+		  return NULL;
+	      }
 	    op0 = expand_debug_expr (t);
 	    if (!op0)
 	      return NULL;
@@ -5964,7 +5971,8 @@ expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
 	  /* Look for SSA names that have their last use here (TERed
 	     names always have only one real use).  */
 	  FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
-	    if ((def = get_gimple_for_ssa_name (op)))
+	    if ((def = get_gimple_for_ssa_name (op))
+		&& is_gimple_assign (def))
 	      {
 		imm_use_iterator imm_iter;
 		use_operand_p use_p;
diff --git a/gcc/expr.cc b/gcc/expr.cc
index 70f2ece..5578e3d 100644
--- a/gcc/expr.cc
+++ b/gcc/expr.cc
@@ -65,6 +65,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "rtx-vector-builder.h"
 #include "tree-pretty-print.h"
 #include "flags.h"
+#include "internal-fn.h"
 
 
 /* If this is nonzero, we do not bother generating VOLATILE
@@ -3827,6 +3828,7 @@ get_def_for_expr (tree name, enum tree_code code)
 
   def_stmt = get_gimple_for_ssa_name (name);
   if (!def_stmt
+      || !is_gimple_assign (def_stmt)
       || gimple_assign_rhs_code (def_stmt) != code)
     return NULL;
 
@@ -3847,6 +3849,7 @@ get_def_for_expr_class (tree name, enum tree_code_class tclass)
 
   def_stmt = get_gimple_for_ssa_name (name);
   if (!def_stmt
+      || !is_gimple_assign (def_stmt)
       || TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) != tclass)
     return NULL;
 
@@ -5695,6 +5698,7 @@ optimize_bitfield_assignment_op (poly_uint64 pbitsize,
 
   srcstmt = get_gimple_for_ssa_name (src);
   if (!srcstmt
+      || !is_gimple_assign (srcstmt)
       || TREE_CODE_CLASS (gimple_assign_rhs_code (srcstmt)) != tcc_binary)
     return false;
 
@@ -11355,11 +11359,24 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode tmode,
 	  && !SSA_NAME_IS_DEFAULT_DEF (exp)
 	  && (optimize || !SSA_NAME_VAR (exp)
 	      || DECL_IGNORED_P (SSA_NAME_VAR (exp)))
+	  && is_gimple_assign (SSA_NAME_DEF_STMT (exp))
 	  && stmt_is_replaceable_p (SSA_NAME_DEF_STMT (exp)))
 	g = SSA_NAME_DEF_STMT (exp);
-      if (g)
+      if (safe_is_a <gassign *> (g))
 	return expand_expr_real_gassign (as_a<gassign *> (g), target, tmode,
 					 modifier, alt_rtl, inner_reference_p);
+      else if (safe_is_a <gcall *> (g))
+	{
+	  /* ???  internal call expansion doesn't follow the usual API
+	     of returning the destination RTX and being passed a desired
+	     target.  */
+	  rtx dest = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
+	  tree tmplhs = make_tree (TREE_TYPE (exp), dest);
+	  gimple_call_set_lhs (g, tmplhs);
+	  expand_internal_call (as_a <gcall *> (g));
+	  gimple_call_set_lhs (g, exp);
+	  return dest;
+	}
 
       ssa_name = exp;
       decl_rtl = get_rtx_for_ssa_name (ssa_name);
diff --git a/gcc/tree-outof-ssa.cc b/gcc/tree-outof-ssa.cc
index 3df8054..e51d5e0 100644
--- a/gcc/tree-outof-ssa.cc
+++ b/gcc/tree-outof-ssa.cc
@@ -45,6 +45,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-coalesce.h"
 #include "tree-outof-ssa.h"
 #include "dojump.h"
+#include "internal-fn.h"
 
 /* FIXME: A lot of code here deals with expanding to RTL.  All that code
    should be in cfgexpand.cc.  */
@@ -60,8 +61,11 @@ ssa_is_replaceable_p (gimple *stmt)
   tree def;
   gimple *use_stmt;
 
-  /* Only consider modify stmts.  */
-  if (!is_gimple_assign (stmt))
+  /* Only consider modify stmts and direct internal fn calls.  */
+  if (!is_gimple_assign (stmt)
+      && (!is_gimple_call (stmt)
+	  || !gimple_call_internal_p (stmt)
+	  || !direct_internal_fn_p (gimple_call_internal_fn (stmt))))
     return false;
 
   /* If the statement may throw an exception, it cannot be replaced.  */
@@ -92,14 +96,11 @@ ssa_is_replaceable_p (gimple *stmt)
 
   /* An assignment with a register variable on the RHS is not
      replaceable.  */
-  if (gimple_assign_rhs_code (stmt) == VAR_DECL
+  if (is_gimple_assign (stmt)
+      && gimple_assign_rhs_code (stmt) == VAR_DECL
       && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt)))
     return false;
 
-  /* No function calls can be replaced.  */
-  if (is_gimple_call (stmt))
-    return false;
-
   /* Leave any stmt with volatile operands alone as well.  */
   if (gimple_has_volatile_ops (stmt))
     return false;