cgraphbuild.c (cgraph_rebuild_references): Rebuild only non-speculative refs.

	* cgraphbuild.c (cgraph_rebuild_references): Rebuild only non-speculative
	refs.
	* cgraph.c (cgraph_update_edge_in_call_site_hash): New function.
	(cgraph_add_edge_to_call_site_hash): Deal with speculative calls.
	(cgraph_set_call_stmt): Likewise.
	(cgraph_create_edge_1): Fix release checking compilatoin;
	clear lto_stmt_uid.
	(cgraph_free_edge): Free indirect info.
	(cgraph_turn_edge_to_speculative): New function.
	(cgraph_speculative_call_info): New function.
	(cgraph_make_edge_direct): Return direct edge; handle speculation.
	(cgraph_redirect_edge_call_stmt_to_callee): Expand speculative
	edges.
	(dump_cgraph_node): Dump speculation.
	(verify_edge_count_and_frequency): Accept speculative edges.
	(verify_edge_corresponds_to_fndecl): Handle partitioned cgraph.
	(verify_cgraph_node): Handle speculation.
	* cgraph.h (cgraph_edge): Add SPECULATIVE flag.
	(cgraph_set_call_stmt): Update prototype.
	(cgraph_make_edge_direct): Update prototype.
	(cgraph_speculative_call_info): Declare.
	* ipa-cp.c (ipcp_discover_new_direct_edges): Be ready for edge
	to change; update call of ipa_find_references.
	* ipa-ref.c (ipa_record_reference): Fix return value; clear
	lto_stmt_uid and speculative flags.
	(ipa_dump_references): Dump speculation.
	(ipa_clone_references): Clone speculative flag.
	(ipa_clone_referring): Likewise.
	(ipa_clone_ref): New function.
	(ipa_find_reference): Look into lto_stmt_uids
	(ipa_clear_stmts_in_references): Do not clear speculative calls.
	* ipa-ref.h (ipa_ref): Add lto_stmt_uid and speculative flags.
	(ipa_find_reference): Update declaration.
	(ipa_clone_ref): Declare.
	* lto-cgraph.c (lto_output_edge): Make lto_stmt_uids start from 0;
	stream speculative flag.
	(lto_output_ref): Stream statements uids and speculation.
	(input_ref): Likewise.
	(input_edge): Stream speuclation.
	* cgraphclones.c (cgraph_clone_edge): Clone speculation.
	(cgraph_set_call_stmt_including_clones): Handle speculation.
	* ipa-inline.c (heap_edge_removal_hook): New function.
	(inline_small_functions): Register it.
	* lto-streamer-in.c (fixup_call_stmt_edges_1): Bounds checking;
	also initialize refs.
	* ipa-prop.c (ipa_make_edge_direct_to_target): Be ready for
	edge to change.
	(try_make_edge_direct_simple_call): Likewise.
	(try_make_edge_direct_simple_call): Likewise.
	(update_indirect_edges_after_inlining): Likewise.
	(remove_described_reference): Look proper lto_stmt_uid.
	(propagate_controlled_uses): Likewise.
	(propagate_controlled_uses): Liekwise.
	* tree-inline.c (copy_bb): Copy speculative edges.
	(redirect_all_calls): New function.
	(copy_cfg_body): Do redirection after loop info
	is updated.
	(delete_unreachable_blocks_update_callgraph): Updadte
	speculation.

From-SVN: r201632

1 files changed, 108 insertions, 50 deletions

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index e33e5a3..00e3198 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -1676,6 +1676,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 		  if (edge)
 		    {
 		      int edge_freq = edge->frequency;
+		      int new_freq;
+		      struct cgraph_edge *old_edge = edge;
 		      edge = cgraph_clone_edge (edge, id->dst_node, stmt,
 					        gimple_uid (stmt),
 					        REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
@@ -1683,25 +1685,52 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 		      /* We could also just rescale the frequency, but
 		         doing so would introduce roundoff errors and make
 			 verifier unhappy.  */
-		      edge->frequency
-		        = compute_call_stmt_bb_frequency (id->dst_node->symbol.decl,
-							  copy_basic_block);
-		      if (dump_file
-		      	  && profile_status_for_function (cfun) != PROFILE_ABSENT
-			  && (edge_freq > edge->frequency + 10
-			      || edge_freq < edge->frequency - 10))
+		      new_freq  = compute_call_stmt_bb_frequency (id->dst_node->symbol.decl,
+								  copy_basic_block);
+
+		      /* Speculative calls consist of two edges - direct and indirect.
+			 Duplicate the whole thing and distribute frequencies accordingly.  */
+		      if (edge->speculative)
+			{
+			  struct cgraph_edge *direct, *indirect;
+			  struct ipa_ref *ref;
+
+			  gcc_assert (!edge->indirect_unknown_callee);
+			  cgraph_speculative_call_info (old_edge, direct, indirect, ref);
+			  indirect = cgraph_clone_edge (indirect, id->dst_node, stmt,
+							gimple_uid (stmt),
+							REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
+							true);
+			  if (old_edge->frequency + indirect->frequency)
+			    {
+			      edge->frequency = MIN (RDIV ((gcov_type)new_freq * old_edge->frequency,
+						           (old_edge->frequency + indirect->frequency)),
+						     CGRAPH_FREQ_MAX);
+			      indirect->frequency = MIN (RDIV ((gcov_type)new_freq * indirect->frequency,
+							       (old_edge->frequency + indirect->frequency)),
+							 CGRAPH_FREQ_MAX);
+			    }
+			  ipa_clone_ref (ref, (symtab_node)id->dst_node, stmt);
+			}
+		      else
 			{
-			  fprintf (dump_file, "Edge frequency estimated by "
-			           "cgraph %i diverge from inliner's estimate %i\n",
-			  	   edge_freq,
-				   edge->frequency);
-			  fprintf (dump_file,
-			  	   "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
-				   bb->index,
-				   bb->frequency,
-				   copy_basic_block->frequency);
+			  edge->frequency = new_freq;
+			  if (dump_file
+			      && profile_status_for_function (cfun) != PROFILE_ABSENT
+			      && (edge_freq > edge->frequency + 10
+				  || edge_freq < edge->frequency - 10))
+			    {
+			      fprintf (dump_file, "Edge frequency estimated by "
+				       "cgraph %i diverge from inliner's estimate %i\n",
+				       edge_freq,
+				       edge->frequency);
+			      fprintf (dump_file,
+				       "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
+				       bb->index,
+				       bb->frequency,
+				       copy_basic_block->frequency);
+			    }
 			}
-		      stmt = cgraph_redirect_edge_call_stmt_to_callee (edge);
 		    }
 		  break;
 
@@ -2232,6 +2261,23 @@ copy_loops (bitmap blocks_to_copy,
     }
 }
 
+/* Call cgraph_redirect_edge_call_stmt_to_callee on all calls in BB */
+
+void
+redirect_all_calls (copy_body_data * id, basic_block bb)
+{
+  gimple_stmt_iterator si;
+  for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+    {
+      if (is_gimple_call (gsi_stmt (si)))
+	{
+	  struct cgraph_edge *edge = cgraph_edge (id->dst_node, gsi_stmt (si));
+	  if (edge)
+	    cgraph_redirect_edge_call_stmt_to_callee (edge);
+	}
+    }
+}
+
 /* Make a copy of the body of FN so that it can be inserted inline in
    another function.  Walks FN via CFG, returns new fndecl.  */
 
@@ -2356,6 +2402,10 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,
 	    && bb->index != ENTRY_BLOCK
 	    && bb->index != EXIT_BLOCK)
 	  maybe_move_debug_stmts_to_successors (id, (basic_block) bb->aux);
+	/* Update call edge destinations.  This can not be done before loop
+	   info is updated, because we may split basic blocks.  */
+	if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
+	  redirect_all_calls (id, (basic_block)bb->aux);
 	((basic_block)bb->aux)->aux = NULL;
 	bb->aux = NULL;
       }
@@ -2367,6 +2417,10 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,
       if (need_debug_cleanup)
 	maybe_move_debug_stmts_to_successors (id, BASIC_BLOCK (last));
       BASIC_BLOCK (last)->aux = NULL;
+      /* Update call edge destinations.  This can not be done before loop
+	 info is updated, because we may split basic blocks.  */
+      if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
+	redirect_all_calls (id, BASIC_BLOCK (last));
     }
   entry_block_map->aux = NULL;
   exit_block_map->aux = NULL;
@@ -4941,43 +4995,47 @@ delete_unreachable_blocks_update_callgraph (copy_body_data *id)
           gimple_stmt_iterator bsi;
 
           for (bsi = gsi_start_bb (b); !gsi_end_p (bsi); gsi_next (&bsi))
-	    if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL)
-	      {
-	        struct cgraph_edge *e;
-		struct cgraph_node *node;
+	    {
+	      struct cgraph_edge *e;
+	      struct cgraph_node *node;
 
-	        if ((e = cgraph_edge (id->dst_node, gsi_stmt (bsi))) != NULL)
+	      ipa_remove_stmt_references ((symtab_node)id->dst_node, gsi_stmt (bsi));
+
+	      if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL
+		  &&(e = cgraph_edge (id->dst_node, gsi_stmt (bsi))) != NULL)
+		{
+		  if (!e->inline_failed)
+		    cgraph_remove_node_and_inline_clones (e->callee, id->dst_node);
+		  else
+		    cgraph_remove_edge (e);
+		}
+	      if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
+		  && id->dst_node->clones)
+		for (node = id->dst_node->clones; node != id->dst_node;)
 		  {
-		    if (!e->inline_failed)
-		      cgraph_remove_node_and_inline_clones (e->callee, id->dst_node);
+		    ipa_remove_stmt_references ((symtab_node)node, gsi_stmt (bsi));
+		    if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL
+			&& (e = cgraph_edge (node, gsi_stmt (bsi))) != NULL)
+		      {
+			if (!e->inline_failed)
+			  cgraph_remove_node_and_inline_clones (e->callee, id->dst_node);
+			else
+			  cgraph_remove_edge (e);
+		      }
+
+		    if (node->clones)
+		      node = node->clones;
+		    else if (node->next_sibling_clone)
+		      node = node->next_sibling_clone;
 		    else
-	              cgraph_remove_edge (e);
+		      {
+			while (node != id->dst_node && !node->next_sibling_clone)
+			  node = node->clone_of;
+			if (node != id->dst_node)
+			  node = node->next_sibling_clone;
+		      }
 		  }
-		if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
-		    && id->dst_node->clones)
-     		  for (node = id->dst_node->clones; node != id->dst_node;)
-		    {
-	              if ((e = cgraph_edge (node, gsi_stmt (bsi))) != NULL)
-			{
-		          if (!e->inline_failed)
-		            cgraph_remove_node_and_inline_clones (e->callee, id->dst_node);
-			  else
-	                    cgraph_remove_edge (e);
-			}
-
-		      if (node->clones)
-			node = node->clones;
-		      else if (node->next_sibling_clone)
-			node = node->next_sibling_clone;
-		      else
-			{
-			  while (node != id->dst_node && !node->next_sibling_clone)
-			    node = node->clone_of;
-			  if (node != id->dst_node)
-			    node = node->next_sibling_clone;
-			}
-		    }
-	      }
+	    }
 	  delete_basic_block (b);
 	  changed = true;
 	}