i386.c (make_resolver_func): Update.

2017-05-23  Jan Hubicka  <hubicka@ucw.cz>

	* config/i386/i386.c (make_resolver_func): Update.
	* Makefile.in: Add profile-count.h and profile-count.o
	* auto-profile.c (afdo_indirect_call): Update to new API.
	(afdo_set_bb_count): Update.
	(afdo_propagate_edge): Update.
	(afdo_propagate_circuit): Update.
	(afdo_calculate_branch_prob): Update.
	(afdo_annotate_cfg): Update.
	* basic-block.h: Include profile-count.h
	(struct edge_def): Turn count to profile_count.
	(struct basic_block_def): Likewie.
	(REG_BR_PROB_BASE): Move to profile-count.h
	(RDIV): Move to profile-count.h
	* bb-reorder.c (max_entry_count): Turn to profile_count.
	(find_traces): Update.
	(rotate_loop):Update.
	(connect_traces):Update.
	(sanitize_hot_paths):Update.
	* bt-load.c (migrate_btr_defs): Update.
	* cfg.c (RDIV): Remove.
	(init_flow): Use alloc_block.
	(alloc_block): Uninitialize count.
	(unchecked_make_edge): Uninitialize count.
	(check_bb_profile): Update.
	(dump_edge_info): Update.
	(dump_bb_info): Update.
	(update_bb_profile_for_threading): Update.
	(scale_bbs_frequencies_int): Update.
	(scale_bbs_frequencies_gcov_type): Update.
	(scale_bbs_frequencies_profile_count): New.
	* cfg.h (update_bb_profile_for_threading): Update.
	(scale_bbs_frequencies_profile_count): Declare.
	* cfgbuild.c (compute_outgoing_frequencies): Update.
	(find_many_sub_basic_blocks): Update.
	* cfgcleanup.c (try_forward_edges): Update.
	(try_crossjump_to_edge): Update.
	* cfgexpand.c (expand_gimple_tailcall): Update.
	(construct_exit_block): Update.
	* cfghooks.c (verify_flow_info): Update.
	(dump_bb_for_graph): Update.
	(split_edge): Update.
	(make_forwarder_block): Update.
	(duplicate_block): Update.
	(account_profile_record): Update.
	* cfgloop.c (find_subloop_latch_edge_by_profile): Update.
	(get_estimated_loop_iterations): Update.
	* cfgloopanal.c (expected_loop_iterations_unbounded): Update.
	(single_likely_exit): Update.
	* cfgloopmanip.c (scale_loop_profile): Update.
	(loopify): Update.
	(set_zero_probability): Update.
	(lv_adjust_loop_entry_edge): Update.
	* cfgrtl.c (force_nonfallthru_and_redirect): Update.
	(purge_dead_edges): Update.
	(rtl_account_profile_record): Update.
	* cgraph.c (cgraph_node::create): Uninitialize count.
	(symbol_table::create_edge): Uninitialize count.
	(cgraph_update_edges_for_call_stmt_node): Update.
	(cgraph_edge::dump_edge_flags): Update.
	(cgraph_node::dump): Update.
	(cgraph_edge::maybe_hot_p): Update.
	* cgraph.h: Include profile-count.h
	(create_clone), create_edge, create_indirect_edge): Update.
	(cgraph_node): Turn count to profile_count.
	(cgraph_edge0: Likewise.
	(make_speculative, clone): Update.
	(create_edge): Update.
	(init_lowered_empty_function): Update.
	* cgraphclones.c (cgraph_edge::clone): Update.
	(duplicate_thunk_for_node): Update.
	(cgraph_node::create_clone): Update.
	* cgraphunit.c (cgraph_node::analyze): Update.
	(cgraph_node::expand_thunk): Update.
	* final.c (dump_basic_block_info): Update.
	* gimple-streamer-in.c (input_bb): Update.
	* gimple-streamer-out.c (output_bb): Update.
	* graphite.c (print_global_statistics): Update.
	(print_graphite_scop_statistics): Update.
	* hsa-brig.c: Include basic-block.h.
	* hsa-dump.c: Include basic-block.h.
	* hsa-gen.c (T sum_slice): Update.
	(convert_switch_statements):Update.
	* hsa-regalloc.c: Include basic-block.h.
	* ipa-chkp.c (chkp_produce_thunks): Update.
	* ipa-cp.c (struct caller_statistics): Update.
	(init_caller_stats): Update.
	(gather_caller_stats): Update.
	(ipcp_cloning_candidate_p): Update.
	(good_cloning_opportunity_p): Update.
	(get_info_about_necessary_edges): Update.
	(dump_profile_updates): Update.
	(update_profiling_info): Update.
	(update_specialized_profile): Update.
	(perhaps_add_new_callers): Update.
	(decide_about_value): Update.
	(ipa_cp_c_finalize): Update.
	* ipa-devirt.c (struct odr_type_warn_count): Update.
	(struct decl_warn_count): Update.
	(struct final_warning_record): Update.
	(possible_polymorphic_call_targets): Update.
	(ipa_devirt): Update.
	* ipa-fnsummary.c (redirect_to_unreachable): Update.
	* ipa-icf.c (sem_function::merge): Update.
	* ipa-inline-analysis.c (do_estimate_edge_time): Update.
	* ipa-inline.c (compute_uninlined_call_time): Update.
	(compute_inlined_call_time): Update.
	(want_inline_small_function_p): Update.
	(want_inline_self_recursive_call_p): Update.
	(edge_badness): Update.
	(lookup_recursive_calls): Update.
	(recursive_inlining): Update.
	(inline_small_functions): Update.
	(dump_overall_stats): Update.
	(dump_inline_stats): Update.
	* ipa-profile.c (ipa_profile_generate_summary): Update.
	(ipa_propagate_frequency): Update.
	(ipa_profile): Update.
	* ipa-prop.c (ipa_make_edge_direct_to_target): Update.
	* ipa-utils.c (ipa_merge_profiles): Update.
	* loop-doloop.c (doloop_modify): Update.
	* loop-unroll.c (report_unroll): Update.
	(unroll_loop_runtime_iterations): Update.
	* lto-cgraph.c (lto_output_edge): Update.
	(lto_output_node): Update.
	(input_node): Update.
	(input_edge): Update.
	(merge_profile_summaries): Update.
	* lto-streamer-in.c (input_cfg): Update.
	* lto-streamer-out.c (output_cfg): Update.
	* mcf.c (create_fixup_graph): Update.
	(adjust_cfg_counts): Update.
	(sum_edge_counts): Update.
	* modulo-sched.c (sms_schedule): Update.
	* postreload-gcse.c (eliminate_partially_redundant_load): Update.
	* predict.c (maybe_hot_count_p): Update.
	(probably_never_executed): Update.
	(dump_prediction): Update.
	(combine_predictions_for_bb): Update.
	(propagate_freq): Update.
	(handle_missing_profiles): Update.
	(counts_to_freqs): Update.
	(rebuild_frequencies): Update.
	(force_edge_cold): Update.
	* predict.h: Include profile-count.h
	(maybe_hot_count_p, counts_to_freqs): UPdate.
	* print-rtl-function.c: Do not include cfg.h
	* print-rtl.c: Include basic-block.h
	* profile-count.c: New file.
	* profile-count.h: New file.
	* profile.c (is_edge_inconsistent): Update.
	(correct_negative_edge_counts): Update.
	(is_inconsistent): Update.
	(set_bb_counts): Update.
	(read_profile_edge_counts): Update.
	(compute_frequency_overlap): Update.
	(compute_branch_probabilities): Update; Initialize and deinitialize
	gcov_count tables.
	(branch_prob): Update.
	* profile.h (bb_gcov_counts, edge_gcov_counts): New.
	(edge_gcov_count): New.
	(bb_gcov_count): New.
	* shrink-wrap.c (try_shrink_wrapping): Update.
	* tracer.c (better_p): Update.
	* trans-mem.c (expand_transaction): Update.
	(ipa_tm_insert_irr_call): Update.
	(ipa_tm_insert_gettmclone_call): Update.
	* tree-call-cdce.c: Update.
	* tree-cfg.c (gimple_duplicate_sese_region): Update.
	(gimple_duplicate_sese_tail): Update.
	(gimple_account_profile_record): Update.
	(execute_fixup_cfg): Update.
	* tree-inline.c (copy_bb): Update.
	(copy_edges_for_bb): Update.
	(initialize_cfun): Update.
	(freqs_to_counts): Update.
	(copy_cfg_body): Update.
	(expand_call_inline): Update.
	* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
	* tree-ssa-loop-ivcanon.c (unloop_loops): Update.
	(try_unroll_loop_completely): Update.
	(try_peel_loop): Update.
	* tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
	* tree-ssa-loop-niter.c (estimate_numbers_of_iterations_loop): Update.
	* tree-ssa-loop-split.c (connect_loops): Update.
	* tree-ssa-loop-unswitch.c (hoist_guard): Update.
	* tree-ssa-reassoc.c (branch_fixup): Update.
	* tree-ssa-tail-merge.c (replace_block_by): Update.
	* tree-ssa-threadupdate.c (create_block_for_threading): Update.
	(compute_path_counts): Update.
	(update_profile): Update.
	(recompute_probabilities): Update.
	(update_joiner_offpath_counts): Update.
	(estimated_freqs_path): Update.
	(freqs_to_counts_path): Update.
	(clear_counts_path): Update.
	(ssa_fix_duplicate_block_edges): Update.
	(duplicate_thread_path): Update.
	* tree-switch-conversion.c (case_bit_test_cmp): Update.
	(struct switch_conv_info): Update.
	* tree-tailcall.c (decrease_profile): Update.
	* tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
	* tree-vect-loop.c (scale_profile_for_vect_loop): Update.
	* value-prof.c (check_counter): Update.
	(gimple_divmod_fixed_value): Update.
	(gimple_mod_pow2): Update.
	(gimple_mod_subtract): Update.
	(gimple_ic_transform): Update.
	(gimple_stringop_fixed_value): Update.
	* value-prof.h (gimple_ic): Update.

	* gcc.dg/tree-ssa/attr-hotcold-2.c: Update template.

From-SVN: r248863

1 files changed, 65 insertions, 33 deletions

diff --git a/gcc/cfg.c b/gcc/cfg.c
index aef053f..3cbe684 100644
--- a/gcc/cfg.c
+++ b/gcc/cfg.c
@@ -59,7 +59,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "dumpfile.h"
 
 
-#define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
 
 /* Called once at initialization time.  */
 
@@ -70,10 +69,10 @@ init_flow (struct function *the_fun)
     the_fun->cfg = ggc_cleared_alloc<control_flow_graph> ();
   n_edges_for_fn (the_fun) = 0;
   ENTRY_BLOCK_PTR_FOR_FN (the_fun)
-    = ggc_cleared_alloc<basic_block_def> ();
+    = alloc_block ();
   ENTRY_BLOCK_PTR_FOR_FN (the_fun)->index = ENTRY_BLOCK;
   EXIT_BLOCK_PTR_FOR_FN (the_fun)
-    = ggc_cleared_alloc<basic_block_def> ();
+    = alloc_block ();
   EXIT_BLOCK_PTR_FOR_FN (the_fun)->index = EXIT_BLOCK;
   ENTRY_BLOCK_PTR_FOR_FN (the_fun)->next_bb
     = EXIT_BLOCK_PTR_FOR_FN (the_fun);
@@ -123,6 +122,7 @@ alloc_block (void)
 {
   basic_block bb;
   bb = ggc_cleared_alloc<basic_block_def> ();
+  bb->count = profile_count::uninitialized ();
   return bb;
 }
 
@@ -263,6 +263,7 @@ unchecked_make_edge (basic_block src, basic_block dst, int flags)
   e = ggc_cleared_alloc<edge_def> ();
   n_edges_for_fn (cfun)++;
 
+  e->count = profile_count::uninitialized ();
   e->src = src;
   e->dest = dst;
   e->flags = flags;
@@ -400,7 +401,6 @@ check_bb_profile (basic_block bb, FILE * file, int indent)
 {
   edge e;
   int sum = 0;
-  gcov_type lsum;
   edge_iterator ei;
   struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl);
   char *s_indent = (char *) alloca ((size_t) indent + 1);
@@ -428,14 +428,18 @@ check_bb_profile (basic_block bb, FILE * file, int indent)
 	    fprintf (file,
 		     ";; %sInvalid sum of outgoing probabilities %.1f%%\n",
 		     s_indent, sum * 100.0 / REG_BR_PROB_BASE);
-	  lsum = 0;
+	  profile_count lsum = profile_count::zero ();
 	  FOR_EACH_EDGE (e, ei, bb->succs)
 	    lsum += e->count;
-	  if (EDGE_COUNT (bb->succs)
-	      && (lsum - bb->count > 100 || lsum - bb->count < -100))
-	    fprintf (file,
-		     ";; %sInvalid sum of outgoing counts %i, should be %i\n",
-		     s_indent, (int) lsum, (int) bb->count);
+	  if (EDGE_COUNT (bb->succs) && lsum.differs_from_p (bb->count))
+	    {
+	      fprintf (file, ";; %sInvalid sum of outgoing counts ",
+		       s_indent);
+	      lsum.dump (file);
+	      fprintf (file, ", should be ");
+	      bb->count.dump (file);
+	      fprintf (file, "\n");
+	    }
 	}
     }
     if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun))
@@ -447,12 +451,18 @@ check_bb_profile (basic_block bb, FILE * file, int indent)
 	fprintf (file,
 		 ";; %sInvalid sum of incoming frequencies %i, should be %i\n",
 		 s_indent, sum, bb->frequency);
-      lsum = 0;
+      profile_count lsum = profile_count::zero ();
       FOR_EACH_EDGE (e, ei, bb->preds)
 	lsum += e->count;
-      if (lsum - bb->count > 100 || lsum - bb->count < -100)
-	fprintf (file, ";; %sInvalid sum of incoming counts %i, should be %i\n",
-		 s_indent, (int) lsum, (int) bb->count);
+      if (lsum.differs_from_p (bb->count))
+	{
+	  fprintf (file, ";; %sInvalid sum of incoming counts ",
+		   s_indent);
+	  lsum.dump (file);
+	  fprintf (file, ", should be ");
+	  bb->count.dump (file);
+	  fprintf (file, "\n");
+	}
     }
   if (BB_PARTITION (bb) == BB_COLD_PARTITION)
     {
@@ -491,10 +501,10 @@ dump_edge_info (FILE *file, edge e, dump_flags_t flags, int do_succ)
   if (e->probability && do_details)
     fprintf (file, " [%.1f%%] ", e->probability * 100.0 / REG_BR_PROB_BASE);
 
-  if (e->count && do_details)
+  if (e->count.initialized_p () && do_details)
     {
       fputs (" count:", file);
-      fprintf (file, "%" PRId64, e->count);
+      e->count.dump (file);
     }
 
   if (e->flags && do_details)
@@ -741,8 +751,11 @@ dump_bb_info (FILE *outf, basic_block bb, int indent, dump_flags_t flags,
       if (flags & TDF_DETAILS)
 	{
 	  struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl);
-	  fprintf (outf, ", count " "%" PRId64,
-		   (int64_t) bb->count);
+	  if (bb->count.initialized_p ())
+	    {
+	      fputs (", count ", outf);
+	      bb->count.dump (outf);
+	    }
 	  fprintf (outf, ", freq %i", bb->frequency);
 	  if (maybe_hot_bb_p (fun, bb))
 	    fputs (", maybe hot", outf);
@@ -844,20 +857,19 @@ brief_dump_cfg (FILE *file, dump_flags_t flags)
    respectively.  */
 void
 update_bb_profile_for_threading (basic_block bb, int edge_frequency,
-				 gcov_type count, edge taken_edge)
+				 profile_count count, edge taken_edge)
 {
   edge c;
   int prob;
   edge_iterator ei;
 
-  bb->count -= count;
-  if (bb->count < 0)
+  if (bb->count < count)
     {
       if (dump_file)
 	fprintf (dump_file, "bb %i count became negative after threading",
 		 bb->index);
-      bb->count = 0;
     }
+  bb->count -= count;
 
   bb->frequency -= edge_frequency;
   if (bb->frequency < 0)
@@ -913,14 +925,13 @@ update_bb_profile_for_threading (basic_block bb, int edge_frequency,
     }
 
   gcc_assert (bb == taken_edge->src);
-  taken_edge->count -= count;
-  if (taken_edge->count < 0)
+  if (taken_edge->count < count)
     {
       if (dump_file)
 	fprintf (dump_file, "edge %i->%i count became negative after threading",
 		 taken_edge->src->index, taken_edge->dest->index);
-      taken_edge->count = 0;
     }
+  taken_edge->count -= count;
 }
 
 /* Multiply all frequencies of basic blocks in array BBS of length NBBS
@@ -954,9 +965,9 @@ scale_bbs_frequencies_int (basic_block *bbs, int nbbs, int num, int den)
       /* Make sure the frequencies do not grow over BB_FREQ_MAX.  */
       if (bbs[i]->frequency > BB_FREQ_MAX)
 	bbs[i]->frequency = BB_FREQ_MAX;
-      bbs[i]->count = RDIV (bbs[i]->count * num, den);
+      bbs[i]->count = bbs[i]->count.apply_scale (num, den);
       FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	e->count = RDIV (e->count * num, den);
+	e->count = e->count.apply_scale (num, den);
     }
 }
 
@@ -983,14 +994,14 @@ scale_bbs_frequencies_gcov_type (basic_block *bbs, int nbbs, gcov_type num,
 	edge_iterator ei;
 	bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
 	if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
-	  bbs[i]->count = RDIV (bbs[i]->count * num, den);
+	  bbs[i]->count = bbs[i]->count.apply_scale (num, den);
 	else
-	  bbs[i]->count = RDIV (bbs[i]->count * fraction, 65536);
+	  bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
 	FOR_EACH_EDGE (e, ei, bbs[i]->succs)
 	  if (bbs[i]->count <= MAX_SAFE_MULTIPLIER)
-	    e->count = RDIV (e->count * num, den);
+	    e->count =  e->count.apply_scale (num, den);
 	  else
-	    e->count = RDIV (e->count * fraction, 65536);
+	    e->count = e->count.apply_scale (fraction, 65536);
       }
    else
     for (i = 0; i < nbbs; i++)
@@ -1000,12 +1011,33 @@ scale_bbs_frequencies_gcov_type (basic_block *bbs, int nbbs, gcov_type num,
 	  bbs[i]->frequency = RDIV (bbs[i]->frequency * num, den);
 	else
 	  bbs[i]->frequency = RDIV (bbs[i]->frequency * fraction, 65536);
-	bbs[i]->count = RDIV (bbs[i]->count * fraction, 65536);
+	bbs[i]->count = bbs[i]->count.apply_scale (fraction, 65536);
 	FOR_EACH_EDGE (e, ei, bbs[i]->succs)
-	  e->count = RDIV (e->count * fraction, 65536);
+	  e->count = e->count.apply_scale (fraction, 65536);
       }
 }
 
+/* Multiply all frequencies of basic blocks in array BBS of length NBBS
+   by NUM/DEN, in profile_count arithmetic.  More accurate than previous
+   function but considerably slower.  */
+void
+scale_bbs_frequencies_profile_count (basic_block *bbs, int nbbs,
+				     profile_count num, profile_count den)
+{
+  int i;
+  edge e;
+
+  for (i = 0; i < nbbs; i++)
+    {
+      edge_iterator ei;
+      bbs[i]->frequency = RDIV (bbs[i]->frequency * num.to_gcov_type (),
+				den.to_gcov_type ());
+      bbs[i]->count = bbs[i]->count.apply_scale (num, den);
+      FOR_EACH_EDGE (e, ei, bbs[i]->succs)
+	e->count =  e->count.apply_scale (num, den);
+    }
+}
+
 /* Helper types for hash tables.  */
 
 struct htab_bb_copy_original_entry