i386.c (make_resolver_func): Update.

2017-05-23  Jan Hubicka  <hubicka@ucw.cz>

	* config/i386/i386.c (make_resolver_func): Update.
	* Makefile.in: Add profile-count.h and profile-count.o
	* auto-profile.c (afdo_indirect_call): Update to new API.
	(afdo_set_bb_count): Update.
	(afdo_propagate_edge): Update.
	(afdo_propagate_circuit): Update.
	(afdo_calculate_branch_prob): Update.
	(afdo_annotate_cfg): Update.
	* basic-block.h: Include profile-count.h
	(struct edge_def): Turn count to profile_count.
	(struct basic_block_def): Likewie.
	(REG_BR_PROB_BASE): Move to profile-count.h
	(RDIV): Move to profile-count.h
	* bb-reorder.c (max_entry_count): Turn to profile_count.
	(find_traces): Update.
	(rotate_loop):Update.
	(connect_traces):Update.
	(sanitize_hot_paths):Update.
	* bt-load.c (migrate_btr_defs): Update.
	* cfg.c (RDIV): Remove.
	(init_flow): Use alloc_block.
	(alloc_block): Uninitialize count.
	(unchecked_make_edge): Uninitialize count.
	(check_bb_profile): Update.
	(dump_edge_info): Update.
	(dump_bb_info): Update.
	(update_bb_profile_for_threading): Update.
	(scale_bbs_frequencies_int): Update.
	(scale_bbs_frequencies_gcov_type): Update.
	(scale_bbs_frequencies_profile_count): New.
	* cfg.h (update_bb_profile_for_threading): Update.
	(scale_bbs_frequencies_profile_count): Declare.
	* cfgbuild.c (compute_outgoing_frequencies): Update.
	(find_many_sub_basic_blocks): Update.
	* cfgcleanup.c (try_forward_edges): Update.
	(try_crossjump_to_edge): Update.
	* cfgexpand.c (expand_gimple_tailcall): Update.
	(construct_exit_block): Update.
	* cfghooks.c (verify_flow_info): Update.
	(dump_bb_for_graph): Update.
	(split_edge): Update.
	(make_forwarder_block): Update.
	(duplicate_block): Update.
	(account_profile_record): Update.
	* cfgloop.c (find_subloop_latch_edge_by_profile): Update.
	(get_estimated_loop_iterations): Update.
	* cfgloopanal.c (expected_loop_iterations_unbounded): Update.
	(single_likely_exit): Update.
	* cfgloopmanip.c (scale_loop_profile): Update.
	(loopify): Update.
	(set_zero_probability): Update.
	(lv_adjust_loop_entry_edge): Update.
	* cfgrtl.c (force_nonfallthru_and_redirect): Update.
	(purge_dead_edges): Update.
	(rtl_account_profile_record): Update.
	* cgraph.c (cgraph_node::create): Uninitialize count.
	(symbol_table::create_edge): Uninitialize count.
	(cgraph_update_edges_for_call_stmt_node): Update.
	(cgraph_edge::dump_edge_flags): Update.
	(cgraph_node::dump): Update.
	(cgraph_edge::maybe_hot_p): Update.
	* cgraph.h: Include profile-count.h
	(create_clone), create_edge, create_indirect_edge): Update.
	(cgraph_node): Turn count to profile_count.
	(cgraph_edge0: Likewise.
	(make_speculative, clone): Update.
	(create_edge): Update.
	(init_lowered_empty_function): Update.
	* cgraphclones.c (cgraph_edge::clone): Update.
	(duplicate_thunk_for_node): Update.
	(cgraph_node::create_clone): Update.
	* cgraphunit.c (cgraph_node::analyze): Update.
	(cgraph_node::expand_thunk): Update.
	* final.c (dump_basic_block_info): Update.
	* gimple-streamer-in.c (input_bb): Update.
	* gimple-streamer-out.c (output_bb): Update.
	* graphite.c (print_global_statistics): Update.
	(print_graphite_scop_statistics): Update.
	* hsa-brig.c: Include basic-block.h.
	* hsa-dump.c: Include basic-block.h.
	* hsa-gen.c (T sum_slice): Update.
	(convert_switch_statements):Update.
	* hsa-regalloc.c: Include basic-block.h.
	* ipa-chkp.c (chkp_produce_thunks): Update.
	* ipa-cp.c (struct caller_statistics): Update.
	(init_caller_stats): Update.
	(gather_caller_stats): Update.
	(ipcp_cloning_candidate_p): Update.
	(good_cloning_opportunity_p): Update.
	(get_info_about_necessary_edges): Update.
	(dump_profile_updates): Update.
	(update_profiling_info): Update.
	(update_specialized_profile): Update.
	(perhaps_add_new_callers): Update.
	(decide_about_value): Update.
	(ipa_cp_c_finalize): Update.
	* ipa-devirt.c (struct odr_type_warn_count): Update.
	(struct decl_warn_count): Update.
	(struct final_warning_record): Update.
	(possible_polymorphic_call_targets): Update.
	(ipa_devirt): Update.
	* ipa-fnsummary.c (redirect_to_unreachable): Update.
	* ipa-icf.c (sem_function::merge): Update.
	* ipa-inline-analysis.c (do_estimate_edge_time): Update.
	* ipa-inline.c (compute_uninlined_call_time): Update.
	(compute_inlined_call_time): Update.
	(want_inline_small_function_p): Update.
	(want_inline_self_recursive_call_p): Update.
	(edge_badness): Update.
	(lookup_recursive_calls): Update.
	(recursive_inlining): Update.
	(inline_small_functions): Update.
	(dump_overall_stats): Update.
	(dump_inline_stats): Update.
	* ipa-profile.c (ipa_profile_generate_summary): Update.
	(ipa_propagate_frequency): Update.
	(ipa_profile): Update.
	* ipa-prop.c (ipa_make_edge_direct_to_target): Update.
	* ipa-utils.c (ipa_merge_profiles): Update.
	* loop-doloop.c (doloop_modify): Update.
	* loop-unroll.c (report_unroll): Update.
	(unroll_loop_runtime_iterations): Update.
	* lto-cgraph.c (lto_output_edge): Update.
	(lto_output_node): Update.
	(input_node): Update.
	(input_edge): Update.
	(merge_profile_summaries): Update.
	* lto-streamer-in.c (input_cfg): Update.
	* lto-streamer-out.c (output_cfg): Update.
	* mcf.c (create_fixup_graph): Update.
	(adjust_cfg_counts): Update.
	(sum_edge_counts): Update.
	* modulo-sched.c (sms_schedule): Update.
	* postreload-gcse.c (eliminate_partially_redundant_load): Update.
	* predict.c (maybe_hot_count_p): Update.
	(probably_never_executed): Update.
	(dump_prediction): Update.
	(combine_predictions_for_bb): Update.
	(propagate_freq): Update.
	(handle_missing_profiles): Update.
	(counts_to_freqs): Update.
	(rebuild_frequencies): Update.
	(force_edge_cold): Update.
	* predict.h: Include profile-count.h
	(maybe_hot_count_p, counts_to_freqs): UPdate.
	* print-rtl-function.c: Do not include cfg.h
	* print-rtl.c: Include basic-block.h
	* profile-count.c: New file.
	* profile-count.h: New file.
	* profile.c (is_edge_inconsistent): Update.
	(correct_negative_edge_counts): Update.
	(is_inconsistent): Update.
	(set_bb_counts): Update.
	(read_profile_edge_counts): Update.
	(compute_frequency_overlap): Update.
	(compute_branch_probabilities): Update; Initialize and deinitialize
	gcov_count tables.
	(branch_prob): Update.
	* profile.h (bb_gcov_counts, edge_gcov_counts): New.
	(edge_gcov_count): New.
	(bb_gcov_count): New.
	* shrink-wrap.c (try_shrink_wrapping): Update.
	* tracer.c (better_p): Update.
	* trans-mem.c (expand_transaction): Update.
	(ipa_tm_insert_irr_call): Update.
	(ipa_tm_insert_gettmclone_call): Update.
	* tree-call-cdce.c: Update.
	* tree-cfg.c (gimple_duplicate_sese_region): Update.
	(gimple_duplicate_sese_tail): Update.
	(gimple_account_profile_record): Update.
	(execute_fixup_cfg): Update.
	* tree-inline.c (copy_bb): Update.
	(copy_edges_for_bb): Update.
	(initialize_cfun): Update.
	(freqs_to_counts): Update.
	(copy_cfg_body): Update.
	(expand_call_inline): Update.
	* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
	* tree-ssa-loop-ivcanon.c (unloop_loops): Update.
	(try_unroll_loop_completely): Update.
	(try_peel_loop): Update.
	* tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
	* tree-ssa-loop-niter.c (estimate_numbers_of_iterations_loop): Update.
	* tree-ssa-loop-split.c (connect_loops): Update.
	* tree-ssa-loop-unswitch.c (hoist_guard): Update.
	* tree-ssa-reassoc.c (branch_fixup): Update.
	* tree-ssa-tail-merge.c (replace_block_by): Update.
	* tree-ssa-threadupdate.c (create_block_for_threading): Update.
	(compute_path_counts): Update.
	(update_profile): Update.
	(recompute_probabilities): Update.
	(update_joiner_offpath_counts): Update.
	(estimated_freqs_path): Update.
	(freqs_to_counts_path): Update.
	(clear_counts_path): Update.
	(ssa_fix_duplicate_block_edges): Update.
	(duplicate_thread_path): Update.
	* tree-switch-conversion.c (case_bit_test_cmp): Update.
	(struct switch_conv_info): Update.
	* tree-tailcall.c (decrease_profile): Update.
	* tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
	* tree-vect-loop.c (scale_profile_for_vect_loop): Update.
	* value-prof.c (check_counter): Update.
	(gimple_divmod_fixed_value): Update.
	(gimple_mod_pow2): Update.
	(gimple_mod_subtract): Update.
	(gimple_ic_transform): Update.
	(gimple_stringop_fixed_value): Update.
	* value-prof.h (gimple_ic): Update.

	* gcc.dg/tree-ssa/attr-hotcold-2.c: Update template.

From-SVN: r248863

1 files changed, 72 insertions, 56 deletions

diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c
index 0ebe147..8900556 100644
--- a/gcc/ipa-inline.c
+++ b/gcc/ipa-inline.c
@@ -123,8 +123,8 @@ typedef fibonacci_node <sreal, cgraph_edge> edge_heap_node_t;
 
 /* Statistics we collect about inlining algorithm.  */
 static int overall_size;
-static gcov_type max_count;
-static gcov_type spec_rem;
+static profile_count max_count;
+static profile_count spec_rem;
 
 /* Pre-computed constants 1/CGRAPH_FREQ_BASE and 1/100. */
 static sreal cgraph_freq_base_rec, percent_rec;
@@ -637,8 +637,10 @@ compute_uninlined_call_time (struct cgraph_edge *edge,
 			 ? edge->caller->global.inlined_to
 			 : edge->caller);
 
-  if (edge->count && caller->count)
-    uninlined_call_time *= (sreal)edge->count / caller->count;
+  if (edge->count > profile_count::zero ()
+      && caller->count > profile_count::zero ())
+    uninlined_call_time *= (sreal)edge->count.to_gcov_type ()
+			   / caller->count.to_gcov_type ();
   if (edge->frequency)
     uninlined_call_time *= cgraph_freq_base_rec * edge->frequency;
   else
@@ -660,8 +662,9 @@ compute_inlined_call_time (struct cgraph_edge *edge,
 			 : edge->caller);
   sreal caller_time = ipa_fn_summaries->get (caller)->time;
 
-  if (edge->count && caller->count)
-    time *= (sreal)edge->count / caller->count;
+  if (edge->count > profile_count::zero ()
+      && caller->count > profile_count::zero ())
+    time *= (sreal)edge->count.to_gcov_type () / caller->count.to_gcov_type ();
   if (edge->frequency)
     time *= cgraph_freq_base_rec * edge->frequency;
   else
@@ -718,7 +721,7 @@ want_inline_small_function_p (struct cgraph_edge *e, bool report)
      promote non-inline functions to inline and we increase
      MAX_INLINE_INSNS_SINGLE 16-fold for inline functions.  */
   else if ((!DECL_DECLARED_INLINE_P (callee->decl)
-	   && (!e->count || !e->maybe_hot_p ()))
+	   && (!e->count.initialized_p () || !e->maybe_hot_p ()))
 	   && ipa_fn_summaries->get (callee)->min_size
 		- ipa_call_summaries->get (e)->call_stmt_size
 	      > MAX (MAX_INLINE_INSNS_SINGLE, MAX_INLINE_INSNS_AUTO))
@@ -726,7 +729,8 @@ want_inline_small_function_p (struct cgraph_edge *e, bool report)
       e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
       want_inline = false;
     }
-  else if ((DECL_DECLARED_INLINE_P (callee->decl) || e->count)
+  else if ((DECL_DECLARED_INLINE_P (callee->decl)
+	    || e->count > profile_count::zero ())
 	   && ipa_fn_summaries->get (callee)->min_size
 		- ipa_call_summaries->get (e)->call_stmt_size
 	      > 16 * MAX_INLINE_INSNS_SINGLE)
@@ -836,7 +840,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
       reason = "recursive call is cold";
       want_inline = false;
     }
-  else if (max_count && !outer_node->count)
+  else if (outer_node->count == profile_count::zero ())
     {
       reason = "not executed in profile";
       want_inline = false;
@@ -874,14 +878,15 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
       int i;
       for (i = 1; i < depth; i++)
 	max_prob = max_prob * max_prob / CGRAPH_FREQ_BASE;
-      if (max_count
-	  && (edge->count * CGRAPH_FREQ_BASE / outer_node->count
+      if (max_count > profile_count::zero () && edge->count > profile_count::zero ()
+	  && (edge->count.to_gcov_type () * CGRAPH_FREQ_BASE
+	      / outer_node->count.to_gcov_type ()
 	      >= max_prob))
 	{
 	  reason = "profile of recursive call is too large";
 	  want_inline = false;
 	}
-      if (!max_count
+      if (max_count == profile_count::zero ()
 	  && (edge->frequency * CGRAPH_FREQ_BASE / caller_freq
 	      >= max_prob))
 	{
@@ -907,14 +912,15 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge,
      methods.  */
   else
     {
-      if (max_count
-	  && (edge->count * 100 / outer_node->count
+      if (max_count > profile_count::zero ()
+	  && (edge->count.to_gcov_type () * 100
+	      / outer_node->count.to_gcov_type ()
 	      <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
 	{
 	  reason = "profile of recursive call is too small";
 	  want_inline = false;
 	}
-      else if (!max_count
+      else if (max_count == profile_count::zero ()
 	       && (edge->frequency * 100 / caller_freq
 	           <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY)))
 	{
@@ -1058,7 +1064,8 @@ edge_badness (struct cgraph_edge *edge, bool dump)
      Again use negative value to make calls with profile appear hotter
      then calls without.
   */
-  else if (opt_for_fn (caller->decl, flag_guess_branch_prob) || caller->count)
+  else if (opt_for_fn (caller->decl, flag_guess_branch_prob)
+	   || caller->count > profile_count::zero ())
     {
       sreal numerator, denominator;
       int overall_growth;
@@ -1068,8 +1075,8 @@ edge_badness (struct cgraph_edge *edge, bool dump)
 		   - inlined_time);
       if (numerator == 0)
 	numerator = ((sreal) 1 >> 8);
-      if (caller->count)
-	numerator *= caller->count;
+      if (caller->count > profile_count::zero ())
+	numerator *= caller->count.to_gcov_type ();
       else if (opt_for_fn (caller->decl, flag_branch_probabilities))
 	numerator = numerator >> 11;
       denominator = growth;
@@ -1155,7 +1162,8 @@ edge_badness (struct cgraph_edge *edge, bool dump)
 		   " %i (compensated)\n",
 		   badness.to_double (),
 		  (double)edge->frequency / CGRAPH_FREQ_BASE,
-		   edge->count, caller->count,
+		   edge->count.initialized_p () ? edge->count.to_gcov_type () : -1,
+		   caller->count.initialized_p () ? caller->count.to_gcov_type () : -1,
 		   compute_uninlined_call_time (edge,
 						unspec_edge_time).to_double (),
 		   compute_inlined_call_time (edge, edge_time).to_double (),
@@ -1417,8 +1425,10 @@ lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where,
       {
 	/* When profile feedback is available, prioritize by expected number
 	   of calls.  */
-        heap->insert (!max_count ? -e->frequency
-		      : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))),
+        heap->insert (!(max_count > 0) || !e->count.initialized_p () ? -e->frequency
+		      : -(e->count.to_gcov_type ()
+			/ ((max_count.to_gcov_type () + (1<<24) - 1)
+			   / (1<<24))),
 		      e);
       }
   for (e = where->callees; e; e = e->next_callee)
@@ -1506,10 +1516,11 @@ recursive_inlining (struct cgraph_edge *edge,
 	{
 	  fprintf (dump_file,
 		   "   Inlining call of depth %i", depth);
-	  if (node->count)
+	  if (node->count > profile_count::zero ())
 	    {
 	      fprintf (dump_file, " called approx. %.2f times per call",
-		       (double)curr->count / node->count);
+		       (double)curr->count.to_gcov_type ()
+		       / node->count.to_gcov_type ());
 	    }
 	  fprintf (dump_file, "\n");
 	}
@@ -1731,7 +1742,7 @@ inline_small_functions (void)
   /* Compute overall unit size and other global parameters used by badness
      metrics.  */
 
-  max_count = 0;
+  max_count = profile_count::uninitialized ();
   ipa_reduced_postorder (order, true, true, NULL);
   free (order);
 
@@ -1771,7 +1782,7 @@ inline_small_functions (void)
 	  }
 
 	for (edge = node->callers; edge; edge = edge->next_caller)
-	  if (max_count < edge->count)
+	  if (!(max_count >= edge->count))
 	    max_count = edge->count;
       }
   ipa_free_postorder_info ();
@@ -1835,7 +1846,7 @@ inline_small_functions (void)
     }
 
   gcc_assert (in_lto_p
-	      || !max_count
+	      || !(max_count > 0)
 	      || (profile_info && flag_branch_probabilities));
 
   while (!edge_heap.empty ())
@@ -1880,9 +1891,7 @@ inline_small_functions (void)
 	 Increases of badness are handled lazilly; when we see key with out
 	 of date value on it, we re-insert it now.  */
       current_badness = edge_badness (edge, false);
-      /* Disable checking for profile because roundoff errors may cause slight
-         deviations in the order.  */
-      gcc_assert (max_count || cached_badness == current_badness);
+      gcc_assert (cached_badness == current_badness);
       gcc_assert (current_badness >= badness);
 #else
       current_badness = edge_badness (edge, false);
@@ -1927,9 +1936,12 @@ inline_small_functions (void)
 		   : -1,
 		   badness.to_double (),
 		   edge->frequency / (double)CGRAPH_FREQ_BASE);
-	  if (edge->count)
-	    fprintf (dump_file," Called %" PRId64"x\n",
-		     edge->count);
+	  if (edge->count.initialized_p ())
+	    {
+	      fprintf (dump_file, " Called ");
+	      edge->count.dump (dump_file);
+	      fprintf (dump_file, "times\n");
+            }
 	  if (dump_flags & TDF_DETAILS)
 	    edge_badness (edge, true);
 	}
@@ -2027,7 +2039,7 @@ inline_small_functions (void)
       update_caller_keys (&edge_heap, where, updated_nodes, NULL);
       /* Offline copy count has possibly changed, recompute if profile is
 	 available.  */
-      if (max_count)
+      if (max_count > profile_count::zero ())
         {
 	  struct cgraph_node *n = cgraph_node::get (edge->callee->decl);
 	  if (n != edge->callee && n->analyzed)
@@ -2233,7 +2245,8 @@ dump_overall_stats (void)
       {
 	sreal time = ipa_fn_summaries->get (node)->time;
 	sum += time;
-	sum_weighted += time * node->count;
+	if (node->count.initialized_p ())
+	  sum_weighted += time * node->count.to_gcov_type ();
       }
   fprintf (dump_file, "Overall time estimate: "
 	   "%f weighted by profile: "
@@ -2263,56 +2276,59 @@ dump_inline_stats (void)
       {
 	if (e->inline_failed)
 	  {
-	    reason[(int) e->inline_failed][0] += e->count;
+	    if (e->count.initialized_p ())
+	      reason[(int) e->inline_failed][0] += e->count.to_gcov_type ();
 	    reason[(int) e->inline_failed][1] += e->frequency;
 	    reason[(int) e->inline_failed][2] ++;
-	    if (DECL_VIRTUAL_P (e->callee->decl))
+	    if (DECL_VIRTUAL_P (e->callee->decl)
+		&& e->count.initialized_p ())
 	      {
 		if (e->indirect_inlining_edge)
-		  noninlined_virt_indir_cnt += e->count;
+		  noninlined_virt_indir_cnt += e->count.to_gcov_type ();
 		else
-		  noninlined_virt_cnt += e->count;
+		  noninlined_virt_cnt += e->count.to_gcov_type ();
 	      }
-	    else
+	    else if (e->count.initialized_p ())
 	      {
 		if (e->indirect_inlining_edge)
-		  noninlined_indir_cnt += e->count;
+		  noninlined_indir_cnt += e->count.to_gcov_type ();
 		else
-		  noninlined_cnt += e->count;
+		  noninlined_cnt += e->count.to_gcov_type ();
 	      }
 	  }
-	else
+	else if (e->count.initialized_p ())
 	  {
 	    if (e->speculative)
 	      {
 		if (DECL_VIRTUAL_P (e->callee->decl))
-		  inlined_speculative_ply += e->count;
+		  inlined_speculative_ply += e->count.to_gcov_type ();
 		else
-		  inlined_speculative += e->count;
+		  inlined_speculative += e->count.to_gcov_type ();
 	      }
 	    else if (DECL_VIRTUAL_P (e->callee->decl))
 	      {
 		if (e->indirect_inlining_edge)
-		  inlined_virt_indir_cnt += e->count;
+		  inlined_virt_indir_cnt += e->count.to_gcov_type ();
 		else
-		  inlined_virt_cnt += e->count;
+		  inlined_virt_cnt += e->count.to_gcov_type ();
 	      }
 	    else
 	      {
 		if (e->indirect_inlining_edge)
-		  inlined_indir_cnt += e->count;
+		  inlined_indir_cnt += e->count.to_gcov_type ();
 		else
-		  inlined_cnt += e->count;
+		  inlined_cnt += e->count.to_gcov_type ();
 	      }
 	  }
       }
     for (e = node->indirect_calls; e; e = e->next_callee)
-      if (e->indirect_info->polymorphic)
-	indirect_poly_cnt += e->count;
-      else
-	indirect_cnt += e->count;
+      if (e->indirect_info->polymorphic
+	  & e->count.initialized_p ())
+	indirect_poly_cnt += e->count.to_gcov_type ();
+      else if (e->count.initialized_p ())
+	indirect_cnt += e->count.to_gcov_type ();
   }
-  if (max_count)
+  if (max_count.initialized_p ())
     {
       fprintf (dump_file,
 	       "Inlined %" PRId64 " + speculative "
@@ -2331,9 +2347,9 @@ dump_inline_stats (void)
 	       inlined_indir_cnt, inlined_virt_cnt, inlined_virt_indir_cnt,
 	       noninlined_cnt, noninlined_indir_cnt, noninlined_virt_cnt,
 	       noninlined_virt_indir_cnt, indirect_cnt, indirect_poly_cnt);
-      fprintf (dump_file,
-	       "Removed speculations %" PRId64 "\n",
-	       spec_rem);
+      fprintf (dump_file, "Removed speculations ");
+      spec_rem.dump (dump_file);
+      fprintf (dump_file, "\n");
     }
   dump_overall_stats ();
   fprintf (dump_file, "\nWhy inlining failed?\n");