asan.c (create_cond_insert_point): Do not update edge count.

	* asan.c (create_cond_insert_point): Do not update edge count.
	* auto-profile.c (afdo_propagate_edge): Update for edge count removal.
	(afdo_propagate_circuit): Likewise.
	(afdo_calculate_branch_prob): Likewise.
	(afdo_annotate_cfg): Likewise.
	* basic-block.h (struct edge_def): Remove count.
	(edge_def::count): New accessor.
	* bb-reorder.c (rotate_loop): Update.
	(find_traces_1_round): Update.
	(connect_traces): Update.
	(sanitize_hot_paths): Update.
	* cfg.c (unchecked_make_edge): Update.
	(make_single_succ_edge): Update.
	(check_bb_profile): Update.
	(dump_edge_info): Update.
	(update_bb_profile_for_threading): Update.
	(scale_bbs_frequencies_int): Update.
	(scale_bbs_frequencies_gcov_type): Update.
	(scale_bbs_frequencies_profile_count): Update.
	(scale_bbs_frequencies): Update.
	* cfganal.c (connect_infinite_loops_to_exit): Update.
	* cfgbuild.c (compute_outgoing_frequencies): Update.
	(find_many_sub_basic_blocks): Update.
	* cfgcleanup.c (try_forward_edges): Update.
	(try_crossjump_to_edge): Update
	* cfgexpand.c (expand_gimple_cond): Update
	(expand_gimple_tailcall): Update
	(construct_exit_block): Update
	* cfghooks.c (verify_flow_info): Update
	(redirect_edge_succ_nodup): Update
	(split_edge): Update
	(make_forwarder_block): Update
	(duplicate_block): Update
	(account_profile_record): Update
	* cfgloop.c (find_subloop_latch_edge_by_profile): Update.
	* cfgloopanal.c (expected_loop_iterations_unbounded): Update.
	* cfgloopmanip.c (scale_loop_profile): Update.
	(loopify): Update.
	(lv_adjust_loop_entry_edge): Update.
	* cfgrtl.c (try_redirect_by_replacing_jump): Update.
	(force_nonfallthru_and_redirect): Update.
	(purge_dead_edges): Update.
	(rtl_flow_call_edges_add): Update.
	* cgraphunit.c (init_lowered_empty_function): Update.
	(cgraph_node::expand_thunk): Update.
	* gimple-pretty-print.c (dump_probability): Update.
	(dump_edge_probability): Update.
	* gimple-ssa-isolate-paths.c (isolate_path): Update.
	* haifa-sched.c (sched_create_recovery_edges): Update.
	* hsa-gen.c (convert_switch_statements): Update.
	* ifcvt.c (dead_or_predicable): Update.
	* ipa-inline-transform.c (inline_transform): Update.
	* ipa-split.c (split_function): Update.
	* ipa-utils.c (ipa_merge_profiles): Update.
	* loop-doloop.c (add_test): Update.
	* loop-unroll.c (unroll_loop_runtime_iterations): Update.
	* lto-streamer-in.c (input_cfg): Update.
	(input_function): Update.
	* lto-streamer-out.c (output_cfg): Update.
	* modulo-sched.c (sms_schedule): Update.
	* postreload-gcse.c (eliminate_partially_redundant_load): Update.
	* predict.c (maybe_hot_edge_p): Update.
	(unlikely_executed_edge_p): Update.
	(probably_never_executed_edge_p): Update.
	(dump_prediction): Update.
	(drop_profile): Update.
	(propagate_unlikely_bbs_forward): Update.
	(determine_unlikely_bbs): Update.
	(force_edge_cold): Update.
	* profile.c (compute_branch_probabilities): Update.
	* reg-stack.c (better_edge): Update.
	* shrink-wrap.c (handle_simple_exit): Update.
	* tracer.c (better_p): Update.
	* trans-mem.c (expand_transaction): Update.
	(split_bb_make_tm_edge): Update.
	* tree-call-cdce.c: Update.
	* tree-cfg.c (gimple_find_sub_bbs): Update.
	(gimple_split_edge): Update.
	(gimple_duplicate_sese_region): Update.
	(gimple_duplicate_sese_tail): Update.
	(gimple_flow_call_edges_add): Update.
	(insert_cond_bb): Update.
	(execute_fixup_cfg): Update.
	* tree-cfgcleanup.c (cleanup_control_expr_graph): Update.
	* tree-complex.c (expand_complex_div_wide): Update.
	* tree-eh.c (lower_resx): Update.
	(unsplit_eh): Update.
	(cleanup_empty_eh_move_lp): Update.
	* tree-inline.c (copy_edges_for_bb): Update.
	(freqs_to_counts): Update.
	(copy_cfg_body): Update.
	* tree-ssa-dce.c (remove_dead_stmt): Update.
	* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
	* tree-ssa-loop-im.c (execute_sm_if_changed): Update.
	* tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update.
	(unloop_loops): Update.
	* tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
	* tree-ssa-loop-split.c (connect_loops): Update.
	(split_loop): Update.
	* tree-ssa-loop-unswitch.c (hoist_guard): Update.
	* tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update.
	* tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update.
	* tree-ssa-reassoc.c (branch_fixup): Update.
	* tree-ssa-tail-merge.c (replace_block_by): Update.
	* tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update.
	(compute_path_counts): Update.
	(update_profile): Update.
	(recompute_probabilities): Update.
	(update_joiner_offpath_counts): Update.
	(estimated_freqs_path): Update.
	(freqs_to_counts_path): Update.
	(clear_counts_path): Update.
	(ssa_fix_duplicate_block_edges): Update.
	(duplicate_thread_path): Update.
	* tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update.
	(case_bit_test_cmp): Update.
	(collect_switch_conv_info): Update.
	(gen_inbound_check): Update.
	(do_jump_if_equal): Update.
	(emit_cmp_and_jump_insns): Update.
	* tree-tailcall.c (decrease_profile): Update.
	(eliminate_tail_call): Update.
	* tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
	(vect_do_peeling): Update.
	* tree-vect-loop.c (scale_profile_for_vect_loop): Update.
	* ubsan.c (ubsan_expand_null_ifn): Update.
	(ubsan_expand_ptr_ifn): Update.
	* value-prof.c (gimple_divmod_fixed_value): Update.
	(gimple_mod_pow2): Update.
	(gimple_mod_subtract): Update.
	(gimple_ic): Update.
	(gimple_stringop_fixed_value): Update.

From-SVN: r253910

1 files changed, 83 insertions, 162 deletions

diff --git a/gcc/tree-ssa-threadupdate.c b/gcc/tree-ssa-threadupdate.c
index 28c81a6..8681707 100644
--- a/gcc/tree-ssa-threadupdate.c
+++ b/gcc/tree-ssa-threadupdate.c
@@ -303,7 +303,6 @@ remove_ctrl_stmt_and_useless_edges (basic_block bb, basic_block dest_bb)
       else
 	{
 	  e->probability = profile_probability::always ();
-	  e->count = bb->count;
 	  ei_next (&ei);
 	}
     }
@@ -741,7 +740,7 @@ compute_path_counts (struct redirection_data *rd,
 	     same last path edge in the case where the last edge has a nocopy
 	     source block.  */
 	  gcc_assert (ein_path->last ()->e == elast);
-	  path_in_count += ein->count;
+	  path_in_count += ein->count ();
 	  path_in_freq += EDGE_FREQUENCY (ein);
 	}
       else if (!ein_path)
@@ -749,7 +748,7 @@ compute_path_counts (struct redirection_data *rd,
 	  /* Keep track of the incoming edges that are not on any jump-threading
 	     path.  These counts will still flow out of original path after all
 	     jump threading is complete.  */
-	    nonpath_count += ein->count;
+	    nonpath_count += ein->count ();
 	}
     }
 
@@ -789,7 +788,7 @@ compute_path_counts (struct redirection_data *rd,
   for (unsigned int i = 1; i < path->length (); i++)
     {
       edge epath = (*path)[i]->e;
-      profile_count cur_count = epath->count;
+      profile_count cur_count = epath->count ();
       if ((*path)[i]->type == EDGE_COPY_SRC_JOINER_BLOCK)
 	{
 	  has_joiner = true;
@@ -809,13 +808,13 @@ compute_path_counts (struct redirection_data *rd,
 		     they are redirected by an invocation of this routine.  */
 		  && !bitmap_bit_p (local_info->duplicate_blocks,
 				    ein->src->index))
-		nonpath_count += ein->count;
+		nonpath_count += ein->count ();
 	    }
 	}
       if (cur_count < path_out_count)
 	path_out_count = cur_count;
-      if (epath->count < min_path_count)
-	min_path_count = epath->count;
+      if (epath->count () < min_path_count)
+	min_path_count = epath->count ();
     }
 
   /* We computed path_out_count above assuming that this path targeted
@@ -830,12 +829,12 @@ compute_path_counts (struct redirection_data *rd,
      (since any path through the joiner with a different elast will not
      include a copy of this elast in its duplicated path).
      So ensure that this path's path_out_count is at least the
-     difference between elast->count and nonpath_count.  Otherwise the edge
+     difference between elast->count () and nonpath_count.  Otherwise the edge
      counts after threading will not be sane.  */
   if (local_info->need_profile_correction
-      && has_joiner && path_out_count < elast->count - nonpath_count)
+      && has_joiner && path_out_count < elast->count () - nonpath_count)
     {
-      path_out_count = elast->count - nonpath_count;
+      path_out_count = elast->count () - nonpath_count;
       /* But neither can we go above the minimum count along the path
 	 we are duplicating.  This can be an issue due to profile
 	 insanities coming in to this pass.  */
@@ -858,17 +857,54 @@ static void
 update_profile (edge epath, edge edup, profile_count path_in_count,
 		profile_count path_out_count, int path_in_freq)
 {
+  if (!(path_in_count > 0))
+    return;
 
   /* First update the duplicated block's count / frequency.  */
   if (edup)
     {
       basic_block dup_block = edup->src;
-      gcc_assert (!dup_block->count.initialized_p ());
+
+      /* Edup's count is reduced by path_out_count.  We need to redistribute
+         probabilities to the remaining edges.  */
+
+      edge esucc;
+      edge_iterator ei;
+      profile_probability edup_prob
+	 = path_out_count.probability_in (path_in_count);
+
+      /* Either scale up or down the remaining edges.
+	 probabilities are always in range <0,1> and thus we can't do
+	 both by same loop.  */
+      if (edup->probability > edup_prob)
+	{
+	   profile_probability rev_scale
+	     = (profile_probability::always () - edup->probability)
+	       / (profile_probability::always () - edup_prob);
+	   FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+	     if (esucc != edup)
+	       esucc->probability /= rev_scale;
+	}
+      else if (edup->probability < edup_prob)
+	{
+	   profile_probability scale
+	     = (profile_probability::always () - edup_prob)
+	       / (profile_probability::always () - edup->probability);
+	  FOR_EACH_EDGE (esucc, ei, dup_block->succs)
+	    if (esucc != edup)
+	      esucc->probability *= scale;
+	}
+      edup->probability = edup_prob;
+
+      /* FIXME once freqs_to_counts is dropped re-enable this check.  */
+      gcc_assert (!dup_block->count.initialized_p () || 1);
       gcc_assert (dup_block->frequency == 0);
       dup_block->count = path_in_count;
       dup_block->frequency = path_in_freq;
     }
 
+  profile_count final_count = epath->count () - path_out_count;
+
   /* Now update the original block's count and frequency in the
      opposite manner - remove the counts/freq that will flow
      into the duplicated block.  Handle underflow due to precision/
@@ -883,107 +919,31 @@ update_profile (edge epath, edge edup, profile_count path_in_count,
      out of it (in the joiner case this is the count along the duplicated path
      out of the duplicated joiner).  This count can then be removed from the
      original path edge.  */
-  if (edup)
-    edup->count = path_out_count;
-  epath->count -= path_out_count;
-  /* FIXME: can epath->count be legally uninitialized here?  */
-}
-
-
-/* The duplicate and original joiner blocks may end up with different
-   probabilities (different from both the original and from each other).
-   Recompute the probabilities here once we have updated the edge
-   counts and frequencies.  */
-
-static void
-recompute_probabilities (basic_block bb)
-{
-  edge esucc;
-  edge_iterator ei;
-  FOR_EACH_EDGE (esucc, ei, bb->succs)
-    {
-      if (!(bb->count > 0))
-	continue;
-
-      /* Prevent overflow computation due to insane profiles.  */
-      if (esucc->count < bb->count)
-	esucc->probability = esucc->count.probability_in (bb->count).guessed ();
-      else
-	/* Can happen with missing/guessed probabilities, since we
-	   may determine that more is flowing along duplicated
-	   path than joiner succ probabilities allowed.
-	   Counts and freqs will be insane after jump threading,
-	   at least make sure probability is sane or we will
-	   get a flow verification error.
-	   Not much we can do to make counts/freqs sane without
-	   redoing the profile estimation.  */
-	esucc->probability = profile_probability::guessed_always ();
-    }
-}
-
-
-/* Update the counts of the original and duplicated edges from a joiner
-   that go off path, given that we have already determined that the
-   duplicate joiner DUP_BB has incoming count PATH_IN_COUNT and
-   outgoing count along the path PATH_OUT_COUNT.  The original (on-)path
-   edge from joiner is EPATH.  */
-
-static void
-update_joiner_offpath_counts (edge epath, basic_block dup_bb,
-			      profile_count path_in_count,
-			      profile_count path_out_count)
-{
-  /* Compute the count that currently flows off path from the joiner.
-     In other words, the total count of joiner's out edges other than
-     epath.  Compute this by walking the successors instead of
-     subtracting epath's count from the joiner bb count, since there
-     are sometimes slight insanities where the total out edge count is
-     larger than the bb count (possibly due to rounding/truncation
-     errors).  */
-  profile_count total_orig_off_path_count = profile_count::zero ();
-  edge enonpath;
-  edge_iterator ei;
-  FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
+  if (epath->src->count > 0)
     {
-      if (enonpath == epath)
-	continue;
-      total_orig_off_path_count += enonpath->count;
-    }
-
-  /* For the path that we are duplicating, the amount that will flow
-     off path from the duplicated joiner is the delta between the
-     path's cumulative in count and the portion of that count we
-     estimated above as flowing from the joiner along the duplicated
-     path.  */
-  profile_count total_dup_off_path_count = path_in_count - path_out_count;
-
-  /* Now do the actual updates of the off-path edges.  */
-  FOR_EACH_EDGE (enonpath, ei, epath->src->succs)
-    {
-      /* Look for edges going off of the threading path.  */
-      if (enonpath == epath)
-	continue;
+      edge esucc;
+      edge_iterator ei;
+      profile_probability epath_prob = final_count.probability_in (epath->src->count);
 
-      /* Find the corresponding edge out of the duplicated joiner.  */
-      edge enonpathdup = find_edge (dup_bb, enonpath->dest);
-      gcc_assert (enonpathdup);
-
-      /* We can't use the original probability of the joiner's out
-	 edges, since the probabilities of the original branch
-	 and the duplicated branches may vary after all threading is
-	 complete.  But apportion the duplicated joiner's off-path
-	 total edge count computed earlier (total_dup_off_path_count)
-	 among the duplicated off-path edges based on their original
-	 ratio to the full off-path count (total_orig_off_path_count).
-	 */
-      profile_probability scale
-		 = enonpath->count.probability_in (total_orig_off_path_count);
-      /* Give the duplicated offpath edge a portion of the duplicated
-	 total.  */
-      enonpathdup->count = total_dup_off_path_count.apply_probability (scale);
-      /* Now update the original offpath edge count, handling underflow
-	 due to rounding errors.  */
-      enonpath->count -= enonpathdup->count;
+      if (epath->probability > epath_prob)
+	{
+	   profile_probability rev_scale
+	     = (profile_probability::always () - epath->probability)
+	       / (profile_probability::always () - epath_prob);
+	   FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+	     if (esucc != epath)
+	       esucc->probability /= rev_scale;
+	}
+      else if (epath->probability < epath_prob)
+	{
+	   profile_probability scale
+	     = (profile_probability::always () - epath_prob)
+	       / (profile_probability::always () - epath->probability);
+	  FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+	    if (esucc != epath)
+	      esucc->probability *= scale;
+	}
+      epath->probability = epath_prob;
     }
 }
 
@@ -1002,7 +962,7 @@ estimated_freqs_path (struct redirection_data *rd)
   bool non_zero_freq = false;
   FOR_EACH_EDGE (ein, ei, e->dest->preds)
     {
-      if (ein->count > 0)
+      if (ein->count () > 0)
 	return false;
       non_zero_freq |= ein->src->frequency != 0;
     }
@@ -1016,7 +976,7 @@ estimated_freqs_path (struct redirection_data *rd)
       edge esucc;
       FOR_EACH_EDGE (esucc, ei, epath->src->succs)
 	{
-	  if (esucc->count > 0)
+	  if (esucc->count () > 0)
 	    return false;
 	  non_zero_freq |= esucc->src->frequency != 0;
 	}
@@ -1042,34 +1002,19 @@ freqs_to_counts_path (struct redirection_data *rd)
   vec<jump_thread_edge *> *path = THREAD_PATH (e);
   edge ein;
   edge_iterator ei;
-  FOR_EACH_EDGE (ein, ei, e->dest->preds)
-    {
-      /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
-	 errors applying the probability when the frequencies are very
-	 small.  */
-      if (ein->probability.initialized_p ())
-        ein->count = profile_count::from_gcov_type
-		  (apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
-				        ein->probability
-					  .to_reg_br_prob_base ())).guessed ();
-      else
-	/* FIXME: this is hack; we should track uninitialized values.  */
-	ein->count = profile_count::zero ();
-    }
 
+  FOR_EACH_EDGE (ein, ei, e->dest->preds)
+    ein->src->count = profile_count::from_gcov_type
+	  (ein->src->frequency * REG_BR_PROB_BASE);
   for (unsigned int i = 1; i < path->length (); i++)
     {
       edge epath = (*path)[i]->e;
-      edge esucc;
       /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
 	 errors applying the edge probability when the frequencies are very
 	 small.  */
       epath->src->count = 
 	profile_count::from_gcov_type
 	  (epath->src->frequency * REG_BR_PROB_BASE);
-      FOR_EACH_EDGE (esucc, ei, epath->src->succs)
-	esucc->count = 
-	   esucc->src->count.apply_probability (esucc->probability);
     }
 }
 
@@ -1086,21 +1031,20 @@ clear_counts_path (struct redirection_data *rd)
 {
   edge e = rd->incoming_edges->e;
   vec<jump_thread_edge *> *path = THREAD_PATH (e);
-  edge ein, esucc;
-  edge_iterator ei;
   profile_count val = profile_count::uninitialized ();
   if (profile_status_for_fn (cfun) == PROFILE_READ)
     val = profile_count::zero ();
 
+  edge ein;
+  edge_iterator ei;
+
   FOR_EACH_EDGE (ein, ei, e->dest->preds)
-    ein->count = val;
+    ein->src->count = val;
 
   /* First clear counts along original path.  */
   for (unsigned int i = 1; i < path->length (); i++)
     {
       edge epath = (*path)[i]->e;
-      FOR_EACH_EDGE (esucc, ei, epath->src->succs)
-	esucc->count = val;
       epath->src->count = val;
     }
   /* Also need to clear the counts along duplicated path.  */
@@ -1109,8 +1053,6 @@ clear_counts_path (struct redirection_data *rd)
       basic_block dup = rd->dup_blocks[i];
       if (!dup)
 	continue;
-      FOR_EACH_EDGE (esucc, ei, dup->succs)
-	esucc->count = val;
       dup->count = val;
     }
 }
@@ -1230,17 +1172,6 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	  update_profile (epath, e2, path_in_count, path_out_count,
 			  path_in_freq);
 
-	  /* Next we need to update the counts of the original and duplicated
-	     edges from the joiner that go off path.  */
-	  update_joiner_offpath_counts (epath, e2->src, path_in_count,
-					path_out_count);
-
-	  /* Finally, we need to set the probabilities on the duplicated
-	     edges out of the duplicated joiner (e2->src).  The probabilities
-	     along the original path will all be updated below after we finish
-	     processing the whole path.  */
-	  recompute_probabilities (e2->src);
-
 	  /* Record the frequency flowing to the downstream duplicated
 	     path blocks.  */
 	  cur_path_freq = EDGE_FREQUENCY (e2);
@@ -1263,8 +1194,7 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	     been updated at the end of that handling to the edge frequency
 	     along the duplicated joiner path edge.  */
 	  update_profile (epath, EDGE_SUCC (rd->dup_blocks[count], 0),
-			  path_out_count, path_out_count,
-			  cur_path_freq);
+			  path_out_count, path_out_count, cur_path_freq);
 	}
       else
 	{
@@ -1294,14 +1224,6 @@ ssa_fix_duplicate_block_edges (struct redirection_data *rd,
 	}
     }
 
-  /* Now walk orig blocks and update their probabilities, since the
-     counts and freqs should be updated properly by above loop.  */
-  for (unsigned int i = 1; i < path->length (); i++)
-    {
-      edge epath = (*path)[i]->e;
-      recompute_probabilities (epath->src);
-    }
-
   /* Done with all profile and frequency updates, clear counts if they
      were copied.  */
   if (do_freqs_to_counts)
@@ -2247,7 +2169,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
      invalidating the property that is propagated by executing all the blocks of
      the jump-thread path in order.  */
 
-  curr_count = entry->count;
+  curr_count = entry->count ();
   curr_freq = EDGE_FREQUENCY (entry);
 
   for (i = 0; i < n_region; i++)
@@ -2300,7 +2222,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
 	  if (i + 1 != n_region)
 	    {
 	      curr_freq = EDGE_FREQUENCY (single_succ_edge (bb));
-	      curr_count = single_succ_edge (bb)->count;
+	      curr_count = single_succ_edge (bb)->count ();
 	    }
 	  continue;
 	}
@@ -2331,7 +2253,7 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
 	else
 	  {
 	    curr_freq = EDGE_FREQUENCY (e);
-	    curr_count = e->count;
+	    curr_count = e->count ();
 	  }
     }
 
@@ -2353,7 +2275,6 @@ duplicate_thread_path (edge entry, edge exit, basic_block *region,
     {
       rescan_loop_exit (e, true, false);
       e->probability = profile_probability::always ();
-      e->count = region_copy[n_region - 1]->count;
     }
 
   /* Redirect the entry and add the phi node arguments.  */