aboutsummaryrefslogtreecommitdiff
path: root/gcc/auto-profile.cc
diff options
context:
space:
mode:
Diffstat (limited to 'gcc/auto-profile.cc')
-rw-r--r--gcc/auto-profile.cc322
1 files changed, 224 insertions, 98 deletions
diff --git a/gcc/auto-profile.cc b/gcc/auto-profile.cc
index a970eb8..5226e455 100644
--- a/gcc/auto-profile.cc
+++ b/gcc/auto-profile.cc
@@ -53,6 +53,7 @@ along with GCC; see the file COPYING3. If not see
#include "auto-profile.h"
#include "tree-pretty-print.h"
#include "gimple-pretty-print.h"
+#include "output.h"
/* The following routines implements AutoFDO optimization.
@@ -430,7 +431,8 @@ public:
void
set_call_location (location_t l)
{
- gcc_checking_assert (call_location_ == UNKNOWN_LOCATION);
+ gcc_checking_assert (call_location_ == UNKNOWN_LOCATION
+ && l != UNKNOWN_LOCATION);
call_location_= l;
}
@@ -622,9 +624,11 @@ get_original_name (const char *name, bool alloc = true)
}
/* Suffixes of clones that compiler generates after auto-profile. */
const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"};
- for (unsigned i = 0; i < sizeof (suffixes); ++i)
+ for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i)
{
- if (strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
+ int len = strlen (suffixes[i]);
+ if (len == last_dot - next_dot - 1
+ && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
{
*next_dot = 0;
return get_original_name (ret, false);
@@ -683,6 +687,26 @@ dump_afdo_loc (FILE *f, unsigned loc)
fprintf (f, "%i", loc >> 16);
}
+/* Return assembler name as in symbol table and DW_AT_linkage_name. */
+
+static const char *
+raw_symbol_name (const char *asmname)
+{
+ /* If we start supporting user_label_prefixes, add_linkage_attr will also
+ need to be fixed. */
+ if (strlen (user_label_prefix))
+ sorry ("auto-profile is not supported for targets with user label prefix");
+ return asmname + (asmname[0] == '*');
+}
+
+/* Convenience wrapper that looks up assembler name. */
+
+static const char *
+raw_symbol_name (tree decl)
+{
+ return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
+}
+
/* Dump STACK to F. */
static void
@@ -693,7 +717,7 @@ dump_inline_stack (FILE *f, inline_stack *stack)
{
fprintf (f, "%s%s:",
first ? "" : "; ",
- IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (p.decl)));
+ raw_symbol_name (p.decl));
dump_afdo_loc (f, p.afdo_loc);
first = false;
}
@@ -815,7 +839,7 @@ string_table::get_index (const char *name) const
int
string_table::get_index_by_decl (tree decl) const
{
- const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
+ const char *name = raw_symbol_name (decl);
int ret = get_index (name);
if (ret != -1)
return ret;
@@ -878,10 +902,9 @@ function_instance::~function_instance ()
cgraph_node *
function_instance::get_cgraph_node ()
{
- for (symtab_node *n = cgraph_node::get_for_asmname
- (get_identifier
- (afdo_string_table->get_name (name ())));
- n; n = n->next_sharing_asm_name)
+ const char *sname = afdo_string_table->get_name (name ());
+ symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname));
+ for (;n; n = n->next_sharing_asm_name)
if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
if (cn->definition && cn->has_gimple_body_p ())
return cn;
@@ -919,10 +942,10 @@ function_instance::get_function_instance_by_decl (unsigned lineno,
dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
dump_user_location_t::from_location_t (location),
"auto-profile has mismatched function name %s"
- " instaed of %s at loc %i:%i",
+ " insteed of %s at loc %i:%i",
afdo_string_table->get_name (iter.first.second),
- IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
- lineno << 16,
+ raw_symbol_name (decl),
+ lineno >> 16,
lineno & 65535);
}
@@ -1125,10 +1148,13 @@ function_instance::offline_if_in_set (name_index_set &seen,
Return non-zero if it correspons and 2 if renaming was done. */
static int
-match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
+match_with_target (cgraph_node *n,
+ gimple *stmt,
+ function_instance *inlined_fn,
+ cgraph_node *orig_callee)
{
- const char *symbol_name = IDENTIFIER_POINTER
- (DECL_ASSEMBLER_NAME (n->decl));
+ cgraph_node *callee = orig_callee->ultimate_alias_target ();
+ const char *symbol_name = raw_symbol_name (callee->decl);
const char *name = afdo_string_table->get_name (inlined_fn->name ());
if (strcmp (name, symbol_name))
{
@@ -1142,8 +1168,8 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
in_suffix = true;
}
/* Accept dwarf names and stripped suffixes. */
- if (!strcmp (lang_hooks.dwarf_name (n->decl, 0),
- afdo_string_table->get_name (inlined_fn->name ()))
+ if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0),
+ afdo_string_table->get_name (inlined_fn->name ()))
|| (!name[i] && symbol_name[i] == '.')
|| in_suffix)
{
@@ -1157,10 +1183,14 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
inlined_fn->set_name (index);
return 2;
}
- warning_at (gimple_location (stmt), OPT_Wauto_profile,
- "auto-profile contains inlined "
- "function with symbol name %s instead of symbol name %s",
- name, symbol_name);
+ /* Only warn about declarations. It is possible that the function is
+ declared as alias in other module and we inlined cross-module. */
+ if (callee->definition
+ && warning (OPT_Wauto_profile,
+ "auto-profile of %q+F contains inlined "
+ "function with symbol name %s instead of symbol name %s",
+ n->decl, name, symbol_name))
+ inform (gimple_location (stmt), "corresponding call");
return 0;
}
return 1;
@@ -1203,13 +1233,14 @@ function_instance::lookup_count (location_t loc, inline_stack &stack,
if (stack.length ())
{
int c = pos_counts.count (stack[0].afdo_loc);
- if (c > 1)
- warning_at (loc, OPT_Wauto_profile,
- "duplicated count information"
- " in auto-profile of %q+F"
- " with relative location %i discriminator %i",
- node->decl, stack[0].afdo_loc >> 16,
- stack[0].afdo_loc & 65535);
+ if (c > 1
+ && warning (OPT_Wauto_profile,
+ "duplicated count information"
+ " in auto-profile of %q+F"
+ " with relative location %i discriminator %i",
+ node->decl, stack[0].afdo_loc >> 16,
+ stack[0].afdo_loc & 65535))
+ inform (loc, "corresponding source location");
if (c)
return &pos_counts[stack[0].afdo_loc];
}
@@ -1271,6 +1302,7 @@ function_instance::match (cgraph_node *node,
hash_set<const count_info *> counts;
hash_set<const count_info *> targets;
hash_set<const function_instance *> functions;
+ hash_set<const function_instance *> functions_to_offline;
/* We try to fill in lost disciminator if there is unique call
with given line number. This map is used to record them. */
@@ -1374,20 +1406,23 @@ function_instance::match (cgraph_node *node,
inlined_fn_nodisc = iter.second;
cnodis++;
}
- if (c > 1 || cnodis > 1)
- warning_at (gimple_location (stmt), OPT_Wauto_profile,
- "duplicated callsite in auto-profile of %q+F"
- " with relative location %i, discriminator %i",
- node->decl, stack[0].afdo_loc >> 16,
- stack[0].afdo_loc & 65535);
- if (inlined_fn && info && info->targets.size ())
- warning_at (gimple_location (stmt), OPT_Wauto_profile,
- "both call targets and inline callsite"
- " information is present in auto-profile"
- " of function %q+F with relative location"
- " %i, discriminator %i",
- node->decl, stack[0].afdo_loc >> 16,
- stack[0].afdo_loc & 65535);
+ if ((c > 1 || (!c && cnodis > 1))
+ && warning (OPT_Wauto_profile,
+ "duplicated callsite in auto-profile of %q+F"
+ " with relative location %i,"
+ " discriminator %i",
+ node->decl, stack[0].afdo_loc >> 16,
+ stack[0].afdo_loc & 65535))
+ inform (gimple_location (stmt), "corresponding call");
+ if (inlined_fn && info && info->targets.size ()
+ && warning (OPT_Wauto_profile,
+ "both call targets and inline callsite"
+ " information is present in auto-profile"
+ " of function %q+F with relative location"
+ " %i, discriminator %i",
+ node->decl, stack[0].afdo_loc >> 16,
+ stack[0].afdo_loc & 65535))
+ inform (gimple_location (stmt), "corresponding call");
tree callee = gimple_call_fndecl (stmt);
cgraph_node *callee_node;
unsigned int loc = stack[0].afdo_loc;
@@ -1420,11 +1455,17 @@ function_instance::match (cgraph_node *node,
if (lineno_to_call.get
(stack[0].afdo_loc >> 16)->length () == 1)
{
- warning_at (gimple_location (stmt), OPT_Wauto_profile,
- "auto-profile of %q+F seem to contain"
- " lost discriminator %i for call at"
- " relative location %i",
- node->decl, loc & 65535, loc >> 16);
+ if (warning (OPT_Wauto_profile,
+ "auto-profile of %q+F seem to contain"
+ " lost discriminator %i for"
+ " call of %s at relative location %i",
+ node->decl,
+ loc & 65535,
+ afdo_string_table->get_name
+ (inlined_fn_nodisc->name ()),
+ loc >> 16))
+ inform (gimple_location (stmt),
+ "corresponding call");
inlined_fn = inlined_fn_nodisc;
if (dump_file)
fprintf (dump_file, " Lost discriminator %i\n",
@@ -1435,11 +1476,10 @@ function_instance::match (cgraph_node *node,
}
if (callee && (callee_node = cgraph_node::get (callee)))
{
- callee_node = callee_node->ultimate_alias_target ();
if (inlined_fn)
{
int old_name = inlined_fn->name ();
- int r = match_with_target (stmt, inlined_fn,
+ int r = match_with_target (node, stmt, inlined_fn,
callee_node);
if (r == 2)
{
@@ -1456,6 +1496,8 @@ function_instance::match (cgraph_node *node,
}
if (r)
functions.add (inlined_fn);
+ else
+ functions_to_offline.add (inlined_fn);
}
if (info && info->targets.size () > 1)
@@ -1473,19 +1515,24 @@ function_instance::match (cgraph_node *node,
{
if (inlined_fn
&& inlined_fn->get_call_location ()
- != UNKNOWN_LOCATION
- && warning_at (gimple_location (stmt),
- OPT_Wauto_profile,
- "%q+F contains two calls of the same"
- " relative location +%i,"
- " discrimnator %i,"
- " that leads to lost auto-profile",
- node->decl,
- loc << 16,
- loc & 65535))
+ != UNKNOWN_LOCATION)
{
- inform (inlined_fn->get_call_location (),
- "location of the earlier call");
+ if (warning (OPT_Wauto_profile,
+ "function contains two calls of the same"
+ " relative location +%i,"
+ " discrimnator %i,"
+ " that leads to lost auto-profile",
+ loc >> 16,
+ loc & 65535))
+ {
+ inform (gimple_location (stmt),
+ "location of the first call");
+ inform (inlined_fn->get_call_location (),
+ "location of the second call");
+ }
+ if (dump_file)
+ fprintf (dump_file,
+ " Duplicated call location\n");
inlined_fn = NULL;
}
if (inlined_fn)
@@ -1559,11 +1606,18 @@ function_instance::match (cgraph_node *node,
(DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl);
unsigned int start_location = get_combined_location
(DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl);
+ /* When outputting code to builtins location we use line number 0.
+ craeate_gcov is stupid and hapilly computes offsets across files.
+ Silently ignore it. */
+ unsigned int zero_location
+ = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16;
for (position_count_map::const_iterator iter = pos_counts.begin ();
iter != pos_counts.end ();)
if (!counts.contains (&iter->second))
{
- if (iter->first != end_location && iter->first != start_location
+ if (iter->first != end_location
+ && iter->first != start_location
+ && (iter->first & 65535) != zero_location
&& iter->first)
{
if (!warned)
@@ -1601,28 +1655,42 @@ function_instance::match (cgraph_node *node,
iter != callsites.end ();)
if (!functions.contains (iter->second))
{
- if (!warned)
- warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
- OPT_Wauto_profile,
- "auto-profile of %q+F contains extra callsites",
- node->decl);
- if (warned)
- inform (DECL_SOURCE_LOCATION (node->decl),
- "call of %s with relative location +%i, discriminator %i",
- afdo_string_table->get_name (iter->first.second),
- iter->first.first >> 16, iter->first.first & 65535);
- if ((iter->first.first >> 16) > (end_location >> 16) && warned)
- inform (DECL_SOURCE_LOCATION (node->decl),
- "location is after end of function");
- warned = true;
function_instance *f = iter->second;
- if (dump_file)
+ /* If we did not see the corresponding statement, warn. */
+ if (!functions_to_offline.contains (iter->second))
+ {
+ if (!warned)
+ warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
+ OPT_Wauto_profile,
+ "auto-profile of %q+F contains"
+ " extra callsites",
+ node->decl);
+ if (warned)
+ inform (DECL_SOURCE_LOCATION (node->decl),
+ "call of %s with total count %" PRId64
+ ", relative location +%i, discriminator %i",
+ afdo_string_table->get_name (iter->first.second),
+ iter->second->total_count (),
+ iter->first.first >> 16, iter->first.first & 65535);
+ if ((iter->first.first >> 16) > (end_location >> 16) && warned)
+ inform (DECL_SOURCE_LOCATION (node->decl),
+ "location is after end of function");
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "Offlining inline with no corresponding gimple stmt ");
+ f->dump_inline_stack (dump_file);
+ fprintf (dump_file, "\n");
+ }
+ }
+ else if (dump_file)
{
fprintf (dump_file,
- "Offlining inline with no corresponding gimple stmt ");
+ "Offlining mismatched inline ");
f->dump_inline_stack (dump_file);
fprintf (dump_file, "\n");
}
+ warned = true;
callsites.erase (iter);
offline (f, new_functions);
iter = callsites.begin ();
@@ -1921,7 +1989,7 @@ autofdo_source_profile::offline_external_functions ()
FOR_EACH_DEFINED_FUNCTION (node)
{
const char *name
- = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
+ = raw_symbol_name (node->decl);
const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0);
int index = afdo_string_table->get_index (name);
@@ -2103,8 +2171,7 @@ walk_block (tree fn, function_instance *s, tree block)
fprintf (dump_file, ":");
dump_afdo_loc (dump_file, loc);
fprintf (dump_file, " %s\n",
- IDENTIFIER_POINTER
- (DECL_ASSEMBLER_NAME (BLOCK_ABSTRACT_ORIGIN (block))));
+ raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block)));
}
return;
}
@@ -2462,7 +2529,7 @@ autofdo_source_profile::get_callsite_total_count (
{
if (dump_file)
fprintf (dump_file, "Mismatched name of callee %s and profile %s\n",
- IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (edge->callee->decl)),
+ raw_symbol_name (edge->callee->decl),
afdo_string_table->get_name (s->name ()));
return 0;
}
@@ -2556,8 +2623,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
{
if (dump_file)
fprintf (dump_file, "No offline instance for %s\n",
- IDENTIFIER_POINTER
- (DECL_ASSEMBLER_NAME (stack[stack.length () - 1].decl)));
+ raw_symbol_name (stack[stack.length () - 1].decl));
return NULL;
}
function_instance *s = iter->second;
@@ -2579,8 +2645,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
"auto-profile has no inlined function instance "
"for inlined call of %s at relative "
" locaction +%i, discriminator %i\n",
- IDENTIFIER_POINTER
- (DECL_ASSEMBLER_NAME (stack[i - 1].decl)),
+ raw_symbol_name (stack[i - 1].decl),
stack[i].afdo_loc >> 16,
stack[i].afdo_loc & 65535);
return NULL;
@@ -3304,10 +3369,22 @@ cmp (const void *a, const void *b)
return 0;
}
+/* To scalle a connected component of graph we collect desired scales of
+ basic blocks on the boundary and then compute a robust average. */
+
+struct scale
+{
+ /* Scale descired. */
+ sreal scale;
+ /* Weight for averaging computed from execution count of the edge
+ scale originates from. */
+ uint64_t weight;
+};
+
/* Add scale ORIG/ANNOTATED to SCALES. */
static void
-add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
+add_scale (vec <scale> *scales, profile_count annotated, profile_count orig)
{
if (dump_file)
{
@@ -3316,15 +3393,15 @@ add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
annotated.dump (dump_file);
fprintf (dump_file, "\n");
}
- if (orig.nonzero_p ())
+ if (orig.force_nonzero () == orig)
{
sreal scale
= annotated.guessed_local ()
.to_sreal_scale (orig);
if (dump_file)
- fprintf (dump_file, " adding scale %.16f\n",
- scale.to_double ());
- scales->safe_push (scale);
+ fprintf (dump_file, " adding scale %.16f, weight %" PRId64 "\n",
+ scale.to_double (), annotated.value () + 1);
+ scales->safe_push ({scale, annotated.value () + 1});
}
}
@@ -3370,7 +3447,7 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
/* Basic blocks of connected component currently processed. */
auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
/* Scale factors found. */
- auto_vec <sreal, 20> scales;
+ auto_vec <scale, 20> scales;
auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun));
basic_block seed_bb;
@@ -3382,9 +3459,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
>=2 is an id of the component BB belongs to. */
auto_vec <unsigned int, 20> component;
component.safe_grow (last_basic_block_for_fn (cfun));
+ profile_count max_count_in_fn = profile_count::zero ();
FOR_ALL_BB_FN (seed_bb, cfun)
- component[seed_bb->index]
- = is_bb_annotated (seed_bb, *annotated_bb) ? 1 : 0;
+ if (is_bb_annotated (seed_bb, *annotated_bb))
+ {
+ component[seed_bb->index] = 1;
+ max_count_in_fn = max_count_in_fn.max (seed_bb->count);
+ }
+ else
+ component[seed_bb->index] = 0;
FOR_ALL_BB_FN (seed_bb, cfun)
if (!component[seed_bb->index])
{
@@ -3507,12 +3590,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
profile_count annotated_count = e->dest->count;
profile_count out_count = profile_count::zero ();
bool ok = true;
+
for (edge e2: e->dest->preds)
if (AFDO_EINFO (e2)->is_annotated ())
annotated_count -= AFDO_EINFO (e2)->get_count ();
- else if (component[e->src->index] == component_id)
- out_count += e->count ();
- else if (e->probability.nonzero_p ())
+ else if (component[e2->src->index] == component_id)
+ out_count += e2->count ();
+ else if (is_bb_annotated (e2->src, *annotated_bb))
+ annotated_count -= e2->count ();
+ else if (e2->probability.nonzero_p ())
{
ok = false;
break;
@@ -3559,7 +3645,47 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
}
gcc_checking_assert (scales.length ());
scales.qsort (cmp);
- scale_bbs (bbs, scales[scales.length () / 2]);
+
+ uint64_t overall_weight = 0;
+ for (scale &e : scales)
+ overall_weight += e.weight;
+
+ uint64_t cummulated = 0, weight_sum = 0;
+ sreal scale_sum = 0;
+ for (scale &e : scales)
+ {
+ uint64_t prev = cummulated;
+ cummulated += e.weight;
+ if (cummulated >= overall_weight / 4
+ && prev <= 3 * overall_weight / 4)
+ {
+ scale_sum += e.scale * e.weight;
+ weight_sum += e.weight;
+ if (dump_file)
+ fprintf (dump_file, " accounting scale %.16f, weight %" PRId64 "\n",
+ e.scale.to_double (), e.weight);
+ }
+ else if (dump_file)
+ fprintf (dump_file, " ignoring scale %.16f, weight %" PRId64 "\n",
+ e.scale.to_double (), e.weight);
+ }
+ sreal scale = scale_sum / (sreal)weight_sum;
+
+ /* Avoid scaled regions to have very large counts.
+ Otherwise they may dominate ipa-profile's histogram computing cutoff
+ of hot basic blocks. */
+ if (max_count * scale > max_count_in_fn.guessed_local ())
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Scaling by %.16f produces max count ",
+ scale.to_double ());
+ (max_count * scale).dump (dump_file);
+ fprintf (dump_file, " that exceeds max count in fn; capping\n");
+ }
+ scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count);
+ }
+ scale_bbs (bbs, scale);
}
}