diff options
Diffstat (limited to 'gcc/auto-profile.cc')
| -rw-r--r-- | gcc/auto-profile.cc | 532 |
1 files changed, 345 insertions, 187 deletions
diff --git a/gcc/auto-profile.cc b/gcc/auto-profile.cc index a970eb8..d1954b4 100644 --- a/gcc/auto-profile.cc +++ b/gcc/auto-profile.cc @@ -53,6 +53,7 @@ along with GCC; see the file COPYING3. If not see #include "auto-profile.h" #include "tree-pretty-print.h" #include "gimple-pretty-print.h" +#include "output.h" /* The following routines implements AutoFDO optimization. @@ -239,6 +240,8 @@ public: /* Add new name and return its index. */ int add_name (char *); + /* Return cgraph node corresponding to given name index. */ + cgraph_node *get_cgraph_node (int); private: typedef std::map<const char *, unsigned, string_compare> string_index_map; string_vector vector_; @@ -430,7 +433,8 @@ public: void set_call_location (location_t l) { - gcc_checking_assert (call_location_ == UNKNOWN_LOCATION); + gcc_checking_assert (call_location_ == UNKNOWN_LOCATION + && l != UNKNOWN_LOCATION); call_location_= l; } @@ -443,7 +447,6 @@ public: /* Lookup count and warn about duplicates. */ count_info *lookup_count (location_t loc, inline_stack &stack, cgraph_node *node); - private: /* Callsite, represented as (decl_lineno, callee_function_name_index). */ typedef std::pair<unsigned, unsigned> callsite; @@ -622,9 +625,11 @@ get_original_name (const char *name, bool alloc = true) } /* Suffixes of clones that compiler generates after auto-profile. */ const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"}; - for (unsigned i = 0; i < sizeof (suffixes); ++i) + for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i) { - if (strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0) + int len = strlen (suffixes[i]); + if (len == last_dot - next_dot - 1 + && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0) { *next_dot = 0; return get_original_name (ret, false); @@ -683,6 +688,26 @@ dump_afdo_loc (FILE *f, unsigned loc) fprintf (f, "%i", loc >> 16); } +/* Return assembler name as in symbol table and DW_AT_linkage_name. */ + +static const char * +raw_symbol_name (const char *asmname) +{ + /* If we start supporting user_label_prefixes, add_linkage_attr will also + need to be fixed. */ + if (strlen (user_label_prefix)) + sorry ("auto-profile is not supported for targets with user label prefix"); + return asmname + (asmname[0] == '*'); +} + +/* Convenience wrapper that looks up assembler name. */ + +static const char * +raw_symbol_name (tree decl) +{ + return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl))); +} + /* Dump STACK to F. */ static void @@ -693,7 +718,7 @@ dump_inline_stack (FILE *f, inline_stack *stack) { fprintf (f, "%s%s:", first ? "" : "; ", - IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (p.decl))); + raw_symbol_name (p.decl)); dump_afdo_loc (f, p.afdo_loc); first = false; } @@ -815,7 +840,7 @@ string_table::get_index (const char *name) const int string_table::get_index_by_decl (tree decl) const { - const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); + const char *name = raw_symbol_name (decl); int ret = get_index (name); if (ret != -1) return ret; @@ -864,6 +889,29 @@ string_table::read () return true; } +/* Return cgraph node corresponding to given NAME_INDEX, + NULL if unavailable. */ +cgraph_node * +string_table::get_cgraph_node (int name_index) +{ + const char *sname = get_name (name_index); + + symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname)); + for (;n; n = n->next_sharing_asm_name) + if (cgraph_node *cn = dyn_cast <cgraph_node *> (n)) + if (cn->definition && cn->has_gimple_body_p ()) + return cn; + return NULL; +} + +/* Return corresponding cgraph node. */ + +cgraph_node * +function_instance::get_cgraph_node () +{ + return afdo_string_table->get_cgraph_node (name ()); +} + /* Member functions for function_instance. */ function_instance::~function_instance () @@ -874,20 +922,6 @@ function_instance::~function_instance () delete iter->second; } -/* Return corresponding cgraph node, NULL if unavailable. */ -cgraph_node * -function_instance::get_cgraph_node () -{ - for (symtab_node *n = cgraph_node::get_for_asmname - (get_identifier - (afdo_string_table->get_name (name ()))); - n; n = n->next_sharing_asm_name) - if (cgraph_node *cn = dyn_cast <cgraph_node *> (n)) - if (cn->definition && cn->has_gimple_body_p ()) - return cn; - return NULL; -} - /* Traverse callsites of the current function_instance to find one at the location of LINENO and callee name represented in DECL. */ @@ -919,10 +953,10 @@ function_instance::get_function_instance_by_decl (unsigned lineno, dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS, dump_user_location_t::from_location_t (location), "auto-profile has mismatched function name %s" - " instaed of %s at loc %i:%i", + " insteed of %s at loc %i:%i", afdo_string_table->get_name (iter.first.second), - IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), - lineno << 16, + raw_symbol_name (decl), + lineno >> 16, lineno & 65535); } @@ -1125,10 +1159,13 @@ function_instance::offline_if_in_set (name_index_set &seen, Return non-zero if it correspons and 2 if renaming was done. */ static int -match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n) +match_with_target (cgraph_node *n, + gimple *stmt, + function_instance *inlined_fn, + cgraph_node *orig_callee) { - const char *symbol_name = IDENTIFIER_POINTER - (DECL_ASSEMBLER_NAME (n->decl)); + cgraph_node *callee = orig_callee->ultimate_alias_target (); + const char *symbol_name = raw_symbol_name (callee->decl); const char *name = afdo_string_table->get_name (inlined_fn->name ()); if (strcmp (name, symbol_name)) { @@ -1142,7 +1179,7 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n) in_suffix = true; } /* Accept dwarf names and stripped suffixes. */ - if (!strcmp (lang_hooks.dwarf_name (n->decl, 0), + if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0), afdo_string_table->get_name (inlined_fn->name ())) || (!name[i] && symbol_name[i] == '.') || in_suffix) @@ -1157,10 +1194,14 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n) inlined_fn->set_name (index); return 2; } - warning_at (gimple_location (stmt), OPT_Wauto_profile, - "auto-profile contains inlined " - "function with symbol name %s instead of symbol name %s", - name, symbol_name); + /* Only warn about declarations. It is possible that the function + is declared as alias in other module and we inlined cross-module. */ + if (callee->definition + && warning (OPT_Wauto_profile, + "auto-profile of %q+F contains inlined " + "function with symbol name %s instead of symbol name %s", + n->decl, name, symbol_name)) + inform (gimple_location (stmt), "corresponding call"); return 0; } return 1; @@ -1203,13 +1244,14 @@ function_instance::lookup_count (location_t loc, inline_stack &stack, if (stack.length ()) { int c = pos_counts.count (stack[0].afdo_loc); - if (c > 1) - warning_at (loc, OPT_Wauto_profile, - "duplicated count information" - " in auto-profile of %q+F" - " with relative location %i discriminator %i", - node->decl, stack[0].afdo_loc >> 16, - stack[0].afdo_loc & 65535); + if (c > 1 + && warning (OPT_Wauto_profile, + "duplicated count information" + " in auto-profile of %q+F" + " with relative location %i discriminator %i", + node->decl, stack[0].afdo_loc >> 16, + stack[0].afdo_loc & 65535)) + inform (loc, "corresponding source location"); if (c) return &pos_counts[stack[0].afdo_loc]; } @@ -1271,6 +1313,7 @@ function_instance::match (cgraph_node *node, hash_set<const count_info *> counts; hash_set<const count_info *> targets; hash_set<const function_instance *> functions; + hash_set<const function_instance *> functions_to_offline; /* We try to fill in lost disciminator if there is unique call with given line number. This map is used to record them. */ @@ -1374,20 +1417,23 @@ function_instance::match (cgraph_node *node, inlined_fn_nodisc = iter.second; cnodis++; } - if (c > 1 || cnodis > 1) - warning_at (gimple_location (stmt), OPT_Wauto_profile, - "duplicated callsite in auto-profile of %q+F" - " with relative location %i, discriminator %i", - node->decl, stack[0].afdo_loc >> 16, - stack[0].afdo_loc & 65535); - if (inlined_fn && info && info->targets.size ()) - warning_at (gimple_location (stmt), OPT_Wauto_profile, - "both call targets and inline callsite" - " information is present in auto-profile" - " of function %q+F with relative location" - " %i, discriminator %i", - node->decl, stack[0].afdo_loc >> 16, - stack[0].afdo_loc & 65535); + if ((c > 1 || (!c && cnodis > 1)) + && warning (OPT_Wauto_profile, + "duplicated callsite in auto-profile of %q+F" + " with relative location %i," + " discriminator %i", + node->decl, stack[0].afdo_loc >> 16, + stack[0].afdo_loc & 65535)) + inform (gimple_location (stmt), "corresponding call"); + if (inlined_fn && info && info->targets.size () + && warning (OPT_Wauto_profile, + "both call targets and inline callsite" + " information is present in auto-profile" + " of function %q+F with relative location" + " %i, discriminator %i", + node->decl, stack[0].afdo_loc >> 16, + stack[0].afdo_loc & 65535)) + inform (gimple_location (stmt), "corresponding call"); tree callee = gimple_call_fndecl (stmt); cgraph_node *callee_node; unsigned int loc = stack[0].afdo_loc; @@ -1420,11 +1466,17 @@ function_instance::match (cgraph_node *node, if (lineno_to_call.get (stack[0].afdo_loc >> 16)->length () == 1) { - warning_at (gimple_location (stmt), OPT_Wauto_profile, - "auto-profile of %q+F seem to contain" - " lost discriminator %i for call at" - " relative location %i", - node->decl, loc & 65535, loc >> 16); + if (warning (OPT_Wauto_profile, + "auto-profile of %q+F seem to contain" + " lost discriminator %i for" + " call of %s at relative location %i", + node->decl, + loc & 65535, + afdo_string_table->get_name + (inlined_fn_nodisc->name ()), + loc >> 16)) + inform (gimple_location (stmt), + "corresponding call"); inlined_fn = inlined_fn_nodisc; if (dump_file) fprintf (dump_file, " Lost discriminator %i\n", @@ -1435,11 +1487,10 @@ function_instance::match (cgraph_node *node, } if (callee && (callee_node = cgraph_node::get (callee))) { - callee_node = callee_node->ultimate_alias_target (); if (inlined_fn) { int old_name = inlined_fn->name (); - int r = match_with_target (stmt, inlined_fn, + int r = match_with_target (node, stmt, inlined_fn, callee_node); if (r == 2) { @@ -1451,11 +1502,13 @@ function_instance::match (cgraph_node *node, == inlined_fn); callsite key2 = {stack[0].afdo_loc, inlined_fn->name ()}; - callsites[key2] = inlined_fn; callsites.erase (iter); + callsites[key2] = inlined_fn; } if (r) functions.add (inlined_fn); + else + functions_to_offline.add (inlined_fn); } if (info && info->targets.size () > 1) @@ -1473,19 +1526,24 @@ function_instance::match (cgraph_node *node, { if (inlined_fn && inlined_fn->get_call_location () - != UNKNOWN_LOCATION - && warning_at (gimple_location (stmt), - OPT_Wauto_profile, - "%q+F contains two calls of the same" - " relative location +%i," - " discrimnator %i," - " that leads to lost auto-profile", - node->decl, - loc << 16, - loc & 65535)) + != UNKNOWN_LOCATION) { - inform (inlined_fn->get_call_location (), - "location of the earlier call"); + if (warning (OPT_Wauto_profile, + "function contains two calls of the same" + " relative location +%i," + " discrimnator %i," + " that leads to lost auto-profile", + loc >> 16, + loc & 65535)) + { + inform (gimple_location (stmt), + "location of the first call"); + inform (inlined_fn->get_call_location (), + "location of the second call"); + } + if (dump_file) + fprintf (dump_file, + " Duplicated call location\n"); inlined_fn = NULL; } if (inlined_fn) @@ -1507,10 +1565,10 @@ function_instance::match (cgraph_node *node, callsite key2 = {stack[0].afdo_loc, newn ? *newn : inlined_fn->name ()}; + callsites.erase (iter); callsites[key2] = inlined_fn; inlined_fn->set_name (newn ? *newn : inlined_fn->name ()); - callsites.erase (iter); } functions.add (inlined_fn); } @@ -1559,11 +1617,18 @@ function_instance::match (cgraph_node *node, (DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl); unsigned int start_location = get_combined_location (DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl); + /* When outputting code to builtins location we use line number 0. + craeate_gcov is stupid and hapilly computes offsets across files. + Silently ignore it. */ + unsigned int zero_location + = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16; for (position_count_map::const_iterator iter = pos_counts.begin (); iter != pos_counts.end ();) if (!counts.contains (&iter->second)) { - if (iter->first != end_location && iter->first != start_location + if (iter->first != end_location + && iter->first != start_location + && (iter->first & 65535) != zero_location && iter->first) { if (!warned) @@ -1601,25 +1666,38 @@ function_instance::match (cgraph_node *node, iter != callsites.end ();) if (!functions.contains (iter->second)) { - if (!warned) - warned = warning_at (DECL_SOURCE_LOCATION (node->decl), - OPT_Wauto_profile, - "auto-profile of %q+F contains extra callsites", - node->decl); - if (warned) - inform (DECL_SOURCE_LOCATION (node->decl), - "call of %s with relative location +%i, discriminator %i", - afdo_string_table->get_name (iter->first.second), - iter->first.first >> 16, iter->first.first & 65535); - if ((iter->first.first >> 16) > (end_location >> 16) && warned) - inform (DECL_SOURCE_LOCATION (node->decl), - "location is after end of function"); - warned = true; function_instance *f = iter->second; - if (dump_file) + /* If we did not see the corresponding statement, warn. */ + if (!functions_to_offline.contains (iter->second)) + { + if (!warned) + warned = warning_at (DECL_SOURCE_LOCATION (node->decl), + OPT_Wauto_profile, + "auto-profile of %q+F contains" + " extra callsites", + node->decl); + if (warned) + inform (DECL_SOURCE_LOCATION (node->decl), + "call of %s with total count %" PRId64 + ", relative location +%i, discriminator %i", + afdo_string_table->get_name (iter->first.second), + iter->second->total_count (), + iter->first.first >> 16, iter->first.first & 65535); + if ((iter->first.first >> 16) > (end_location >> 16) && warned) + inform (DECL_SOURCE_LOCATION (node->decl), + "location is after end of function"); + if (dump_file) + { + fprintf (dump_file, + "Offlining inline with no corresponding gimple stmt "); + f->dump_inline_stack (dump_file); + fprintf (dump_file, "\n"); + } + } + else if (dump_file) { fprintf (dump_file, - "Offlining inline with no corresponding gimple stmt "); + "Offlining mismatched inline "); f->dump_inline_stack (dump_file); fprintf (dump_file, "\n"); } @@ -1684,9 +1762,9 @@ function_instance::remove_external_functions auto iter = callsites.find (key); callsite key2 = key; key2.second = *to_symbol_name.get (key.second); - callsites[key2] = iter->second; iter->second->set_name (key2.second); callsites.erase (iter); + callsites[key2] = iter->second; } auto_vec <int, 20> target_to_rename; for (auto &iter : pos_counts) @@ -1883,6 +1961,7 @@ autofdo_source_profile::offline_external_functions () cgraph_node *node; name_index_set seen; name_index_map to_symbol_name; + size_t last_name; /* Add renames erasing suffixes produced by late clones, such as .isra, .ipcp. */ @@ -1918,10 +1997,10 @@ autofdo_source_profile::offline_external_functions () index = afdo_string_table->add_name (n2); to_symbol_name.put (i, index); } + last_name = afdo_string_table->num_entries (); FOR_EACH_DEFINED_FUNCTION (node) { - const char *name - = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); + const char *name = raw_symbol_name (node->decl); const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0); int index = afdo_string_table->get_index (name); @@ -1958,11 +2037,17 @@ autofdo_source_profile::offline_external_functions () { if (dump_file) { - fprintf (dump_file, - "Node %s not in auto profile (%s neither %s)\n", - node->dump_name (), - name, - dwarf_name); + if (dwarf_name && strcmp (dwarf_name, name)) + fprintf (dump_file, + "Node %s not in auto profile (%s neither %s)\n", + node->dump_name (), + name, + dwarf_name); + else + fprintf (dump_file, + "Node %s (symbol %s) not in auto profile\n", + node->dump_name (), + name); } } } @@ -2006,72 +2091,87 @@ autofdo_source_profile::offline_external_functions () should be done on unmodified profile and merging works better if mismatches are already resolved both in source and destination. */ while (fns.length () || fns2.length ()) - if (fns.length ()) - { - function_instance *f = fns.pop (); - if (f->get_location () == UNKNOWN_LOCATION) - { - int index = f->name (); - int *newn = to_symbol_name.get (index); - if (newn) - { - f->set_name (*newn); - if (map_.count (index) - && map_[index] == f) - map_.erase (index); - if (!map_.count (*newn)) - map_[*newn] = f; - } - if (cgraph_node *n = f->get_cgraph_node ()) - { - gcc_checking_assert (seen.contains (f->name ())); - f->match (n, fns, to_symbol_name); - } - } - fns2.safe_push (f); - } - else - { - function_instance *f = fns2.pop (); - int index = f->name (); - gcc_checking_assert (f->in_worklist_p ()); + { + /* In case renaming introduced new name, keep seen up to date. */ + for (; last_name < afdo_string_table->num_entries (); last_name++) + { + const char *name = afdo_string_table->get_name (last_name); + symtab_node *n + = afdo_string_table->get_cgraph_node (last_name); + if (dump_file) + fprintf (dump_file, "New name %s %s\n", name, + n ? "wth corresponding definition" + : "with no corresponding definition"); + if (n) + seen.add (last_name); + } + if (fns.length ()) + { + function_instance *f = fns.pop (); + if (f->get_location () == UNKNOWN_LOCATION) + { + int index = f->name (); + int *newn = to_symbol_name.get (index); + if (newn) + { + f->set_name (*newn); + if (map_.count (index) + && map_[index] == f) + map_.erase (index); + if (!map_.count (*newn)) + map_[*newn] = f; + } + if (cgraph_node *n = f->get_cgraph_node ()) + { + gcc_checking_assert (seen.contains (f->name ())); + f->match (n, fns, to_symbol_name); + } + } + fns2.safe_push (f); + } + else + { + function_instance *f = fns2.pop (); + int index = f->name (); + gcc_checking_assert (f->in_worklist_p ()); - /* If map has different function_instance of same name, then - this is a duplicated entry which needs to be merged. */ - if (map_.count (index) && map_[index] != f) - { - if (dump_file) - { - fprintf (dump_file, "Merging duplicate instance: "); - f->dump_inline_stack (dump_file); - fprintf (dump_file, "\n"); - } - map_[index]->merge (f, fns); - gcc_checking_assert (!f->inlined_to ()); - f->clear_in_worklist (); - delete f; - } - /* If name was not seen in the symbol table, remove it. */ - else if (!seen.contains (index)) - { - f->offline_if_in_set (seen, fns); - f->clear_in_worklist (); - if (dump_file) - fprintf (dump_file, "Removing external %s\n", - afdo_string_table->get_name (f->name ())); - if (map_.count (index) && map_[index] == f) - map_.erase (f->name ()); - delete f; - } - /* If this is offline function instance seen in this - translation unit offline external inlines and possibly - rename from dwarf name. */ - else - { - f->remove_external_functions (seen, to_symbol_name, fns); - f->clear_in_worklist (); - } - } + /* If map has different function_instance of same name, then + this is a duplicated entry which needs to be merged. */ + if (map_.count (index) && map_[index] != f) + { + if (dump_file) + { + fprintf (dump_file, "Merging duplicate instance: "); + f->dump_inline_stack (dump_file); + fprintf (dump_file, "\n"); + } + map_[index]->merge (f, fns); + gcc_checking_assert (!f->inlined_to ()); + f->clear_in_worklist (); + delete f; + } + /* If name was not seen in the symbol table, remove it. */ + else if (!seen.contains (index)) + { + f->offline_if_in_set (seen, fns); + f->clear_in_worklist (); + if (dump_file) + fprintf (dump_file, "Removing external %s\n", + afdo_string_table->get_name (f->name ())); + if (map_.count (index) && map_[index] == f) + map_.erase (f->name ()); + delete f; + } + /* If this is offline function instance seen in this + translation unit offline external inlines and possibly + rename from dwarf name. */ + else + { + f->remove_external_functions (seen, to_symbol_name, fns); + f->clear_in_worklist (); + } + } + } if (dump_file) for (auto const &iter : map_) { @@ -2103,8 +2203,7 @@ walk_block (tree fn, function_instance *s, tree block) fprintf (dump_file, ":"); dump_afdo_loc (dump_file, loc); fprintf (dump_file, " %s\n", - IDENTIFIER_POINTER - (DECL_ASSEMBLER_NAME (BLOCK_ABSTRACT_ORIGIN (block)))); + raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block))); } return; } @@ -2221,7 +2320,7 @@ autofdo_source_profile::offline_unrealized_inlines () function_instance * function_instance::read_function_instance (function_instance_stack *stack, - gcov_type head_count) + gcov_type head_count) { unsigned name = gcov_read_unsigned (); unsigned num_pos_counts = gcov_read_unsigned (); @@ -2244,10 +2343,10 @@ function_instance::read_function_instance (function_instance_stack *stack, (*stack)[j]->total_count_ += count; for (unsigned j = 0; j < num_targets; j++) { - /* Only indirect call target histogram is supported now. */ - gcov_read_unsigned (); - gcov_type target_idx = gcov_read_counter (); - s->pos_counts[offset].targets[target_idx] = gcov_read_counter (); + /* Only indirect call target histogram is supported now. */ + gcov_read_unsigned (); + gcov_type target_idx = gcov_read_counter (); + s->pos_counts[offset].targets[target_idx] = gcov_read_counter (); } } for (unsigned i = 0; i < num_callsites; i++) @@ -2462,7 +2561,7 @@ autofdo_source_profile::get_callsite_total_count ( { if (dump_file) fprintf (dump_file, "Mismatched name of callee %s and profile %s\n", - IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (edge->callee->decl)), + raw_symbol_name (edge->callee->decl), afdo_string_table->get_name (s->name ())); return 0; } @@ -2556,8 +2655,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack ( { if (dump_file) fprintf (dump_file, "No offline instance for %s\n", - IDENTIFIER_POINTER - (DECL_ASSEMBLER_NAME (stack[stack.length () - 1].decl))); + raw_symbol_name (stack[stack.length () - 1].decl)); return NULL; } function_instance *s = iter->second; @@ -2579,8 +2677,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack ( "auto-profile has no inlined function instance " "for inlined call of %s at relative " " locaction +%i, discriminator %i\n", - IDENTIFIER_POINTER - (DECL_ASSEMBLER_NAME (stack[i - 1].decl)), + raw_symbol_name (stack[i - 1].decl), stack[i].afdo_loc >> 16, stack[i].afdo_loc & 65535); return NULL; @@ -3304,10 +3401,22 @@ cmp (const void *a, const void *b) return 0; } +/* To scalle a connected component of graph we collect desired scales of + basic blocks on the boundary and then compute a robust average. */ + +struct scale +{ + /* Scale descired. */ + sreal scale; + /* Weight for averaging computed from execution count of the edge + scale originates from. */ + uint64_t weight; +}; + /* Add scale ORIG/ANNOTATED to SCALES. */ static void -add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig) +add_scale (vec <scale> *scales, profile_count annotated, profile_count orig) { if (dump_file) { @@ -3316,15 +3425,15 @@ add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig) annotated.dump (dump_file); fprintf (dump_file, "\n"); } - if (orig.nonzero_p ()) + if (orig.force_nonzero () == orig) { sreal scale = annotated.guessed_local () .to_sreal_scale (orig); if (dump_file) - fprintf (dump_file, " adding scale %.16f\n", - scale.to_double ()); - scales->safe_push (scale); + fprintf (dump_file, " adding scale %.16f, weight %" PRId64 "\n", + scale.to_double (), annotated.value () + 1); + scales->safe_push ({scale, annotated.value () + 1}); } } @@ -3370,7 +3479,7 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) /* Basic blocks of connected component currently processed. */ auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun)); /* Scale factors found. */ - auto_vec <sreal, 20> scales; + auto_vec <scale, 20> scales; auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun)); basic_block seed_bb; @@ -3382,9 +3491,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) >=2 is an id of the component BB belongs to. */ auto_vec <unsigned int, 20> component; component.safe_grow (last_basic_block_for_fn (cfun)); + profile_count max_count_in_fn = profile_count::zero (); FOR_ALL_BB_FN (seed_bb, cfun) - component[seed_bb->index] - = is_bb_annotated (seed_bb, *annotated_bb) ? 1 : 0; + if (is_bb_annotated (seed_bb, *annotated_bb)) + { + component[seed_bb->index] = 1; + max_count_in_fn = max_count_in_fn.max (seed_bb->count); + } + else + component[seed_bb->index] = 0; FOR_ALL_BB_FN (seed_bb, cfun) if (!component[seed_bb->index]) { @@ -3507,12 +3622,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) profile_count annotated_count = e->dest->count; profile_count out_count = profile_count::zero (); bool ok = true; + for (edge e2: e->dest->preds) if (AFDO_EINFO (e2)->is_annotated ()) annotated_count -= AFDO_EINFO (e2)->get_count (); - else if (component[e->src->index] == component_id) - out_count += e->count (); - else if (e->probability.nonzero_p ()) + else if (component[e2->src->index] == component_id) + out_count += e2->count (); + else if (is_bb_annotated (e2->src, *annotated_bb)) + annotated_count -= e2->count (); + else if (e2->probability.nonzero_p ()) { ok = false; break; @@ -3559,7 +3677,47 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) } gcc_checking_assert (scales.length ()); scales.qsort (cmp); - scale_bbs (bbs, scales[scales.length () / 2]); + + uint64_t overall_weight = 0; + for (scale &e : scales) + overall_weight += e.weight; + + uint64_t cummulated = 0, weight_sum = 0; + sreal scale_sum = 0; + for (scale &e : scales) + { + uint64_t prev = cummulated; + cummulated += e.weight; + if (cummulated >= overall_weight / 4 + && prev <= 3 * overall_weight / 4) + { + scale_sum += e.scale * e.weight; + weight_sum += e.weight; + if (dump_file) + fprintf (dump_file, " accounting scale %.16f, weight %" PRId64 "\n", + e.scale.to_double (), e.weight); + } + else if (dump_file) + fprintf (dump_file, " ignoring scale %.16f, weight %" PRId64 "\n", + e.scale.to_double (), e.weight); + } + sreal scale = scale_sum / (sreal)weight_sum; + + /* Avoid scaled regions to have very large counts. + Otherwise they may dominate ipa-profile's histogram computing cutoff + of hot basic blocks. */ + if (max_count * scale > max_count_in_fn.guessed_local ()) + { + if (dump_file) + { + fprintf (dump_file, "Scaling by %.16f produces max count ", + scale.to_double ()); + (max_count * scale).dump (dump_file); + fprintf (dump_file, " that exceeds max count in fn; capping\n"); + } + scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count); + } + scale_bbs (bbs, scale); } } |