From 5fa396ad6cd48bc44e6754096fe71febc2a77983 Mon Sep 17 00:00:00 2001 From: Jan Hubicka Date: Sun, 16 Jul 2017 18:10:58 +0200 Subject: profile-count.h (profile_probability::from_reg_br_prob_note, [...]): New functions. * profile-count.h (profile_probability::from_reg_br_prob_note, profile_probability::to_reg_br_prob_note): New functions. * doc/rtl.texi (REG_BR_PROB_NOTE): Update documentation. * reg-notes.h (REG_BR_PROB, REG_BR_PRED): Update docs. * predict.c (probability_reliable_p): Update. (edge_probability_reliable_p): Update. (br_prob_note_reliable_p): Update. (invert_br_probabilities): Update. (add_reg_br_prob_note): New function. (combine_predictions_for_insn): Update. * asan.c (asan_clear_shadow): Update. * cfgbuild.c (compute_outgoing_frequencies): Update. * cfgrtl.c (force_nonfallthru_and_redirect): Update. (update_br_prob_note): Update. (rtl_verify_edges): Update. (purge_dead_edges): Update. (fixup_reorder_chain): Update. * emit-rtl.c (try_split): Update. * ifcvt.c (cond_exec_process_insns): Update. (cond_exec_process_if_block): Update. (dead_or_predicable): Update. * internal-fn.c (expand_addsub_overflow): Update. (expand_neg_overflow): Update. (expand_mul_overflow): Update. * loop-doloop.c (doloop_modify): Update. * loop-unroll.c (compare_and_jump_seq): Update. * optabs.c (emit_cmp_and_jump_insn_1): Update. * predict.h: Update. * reorg.c (mostly_true_jump): Update. * rtl.h: Update. * config/aarch64/aarch64.c (aarch64_emit_unlikely_jump): Update. * config/alpha/alpha.c (emit_unlikely_jump): Update. * config/arc/arc.c: (emit_unlikely_jump): Update. * config/arm/arm.c: (emit_unlikely_jump): Update. * config/bfin/bfin.c (cbranch_predicted_taken_p): Update. * config/frv/frv.c (frv_print_operand_jump_hint): Update. * config/i386/i386.c (ix86_expand_split_stack_prologue): Update. (ix86_print_operand): Update. (ix86_split_fp_branch): Update. (predict_jump): Update. * config/ia64/ia64.c (ia64_print_operand): Update. * config/mmix/mmix.c (mmix_print_operand): Update. * config/powerpcspe/powerpcspe.c (output_cbranch): Update. (rs6000_expand_split_stack_prologue): Update. * config/rs6000/rs6000.c: Update. * config/s390/s390.c (s390_expand_vec_strlen): Update. (s390_expand_vec_movstr): Update. (s390_expand_cs_tdsi): Update. (s390_expand_split_stack_prologue): Update. * config/sh/sh.c (sh_print_operand): Update. (expand_cbranchsi4): Update. (expand_cbranchdi4): Update. * config/sparc/sparc.c (output_v9branch): Update. * config/spu/spu.c (get_branch_target): Update. (ea_load_store_inline): Update. * config/tilegx/tilegx.c (cbranch_predicted_p): Update. * config/tilepro/tilepro.c: Update. * gcc.dg/predict-8.c: Update. From-SVN: r250239 --- gcc/ChangeLog | 60 ++++++++++++++++++++++++++++++++++ gcc/asan.c | 4 ++- gcc/cfgbuild.c | 8 ++--- gcc/cfgrtl.c | 19 +++++------ gcc/config/aarch64/aarch64.c | 4 +-- gcc/config/alpha/alpha.c | 3 +- gcc/config/arc/arc.c | 4 +-- gcc/config/arm/arm.c | 2 +- gcc/config/bfin/bfin.c | 5 ++- gcc/config/frv/frv.c | 5 ++- gcc/config/i386/i386.c | 12 +++---- gcc/config/ia64/ia64.c | 3 +- gcc/config/mmix/mmix.c | 3 +- gcc/config/powerpcspe/powerpcspe.c | 9 +++--- gcc/config/rs6000/rs6000.c | 9 +++--- gcc/config/s390/s390.c | 20 ++++++------ gcc/config/sh/sh-protos.h | 2 +- gcc/config/sh/sh.c | 66 +++++++++++++++++++++----------------- gcc/config/sh/sh.md | 2 +- gcc/config/sparc/sparc.c | 6 ++-- gcc/config/spu/spu.c | 6 ++-- gcc/config/tilegx/tilegx.c | 5 ++- gcc/config/tilepro/tilepro.c | 5 ++- gcc/doc/rtl.texi | 7 ++-- gcc/emit-rtl.c | 20 +++++++----- gcc/ifcvt.c | 29 +++++++++-------- gcc/internal-fn.c | 12 ++++--- gcc/loop-doloop.c | 13 ++------ gcc/loop-unroll.c | 2 +- gcc/optabs.c | 2 +- gcc/predict.c | 57 +++++++++++++------------------- gcc/predict.h | 5 +++ gcc/profile-count.h | 17 ++++++++++ gcc/reg-notes.def | 11 ++++--- gcc/reorg.c | 3 +- gcc/rtl.h | 1 - gcc/testsuite/ChangeLog | 4 +++ gcc/testsuite/gcc.dg/predict-8.c | 2 +- 38 files changed, 262 insertions(+), 185 deletions(-) (limited to 'gcc') diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 4d04f54..f3bafa5 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,63 @@ +2017-07-16 Jan Hubicka + + * profile-count.h (profile_probability::from_reg_br_prob_note, + profile_probability::to_reg_br_prob_note): New functions. + * doc/rtl.texi (REG_BR_PROB_NOTE): Update documentation. + * reg-notes.h (REG_BR_PROB, REG_BR_PRED): Update docs. + * predict.c (probability_reliable_p): Update. + (edge_probability_reliable_p): Update. + (br_prob_note_reliable_p): Update. + (invert_br_probabilities): Update. + (add_reg_br_prob_note): New function. + (combine_predictions_for_insn): Update. + * asan.c (asan_clear_shadow): Update. + * cfgbuild.c (compute_outgoing_frequencies): Update. + * cfgrtl.c (force_nonfallthru_and_redirect): Update. + (update_br_prob_note): Update. + (rtl_verify_edges): Update. + (purge_dead_edges): Update. + (fixup_reorder_chain): Update. + * emit-rtl.c (try_split): Update. + * ifcvt.c (cond_exec_process_insns): Update. + (cond_exec_process_if_block): Update. + (dead_or_predicable): Update. + * internal-fn.c (expand_addsub_overflow): Update. + (expand_neg_overflow): Update. + (expand_mul_overflow): Update. + * loop-doloop.c (doloop_modify): Update. + * loop-unroll.c (compare_and_jump_seq): Update. + * optabs.c (emit_cmp_and_jump_insn_1): Update. + * predict.h: Update. + * reorg.c (mostly_true_jump): Update. + * rtl.h: Update. + * config/aarch64/aarch64.c (aarch64_emit_unlikely_jump): Update. + * config/alpha/alpha.c (emit_unlikely_jump): Update. + * config/arc/arc.c: (emit_unlikely_jump): Update. + * config/arm/arm.c: (emit_unlikely_jump): Update. + * config/bfin/bfin.c (cbranch_predicted_taken_p): Update. + * config/frv/frv.c (frv_print_operand_jump_hint): Update. + * config/i386/i386.c (ix86_expand_split_stack_prologue): Update. + (ix86_print_operand): Update. + (ix86_split_fp_branch): Update. + (predict_jump): Update. + * config/ia64/ia64.c (ia64_print_operand): Update. + * config/mmix/mmix.c (mmix_print_operand): Update. + * config/powerpcspe/powerpcspe.c (output_cbranch): Update. + (rs6000_expand_split_stack_prologue): Update. + * config/rs6000/rs6000.c: Update. + * config/s390/s390.c (s390_expand_vec_strlen): Update. + (s390_expand_vec_movstr): Update. + (s390_expand_cs_tdsi): Update. + (s390_expand_split_stack_prologue): Update. + * config/sh/sh.c (sh_print_operand): Update. + (expand_cbranchsi4): Update. + (expand_cbranchdi4): Update. + * config/sparc/sparc.c (output_v9branch): Update. + * config/spu/spu.c (get_branch_target): Update. + (ea_load_store_inline): Update. + * config/tilegx/tilegx.c (cbranch_predicted_p): Update. + * config/tilepro/tilepro.c: Update. + 2017-07-16 Eric Botcazou * gimplify.c (mostly_copy_tree_r): Revert latest change. diff --git a/gcc/asan.c b/gcc/asan.c index 89c2731..a56174a 100644 --- a/gcc/asan.c +++ b/gcc/asan.c @@ -1208,7 +1208,9 @@ asan_clear_shadow (rtx shadow_mem, HOST_WIDE_INT len) emit_cmp_and_jump_insns (addr, end, LT, NULL_RTX, Pmode, true, top_label); jump = get_last_insn (); gcc_assert (JUMP_P (jump)); - add_int_reg_note (jump, REG_BR_PROB, REG_BR_PROB_BASE * 80 / 100); + add_reg_br_prob_note (jump, + profile_probability::guessed_always () + .apply_scale (80, 100)); } void diff --git a/gcc/cfgbuild.c b/gcc/cfgbuild.c index 56a2cb9..16b0707 100644 --- a/gcc/cfgbuild.c +++ b/gcc/cfgbuild.c @@ -546,12 +546,10 @@ compute_outgoing_frequencies (basic_block b) probability = XINT (note, 0); e = BRANCH_EDGE (b); e->probability - = profile_probability::from_reg_br_prob_base (probability); - e->count = b->count.apply_probability (probability); + = profile_probability::from_reg_br_prob_note (probability); + e->count = b->count.apply_probability (e->probability); f = FALLTHRU_EDGE (b); - f->probability - = profile_probability::from_reg_br_prob_base (REG_BR_PROB_BASE - - probability); + f->probability = e->probability.invert (); f->count = b->count - e->count; return; } diff --git a/gcc/cfgrtl.c b/gcc/cfgrtl.c index c78cb8e..8c60eed 100644 --- a/gcc/cfgrtl.c +++ b/gcc/cfgrtl.c @@ -1504,8 +1504,8 @@ force_nonfallthru_and_redirect (edge e, basic_block target, rtx jump_label) { int prob = XINT (note, 0); - b->probability = profile_probability::from_reg_br_prob_base (prob); - b->count = e->count.apply_probability (prob); + b->probability = profile_probability::from_reg_br_prob_note (prob); + b->count = e->count.apply_probability (b->probability); e->probability -= e->probability; e->count -= b->count; } @@ -2253,9 +2253,9 @@ update_br_prob_note (basic_block bb) return; note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX); if (!note - || XINT (note, 0) == BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ()) + || XINT (note, 0) == BRANCH_EDGE (bb)->probability.to_reg_br_prob_note ()) return; - XINT (note, 0) = BRANCH_EDGE (bb)->probability.to_reg_br_prob_base (); + XINT (note, 0) = BRANCH_EDGE (bb)->probability.to_reg_br_prob_note (); } /* Get the last insn associated with block BB (that includes barriers and @@ -2456,12 +2456,12 @@ rtl_verify_edges (void) } } else if (XINT (note, 0) - != BRANCH_EDGE (bb)->probability.to_reg_br_prob_base () + != BRANCH_EDGE (bb)->probability.to_reg_br_prob_note () && profile_status_for_fn (cfun) != PROFILE_ABSENT) { error ("verify_flow_info: REG_BR_PROB does not match cfg %i %i", XINT (note, 0), - BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ()); + BRANCH_EDGE (bb)->probability.to_reg_br_prob_note ()); err = 1; } } @@ -3164,9 +3164,9 @@ purge_dead_edges (basic_block bb) b = BRANCH_EDGE (bb); f = FALLTHRU_EDGE (bb); - b->probability = profile_probability::from_reg_br_prob_base + b->probability = profile_probability::from_reg_br_prob_note (XINT (note, 0)); - f->probability = profile_probability::always () - b->probability; + f->probability = b->probability.invert (); b->count = bb->count.apply_probability (b->probability); f->count = bb->count.apply_probability (f->probability); } @@ -3792,7 +3792,8 @@ fixup_reorder_chain (void) rtx note = find_reg_note (bb_end_jump, REG_BR_PROB, 0); if (note - && XINT (note, 0) < REG_BR_PROB_BASE / 2 + && profile_probability::from_reg_br_prob_note + (XINT (note, 0)) < profile_probability::even () && invert_jump (bb_end_jump, (e_fall->dest == EXIT_BLOCK_PTR_FOR_FN (cfun) diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 134d37b..b8a4160 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -12161,10 +12161,8 @@ aarch64_emit_store_exclusive (machine_mode mode, rtx bval, static void aarch64_emit_unlikely_jump (rtx insn) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; - rtx_insn *jump = emit_jump_insn (insn); - add_int_reg_note (jump, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (jump, profile_probability::very_unlikely ()); } /* Expand a compare and swap pattern. */ diff --git a/gcc/config/alpha/alpha.c b/gcc/config/alpha/alpha.c index 00a69c1..1114e66 100644 --- a/gcc/config/alpha/alpha.c +++ b/gcc/config/alpha/alpha.c @@ -4319,10 +4319,9 @@ alpha_expand_builtin_vector_binop (rtx (*gen) (rtx, rtx, rtx), static void emit_unlikely_jump (rtx cond, rtx label) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx x = gen_rtx_IF_THEN_ELSE (VOIDmode, cond, label, pc_rtx); rtx_insn *insn = emit_jump_insn (gen_rtx_SET (pc_rtx, x)); - add_int_reg_note (insn, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (insn, profile_probability::very_unlikely ()); } /* A subroutine of the atomic operation splitters. Emit a load-locked diff --git a/gcc/config/arc/arc.c b/gcc/config/arc/arc.c index ed8fd8e..a92ee4c 100644 --- a/gcc/config/arc/arc.c +++ b/gcc/config/arc/arc.c @@ -9979,10 +9979,8 @@ arc_post_atomic_barrier (enum memmodel model) static void emit_unlikely_jump (rtx insn) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; - rtx_insn *jump = emit_jump_insn (insn); - add_int_reg_note (jump, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (jump, profile_probability::very_unlikely ()); } /* Expand code to perform a 8 or 16-bit compare and swap by doing diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c index f59132c..1b7b382 100644 --- a/gcc/config/arm/arm.c +++ b/gcc/config/arm/arm.c @@ -28271,7 +28271,7 @@ emit_unlikely_jump (rtx insn) int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx_insn *jump = emit_jump_insn (insn); - add_int_reg_note (jump, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (jump, profile_probability::very_unlikely ()); } /* Expand a compare and swap pattern. */ diff --git a/gcc/config/bfin/bfin.c b/gcc/config/bfin/bfin.c index 2073927..9fe90fc 100644 --- a/gcc/config/bfin/bfin.c +++ b/gcc/config/bfin/bfin.c @@ -2435,9 +2435,8 @@ cbranch_predicted_taken_p (rtx insn) if (x) { - int pred_val = XINT (x, 0); - - return pred_val >= REG_BR_PROB_BASE / 2; + return profile_probability::from_reg_br_prob_note (XINT (x, 0)) + >= profile_probability::even (); } return 0; diff --git a/gcc/config/frv/frv.c b/gcc/config/frv/frv.c index 0918b98..c571d63 100644 --- a/gcc/config/frv/frv.c +++ b/gcc/config/frv/frv.c @@ -2621,7 +2621,6 @@ frv_print_operand_jump_hint (rtx_insn *insn) rtx note; rtx labelref; int ret; - int prob = -1; enum { UNKNOWN, BACKWARD, FORWARD } jump_type = UNKNOWN; gcc_assert (JUMP_P (insn)); @@ -2647,8 +2646,8 @@ frv_print_operand_jump_hint (rtx_insn *insn) else { - prob = XINT (note, 0); - ret = ((prob >= (REG_BR_PROB_BASE / 2)) + ret = ((profile_probability::from_reg_br_prob_note (XINT (note, 0)) + >= profile_probability::even ()) ? FRV_JUMP_LIKELY : FRV_JUMP_NOT_LIKELY); } diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 42e0dda..eac7681 100644 --- a/gcc/config/i386/i386.c +++ b/gcc/config/i386/i386.c @@ -15783,8 +15783,7 @@ ix86_expand_split_stack_prologue (void) JUMP_LABEL (jump_insn) = label; /* Mark the jump as very likely to be taken. */ - add_int_reg_note (jump_insn, REG_BR_PROB, - REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100); + add_reg_br_prob_note (jump_insn, profile_probability::very_likely ()); if (split_stack_fn == NULL_RTX) { @@ -19128,7 +19127,8 @@ ix86_print_operand (FILE *file, rtx x, int code) x = find_reg_note (current_output_insn, REG_BR_PROB, 0); if (x) { - int pred_val = XINT (x, 0); + int pred_val = profile_probability::from_reg_br_prob_note + (XINT (x, 0)).to_reg_br_prob_base (); if (pred_val < REG_BR_PROB_BASE * 45 / 100 || pred_val > REG_BR_PROB_BASE * 55 / 100) @@ -23865,8 +23865,8 @@ ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2, (pc_rtx, gen_rtx_IF_THEN_ELSE (VOIDmode, condition, target1, target2))); - if (split_branch_probability >= 0) - add_int_reg_note (i, REG_BR_PROB, split_branch_probability); + if (split_branch_probability.initialized_p ()) + add_reg_br_prob_note (i, split_branch_probability); } void @@ -26911,7 +26911,7 @@ predict_jump (int prob) { rtx_insn *insn = get_last_insn (); gcc_assert (JUMP_P (insn)); - add_int_reg_note (insn, REG_BR_PROB, prob); + add_reg_br_prob_note (insn, profile_probability::from_reg_br_prob_base (prob)); } /* Helper function for the string operations below. Dest VARIABLE whether diff --git a/gcc/config/ia64/ia64.c b/gcc/config/ia64/ia64.c index 617d188..c8e4c74 100644 --- a/gcc/config/ia64/ia64.c +++ b/gcc/config/ia64/ia64.c @@ -5502,7 +5502,8 @@ ia64_print_operand (FILE * file, rtx x, int code) x = find_reg_note (current_output_insn, REG_BR_PROB, 0); if (x) { - int pred_val = XINT (x, 0); + int pred_val = profile_probability::from_reg_br_prob_note + (XINT (x, 0)).to_reg_br_prob_base (); /* Guess top and bottom 10% statically predicted. */ if (pred_val < REG_BR_PROB_BASE / 50 diff --git a/gcc/config/mmix/mmix.c b/gcc/config/mmix/mmix.c index 3278688..9849c19 100644 --- a/gcc/config/mmix/mmix.c +++ b/gcc/config/mmix/mmix.c @@ -1536,7 +1536,8 @@ mmix_print_operand (FILE *stream, rtx x, int code) if (TARGET_BRANCH_PREDICT) { x = find_reg_note (current_output_insn, REG_BR_PROB, 0); - if (x && XINT (x, 0) > REG_BR_PROB_BASE / 2) + if (x && profile_probability::from_reg_br_prob_note (XINT (x, 0)) + > profile_probability::even ()) putc ('P', stream); } return; diff --git a/gcc/config/powerpcspe/powerpcspe.c b/gcc/config/powerpcspe/powerpcspe.c index 06d66d7..b94afd5 100644 --- a/gcc/config/powerpcspe/powerpcspe.c +++ b/gcc/config/powerpcspe/powerpcspe.c @@ -25333,7 +25333,8 @@ output_cbranch (rtx op, const char *label, int reversed, rtx_insn *insn) if (note != NULL_RTX) { /* PROB is the difference from 50%. */ - int prob = XINT (note, 0) - REG_BR_PROB_BASE / 2; + int prob = profile_probability::from_reg_br_prob_note (XINT (note, 0)) + .to_reg_br_prob_base () - REG_BR_PROB_BASE / 2; /* Only hint for highly probable/improbable branches on newer cpus when we have real profile data, as static prediction overrides processor @@ -26123,10 +26124,9 @@ rs6000_split_signbit (rtx dest, rtx src) static void emit_unlikely_jump (rtx cond, rtx label) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx x = gen_rtx_IF_THEN_ELSE (VOIDmode, cond, label, pc_rtx); rtx_insn *insn = emit_jump_insn (gen_rtx_SET (pc_rtx, x)); - add_int_reg_note (insn, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (insn, profile_probability::very_unlikely ()); } /* A subroutine of the atomic operation splitters. Emit a load-locked @@ -32192,8 +32192,7 @@ rs6000_expand_split_stack_prologue (void) insn = emit_jump_insn (gen_rtx_SET (pc_rtx, jump)); JUMP_LABEL (insn) = ok_label; /* Mark the jump as very likely to be taken. */ - add_int_reg_note (insn, REG_BR_PROB, - REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100); + add_reg_br_prob_note (insn, profile_probability::very_likely ()); lr = gen_rtx_REG (Pmode, LR_REGNO); insn = emit_move_insn (r0, lr); diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c index 9bf6f37..e49a091 100644 --- a/gcc/config/rs6000/rs6000.c +++ b/gcc/config/rs6000/rs6000.c @@ -22759,7 +22759,8 @@ output_cbranch (rtx op, const char *label, int reversed, rtx_insn *insn) if (note != NULL_RTX) { /* PROB is the difference from 50%. */ - int prob = XINT (note, 0) - REG_BR_PROB_BASE / 2; + int prob = profile_probability::from_reg_br_prob_note (XINT (note, 0)) + .to_reg_br_prob_base () - REG_BR_PROB_BASE / 2; /* Only hint for highly probable/improbable branches on newer cpus when we have real profile data, as static prediction overrides processor @@ -23528,10 +23529,9 @@ rs6000_split_signbit (rtx dest, rtx src) static void emit_unlikely_jump (rtx cond, rtx label) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx x = gen_rtx_IF_THEN_ELSE (VOIDmode, cond, label, pc_rtx); rtx_insn *insn = emit_jump_insn (gen_rtx_SET (pc_rtx, x)); - add_int_reg_note (insn, REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (insn, profile_probability::very_unlikely ()); } /* A subroutine of the atomic operation splitters. Emit a load-locked @@ -29284,8 +29284,7 @@ rs6000_expand_split_stack_prologue (void) insn = emit_jump_insn (gen_rtx_SET (pc_rtx, jump)); JUMP_LABEL (insn) = ok_label; /* Mark the jump as very likely to be taken. */ - add_int_reg_note (insn, REG_BR_PROB, - REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100); + add_reg_br_prob_note (insn, profile_probability::very_likely ()); lr = gen_rtx_REG (Pmode, LR_REGNO); insn = emit_move_insn (r0, lr); diff --git a/gcc/config/s390/s390.c b/gcc/config/s390/s390.c index 31ced21..bbae89b 100644 --- a/gcc/config/s390/s390.c +++ b/gcc/config/s390/s390.c @@ -5720,8 +5720,6 @@ s390_emit_ccraw_jump (HOST_WIDE_INT mask, enum rtx_code comparison, rtx label) void s390_expand_vec_strlen (rtx target, rtx string, rtx alignment) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; - int very_likely = REG_BR_PROB_BASE - 1; rtx highest_index_to_load_reg = gen_reg_rtx (Pmode); rtx str_reg = gen_reg_rtx (V16QImode); rtx str_addr_base_reg = gen_reg_rtx (Pmode); @@ -5792,7 +5790,8 @@ s390_expand_vec_strlen (rtx target, rtx string, rtx alignment) GEN_INT (VSTRING_FLAG_ZS | VSTRING_FLAG_CS))); add_int_reg_note (s390_emit_ccraw_jump (8, NE, loop_start_label), - REG_BR_PROB, very_likely); + REG_BR_PROB, + profile_probability::very_likely ().to_reg_br_prob_note ()); emit_insn (gen_vec_extractv16qi (len, result_reg, GEN_INT (7))); /* If the string pointer wasn't aligned we have loaded less then 16 @@ -5812,8 +5811,8 @@ s390_expand_vec_strlen (rtx target, rtx string, rtx alignment) emit_insn (gen_movsicc (str_idx_reg, cond, highest_index_to_load_reg, str_idx_reg)); - add_int_reg_note (s390_emit_jump (is_aligned_label, cond), REG_BR_PROB, - very_unlikely); + add_reg_br_prob_note (s390_emit_jump (is_aligned_label, cond), + profile_probability::very_unlikely ()); expand_binop (Pmode, add_optab, str_idx_reg, GEN_INT (-16), str_idx_reg, 1, OPTAB_DIRECT); @@ -5829,7 +5828,6 @@ s390_expand_vec_strlen (rtx target, rtx string, rtx alignment) void s390_expand_vec_movstr (rtx result, rtx dst, rtx src) { - int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx temp = gen_reg_rtx (Pmode); rtx src_addr = XEXP (src, 0); rtx dst_addr = XEXP (dst, 0); @@ -5906,7 +5904,8 @@ s390_expand_vec_movstr (rtx result, rtx dst, rtx src) emit_insn (gen_vec_vfenesv16qi (vpos, vsrc, vsrc, GEN_INT (VSTRING_FLAG_ZS | VSTRING_FLAG_CS))); add_int_reg_note (s390_emit_ccraw_jump (8, EQ, done_label), - REG_BR_PROB, very_unlikely); + REG_BR_PROB, profile_probability::very_unlikely () + .to_reg_br_prob_note ()); emit_move_insn (gen_rtx_MEM (V16QImode, gen_rtx_PLUS (Pmode, dst_addr_reg, offset)), @@ -6929,7 +6928,6 @@ s390_expand_cs_tdsi (machine_mode mode, rtx btarget, rtx vtarget, rtx mem, if (do_const_opt) { - const int very_unlikely = REG_BR_PROB_BASE / 100 - 1; rtx cc = gen_rtx_REG (CCZmode, CC_REGNUM); skip_cs_label = gen_label_rtx (); @@ -6950,7 +6948,8 @@ s390_expand_cs_tdsi (machine_mode mode, rtx btarget, rtx vtarget, rtx mem, emit_insn (gen_rtx_SET (cc, gen_rtx_COMPARE (CCZmode, output, cmp))); } s390_emit_jump (skip_cs_label, gen_rtx_NE (VOIDmode, cc, const0_rtx)); - add_int_reg_note (get_last_insn (), REG_BR_PROB, very_unlikely); + add_reg_br_prob_note (get_last_insn (), + profile_probability::very_unlikely ()); /* If the jump is not taken, OUTPUT is the expected value. */ cmp = output; /* Reload newval to a register manually, *after* the compare and jump @@ -11678,7 +11677,8 @@ s390_expand_split_stack_prologue (void) LABEL_NUSES (call_done)++; /* Mark the jump as very unlikely to be taken. */ - add_int_reg_note (insn, REG_BR_PROB, REG_BR_PROB_BASE / 100); + add_reg_br_prob_note (insn, + profile_probability::very_unlikely ()); if (cfun->machine->split_stack_varargs_pointer != NULL_RTX) { diff --git a/gcc/config/sh/sh-protos.h b/gcc/config/sh/sh-protos.h index 59be4c0..305ab2c 100644 --- a/gcc/config/sh/sh-protos.h +++ b/gcc/config/sh/sh-protos.h @@ -68,7 +68,7 @@ extern bool sh_expand_strlen (rtx *); extern void sh_expand_setmem (rtx *); extern enum rtx_code prepare_cbranch_operands (rtx *, machine_mode mode, enum rtx_code comparison); -extern void expand_cbranchsi4 (rtx *operands, enum rtx_code comparison, int); +extern void expand_cbranchsi4 (rtx *operands, enum rtx_code comparison); extern bool expand_cbranchdi4 (rtx *operands, enum rtx_code comparison); extern void sh_emit_scc_to_t (enum rtx_code, rtx, rtx); extern void sh_emit_compare_and_branch (rtx *, machine_mode); diff --git a/gcc/config/sh/sh.c b/gcc/config/sh/sh.c index ff79b93..6f01dcb 100644 --- a/gcc/config/sh/sh.c +++ b/gcc/config/sh/sh.c @@ -1139,7 +1139,9 @@ sh_print_operand (FILE *stream, rtx x, int code) { rtx note = find_reg_note (current_output_insn, REG_BR_PROB, 0); - if (note && XINT (note, 0) * 2 < REG_BR_PROB_BASE) + if (note + && profile_probability::from_reg_br_prob_note (XINT (note, 0)) + < profile_probability::even ()) fputs ("/u", stream); break; } @@ -1999,8 +2001,9 @@ prepare_cbranch_operands (rtx *operands, machine_mode mode, return comparison; } -void -expand_cbranchsi4 (rtx *operands, enum rtx_code comparison, int probability) +static void +expand_cbranchsi4 (rtx *operands, enum rtx_code comparison, + profile_probability probability) { rtx (*branch_expander) (rtx) = gen_branch_true; comparison = prepare_cbranch_operands (operands, SImode, comparison); @@ -2015,8 +2018,15 @@ expand_cbranchsi4 (rtx *operands, enum rtx_code comparison, int probability) gen_rtx_fmt_ee (comparison, SImode, operands[1], operands[2]))); rtx_insn *jump = emit_jump_insn (branch_expander (operands[3])); - if (probability >= 0) - add_int_reg_note (jump, REG_BR_PROB, probability); + if (probability.initialized_p ()) + add_reg_br_prob_note (jump, probability); +} + +void +expand_cbranchsi4 (rtx *operands, enum rtx_code comparison) +{ + expand_cbranchsi4 (operands, comparison, + profile_probability::uninitialized ()); } /* ??? How should we distribute probabilities when more than one branch @@ -2043,8 +2053,10 @@ expand_cbranchdi4 (rtx *operands, enum rtx_code comparison) rtx_code_label *skip_label = NULL; rtx op1h, op1l, op2h, op2l; int num_branches; - int prob, rev_prob; - int msw_taken_prob = -1, msw_skip_prob = -1, lsw_taken_prob = -1; + profile_probability prob, rev_prob; + profile_probability msw_taken_prob = profile_probability::uninitialized (), + msw_skip_prob = profile_probability::uninitialized (), + lsw_taken_prob = profile_probability::uninitialized (); comparison = prepare_cbranch_operands (operands, DImode, comparison); op1h = gen_highpart_mode (SImode, DImode, operands[1]); @@ -2053,34 +2065,28 @@ expand_cbranchdi4 (rtx *operands, enum rtx_code comparison) op2l = gen_lowpart (SImode, operands[2]); msw_taken = msw_skip = lsw_taken = LAST_AND_UNUSED_RTX_CODE; prob = split_branch_probability; - rev_prob = REG_BR_PROB_BASE - prob; + rev_prob = prob.invert (); switch (comparison) { case EQ: msw_skip = NE; lsw_taken = EQ; - if (prob >= 0) + if (prob.initialized_p ()) { - // If we had more precision, we'd use rev_prob - (rev_prob >> 32) . + /* FIXME: This is not optimal. We do not really know the probablity + that values differ by MCW only, but we should probably distribute + probabilities more evenly. */ msw_skip_prob = rev_prob; - if (REG_BR_PROB_BASE <= 65535) - lsw_taken_prob = prob ? REG_BR_PROB_BASE : 0; - else - { - lsw_taken_prob - = (prob - ? (REG_BR_PROB_BASE - - ((gcov_type) REG_BR_PROB_BASE * rev_prob - / ((gcov_type) prob << 32))) - : 0); - } + lsw_taken_prob = prob > profile_probability::never () + ? profile_probability::guessed_always () + : profile_probability::guessed_never (); } break; case NE: msw_taken = NE; msw_taken_prob = prob; lsw_taken = NE; - lsw_taken_prob = 0; + lsw_taken_prob = profile_probability::guessed_never (); break; case GTU: case GT: msw_taken = comparison; @@ -2133,18 +2139,20 @@ expand_cbranchdi4 (rtx *operands, enum rtx_code comparison) if (comparison != EQ && comparison != NE && num_branches > 1) { if (!CONSTANT_P (operands[2]) - && prob >= (int) (REG_BR_PROB_BASE * 3 / 8U) - && prob <= (int) (REG_BR_PROB_BASE * 5 / 8U)) - { - msw_taken_prob = prob / 2U; - msw_skip_prob - = REG_BR_PROB_BASE * rev_prob / (REG_BR_PROB_BASE + rev_prob); + && prob.initialized_p () + && prob.to_reg_br_prob_base () >= (int) (REG_BR_PROB_BASE * 3 / 8U) + && prob.to_reg_br_prob_base () <= (int) (REG_BR_PROB_BASE * 5 / 8U)) + { + msw_taken_prob = prob.apply_scale (1, 2); + msw_skip_prob = rev_prob.apply_scale (REG_BR_PROB_BASE, + rev_prob.to_reg_br_prob_base () + + REG_BR_PROB_BASE); lsw_taken_prob = prob; } else { msw_taken_prob = prob; - msw_skip_prob = REG_BR_PROB_BASE; + msw_skip_prob = profile_probability::guessed_always (); /* ??? If we have a constant op2h, should we use that when calculating lsw_taken_prob? */ lsw_taken_prob = prob; diff --git a/gcc/config/sh/sh.md b/gcc/config/sh/sh.md index e19e977..733c777 100644 --- a/gcc/config/sh/sh.md +++ b/gcc/config/sh/sh.md @@ -1188,7 +1188,7 @@ (clobber (reg:SI T_REG))] "can_create_pseudo_p ()" { - expand_cbranchsi4 (operands, LAST_AND_UNUSED_RTX_CODE, -1); + expand_cbranchsi4 (operands, LAST_AND_UNUSED_RTX_CODE); DONE; }) diff --git a/gcc/config/sparc/sparc.c b/gcc/config/sparc/sparc.c index dae2411..f4f318a 100644 --- a/gcc/config/sparc/sparc.c +++ b/gcc/config/sparc/sparc.c @@ -7999,7 +7999,8 @@ output_cbranch (rtx op, rtx dest, int label, int reversed, int annul, if (*labelno && insn && (note = find_reg_note (insn, REG_BR_PROB, NULL_RTX))) { strcpy (p, - ((XINT (note, 0) >= REG_BR_PROB_BASE / 2) ^ far) + ((profile_probability::from_reg_br_prob_note (XINT (note, 0)) + >= profile_probability::even ()) ^ far) ? ",pt" : ",pn"); p += 3; spaces -= 3; @@ -8462,7 +8463,8 @@ output_v9branch (rtx op, rtx dest, int reg, int label, int reversed, if (insn && (note = find_reg_note (insn, REG_BR_PROB, NULL_RTX))) { strcpy (p, - ((XINT (note, 0) >= REG_BR_PROB_BASE / 2) ^ far) + ((profile_probability::from_reg_br_prob_note (XINT (note, 0)) + >= profile_probability::even ()) ^ far) ? ",pt" : ",pn"); p += 3; } diff --git a/gcc/config/spu/spu.c b/gcc/config/spu/spu.c index fcb85c0..efee614 100644 --- a/gcc/config/spu/spu.c +++ b/gcc/config/spu/spu.c @@ -2195,7 +2195,8 @@ get_branch_target (rtx_insn *branch) { /* If the more probable case is not a fall through, then try a branch hint. */ - int prob = XINT (note, 0); + int prob = profile_probability::from_reg_br_prob_note + (XINT (note, 0)).to_reg_br_prob_base (); if (prob > (REG_BR_PROB_BASE * 6 / 10) && GET_CODE (XEXP (src, 1)) != PC) lab = XEXP (src, 1); @@ -4335,8 +4336,7 @@ ea_load_store_inline (rtx mem, bool is_store, rtx ea_addr, rtx data_addr) gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp, hit_ref, pc_rtx))); /* Say that this branch is very likely to happen. */ - v = REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100 - 1; - add_int_reg_note (insn, REG_BR_PROB, v); + add_reg_br_prob_note (insn, profile_probability::very_likely ()); ea_load_store (mem, is_store, ea_addr, data_addr); cont_label = gen_label_rtx (); diff --git a/gcc/config/tilegx/tilegx.c b/gcc/config/tilegx/tilegx.c index e070e7e..dafb49d 100644 --- a/gcc/config/tilegx/tilegx.c +++ b/gcc/config/tilegx/tilegx.c @@ -2619,9 +2619,8 @@ cbranch_predicted_p (rtx_insn *insn) if (x) { - int pred_val = XINT (x, 0); - - return pred_val >= REG_BR_PROB_BASE / 2; + return profile_probability::from_reg_br_prob_note (XINT (x, 0)) + >= profile_probability::even (); } return false; diff --git a/gcc/config/tilepro/tilepro.c b/gcc/config/tilepro/tilepro.c index 81019c1..80475b9 100644 --- a/gcc/config/tilepro/tilepro.c +++ b/gcc/config/tilepro/tilepro.c @@ -2419,9 +2419,8 @@ cbranch_predicted_p (rtx_insn *insn) if (x) { - int pred_val = XINT (x, 0); - - return pred_val >= REG_BR_PROB_BASE / 2; + return profile_probability::from_reg_br_prob_note (XINT (x, 0)) + >= profile_probability::even (); } return false; diff --git a/gcc/doc/rtl.texi b/gcc/doc/rtl.texi index b02e5a1..6e2799a1 100644 --- a/gcc/doc/rtl.texi +++ b/gcc/doc/rtl.texi @@ -4022,9 +4022,10 @@ are stored in the @code{REG_NOTES} field of an insn. @item REG_BR_PROB This is used to specify the ratio of branches to non-branches of a branch insn according to the profile data. The note is represented -as an @code{int_list} expression whose integer value is between 0 and -REG_BR_PROB_BASE. Larger values indicate a higher probability that -the branch will be taken. +as an @code{int_list} expression whose integer value is an encoding +of @code{profile_probability} type. @code{profile_probability} provide +member function @code{from_reg_br_prob_note} and @code{to_reg_br_prob_note} +to extract and store the probability into the RTL encoding. @findex REG_BR_PRED @item REG_BR_PRED diff --git a/gcc/emit-rtl.c b/gcc/emit-rtl.c index 041f2a7..2bc5d56 100644 --- a/gcc/emit-rtl.c +++ b/gcc/emit-rtl.c @@ -59,6 +59,7 @@ along with GCC; see the file COPYING3. If not see #include "rtl-iter.h" #include "stor-layout.h" #include "opts.h" +#include "predict.h" struct target_rtl default_target_rtl; #if SWITCHABLE_TARGET @@ -190,9 +191,8 @@ static reg_attrs *get_reg_attrs (tree, int); static rtx gen_const_vector (machine_mode, int); static void copy_rtx_if_shared_1 (rtx *orig); -/* Probability of the conditional branch currently proceeded by try_split. - Set to -1 otherwise. */ -int split_branch_probability = -1; +/* Probability of the conditional branch currently proceeded by try_split. */ +profile_probability split_branch_probability; /* Returns a hash code for X (which is a really a CONST_INT). */ @@ -3662,7 +3662,7 @@ try_split (rtx pat, rtx_insn *trial, int last) rtx_insn *before, *after; rtx note; rtx_insn *seq, *tem; - int probability; + profile_probability probability; rtx_insn *insn_last, *insn; int njumps = 0; rtx_insn *call_insn = NULL; @@ -3673,12 +3673,16 @@ try_split (rtx pat, rtx_insn *trial, int last) if (any_condjump_p (trial) && (note = find_reg_note (trial, REG_BR_PROB, 0))) - split_branch_probability = XINT (note, 0); + split_branch_probability + = profile_probability::from_reg_br_prob_note (XINT (note, 0)); + else + split_branch_probability = profile_probability::uninitialized (); + probability = split_branch_probability; seq = split_insns (pat, trial); - split_branch_probability = -1; + split_branch_probability = profile_probability::uninitialized (); if (!seq) return trial; @@ -3709,7 +3713,7 @@ try_split (rtx pat, rtx_insn *trial, int last) CROSSING_JUMP_P (insn) = CROSSING_JUMP_P (trial); mark_jump_label (PATTERN (insn), insn, 0); njumps++; - if (probability != -1 + if (probability.initialized_p () && any_condjump_p (insn) && !find_reg_note (insn, REG_BR_PROB, 0)) { @@ -3718,7 +3722,7 @@ try_split (rtx pat, rtx_insn *trial, int last) is responsible for this step using split_branch_probability variable. */ gcc_assert (njumps == 1); - add_int_reg_note (insn, REG_BR_PROB, probability); + add_reg_br_prob_note (insn, probability); } } } diff --git a/gcc/ifcvt.c b/gcc/ifcvt.c index fd682a4..46a13c4 100644 --- a/gcc/ifcvt.c +++ b/gcc/ifcvt.c @@ -84,8 +84,6 @@ static rtx_insn *last_active_insn (basic_block, int); static rtx_insn *find_active_insn_before (basic_block, rtx_insn *); static rtx_insn *find_active_insn_after (basic_block, rtx_insn *); static basic_block block_fallthru (basic_block); -static int cond_exec_process_insns (ce_if_block *, rtx_insn *, rtx, rtx, int, - int); static rtx cond_exec_get_condition (rtx_insn *); static rtx noce_get_condition (rtx_insn *, rtx_insn **, bool); static int noce_operand_ok (const_rtx); @@ -335,7 +333,8 @@ cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED, /* if block information */rtx_insn *start, /* first insn to look at */rtx end, /* last insn to look at */rtx test, - /* conditional execution test */int prob_val, + /* conditional execution test */profile_probability + prob_val, /* probability of branch taken. */int mod_ok) { int must_be_last = FALSE; @@ -410,10 +409,11 @@ cond_exec_process_insns (ce_if_block *ce_info ATTRIBUTE_UNUSED, validate_change (insn, &PATTERN (insn), pattern, 1); - if (CALL_P (insn) && prob_val >= 0) + if (CALL_P (insn) && prob_val.initialized_p ()) validate_change (insn, ®_NOTES (insn), gen_rtx_INT_LIST ((machine_mode) REG_BR_PROB, - prob_val, REG_NOTES (insn)), 1); + prob_val.to_reg_br_prob_note (), + REG_NOTES (insn)), 1); insn_done: if (insn == end) @@ -472,8 +472,8 @@ cond_exec_process_if_block (ce_if_block * ce_info, int then_mod_ok; /* whether conditional mods are ok in THEN */ rtx true_expr; /* test for else block insns */ rtx false_expr; /* test for then block insns */ - int true_prob_val; /* probability of else block */ - int false_prob_val; /* probability of then block */ + profile_probability true_prob_val;/* probability of else block */ + profile_probability false_prob_val;/* probability of then block */ rtx_insn *then_last_head = NULL; /* Last match at the head of THEN */ rtx_insn *else_last_head = NULL; /* Last match at the head of ELSE */ rtx_insn *then_first_tail = NULL; /* First match at the tail of THEN */ @@ -618,13 +618,13 @@ cond_exec_process_if_block (ce_if_block * ce_info, note = find_reg_note (BB_END (test_bb), REG_BR_PROB, NULL_RTX); if (note) { - true_prob_val = XINT (note, 0); - false_prob_val = REG_BR_PROB_BASE - true_prob_val; + true_prob_val = profile_probability::from_reg_br_prob_note (XINT (note, 0)); + false_prob_val = true_prob_val.invert (); } else { - true_prob_val = -1; - false_prob_val = -1; + true_prob_val = profile_probability::uninitialized (); + false_prob_val = profile_probability::uninitialized (); } /* If we have && or || tests, do them here. These tests are in the adjacent @@ -5121,7 +5121,9 @@ dead_or_predicable (basic_block test_bb, basic_block merge_bb, return FALSE; rtx note = find_reg_note (jump, REG_BR_PROB, NULL_RTX); - int prob_val = (note ? XINT (note, 0) : -1); + profile_probability prob_val + = (note ? profile_probability::from_reg_br_prob_note (XINT (note, 0)) + : profile_probability::uninitialized ()); if (reversep) { @@ -5130,8 +5132,7 @@ dead_or_predicable (basic_block test_bb, basic_block merge_bb, return FALSE; cond = gen_rtx_fmt_ee (rev, GET_MODE (cond), XEXP (cond, 0), XEXP (cond, 1)); - if (prob_val >= 0) - prob_val = REG_BR_PROB_BASE - prob_val; + prob_val = prob_val.invert (); } if (cond_exec_process_insns (NULL, head, end, cond, prob_val, 0) diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c index 1dc7541..1c9d1ea 100644 --- a/gcc/internal-fn.c +++ b/gcc/internal-fn.c @@ -741,7 +741,8 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) - add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + add_reg_br_prob_note (last, + profile_probability::very_unlikely ()); emit_jump (done_label); goto do_error_label; } @@ -961,7 +962,8 @@ expand_addsub_overflow (location_t loc, tree_code code, tree lhs, && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) - add_int_reg_note (last, REG_BR_PROB, PROB_UNLIKELY); + add_reg_br_prob_note (last, + profile_probability::very_unlikely ()); emit_jump (done_label); goto do_error_label; } @@ -1110,7 +1112,8 @@ expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan, && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) - add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + add_reg_br_prob_note (last, + profile_probability::very_unlikely ()); emit_jump (done_label); } else @@ -1431,7 +1434,8 @@ expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1, && JUMP_P (last) && any_condjump_p (last) && !find_reg_note (last, REG_BR_PROB, 0)) - add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY); + add_reg_br_prob_note (last, + profile_probability::very_unlikely ()); emit_jump (done_label); } else diff --git a/gcc/loop-doloop.c b/gcc/loop-doloop.c index b4995b3..49861b8 100644 --- a/gcc/loop-doloop.c +++ b/gcc/loop-doloop.c @@ -419,7 +419,6 @@ doloop_modify (struct loop *loop, struct niter_desc *desc, bool increment_count; basic_block loop_end = desc->out_edge->src; machine_mode mode; - rtx true_prob_val; widest_int iterations; jump_insn = BB_END (loop_end); @@ -434,10 +433,6 @@ doloop_modify (struct loop *loop, struct niter_desc *desc, fputs (" iterations).\n", dump_file); } - /* Get the probability of the original branch. If it exists we would - need to update REG_BR_PROB of the new jump_insn. */ - true_prob_val = find_reg_note (jump_insn, REG_BR_PROB, NULL_RTX); - /* Discard original jump to continue loop. The original compare result may still be live, so it cannot be discarded explicitly. */ delete_insn (jump_insn); @@ -580,12 +575,8 @@ doloop_modify (struct loop *loop, struct niter_desc *desc, add_reg_note (jump_insn, REG_NONNEG, NULL_RTX); /* Update the REG_BR_PROB note. */ - if (true_prob_val && desc->in_edge->probability.initialized_p ()) - { - /* Seems safer to use the branch probability. */ - add_int_reg_note (jump_insn, REG_BR_PROB, - desc->in_edge->probability.to_reg_br_prob_base ()); - } + if (desc->in_edge->probability.initialized_p ()) + add_reg_br_prob_note (jump_insn, desc->in_edge->probability); } /* Called through note_stores. */ diff --git a/gcc/loop-unroll.c b/gcc/loop-unroll.c index 5e2e77a..84145bb 100644 --- a/gcc/loop-unroll.c +++ b/gcc/loop-unroll.c @@ -816,7 +816,7 @@ compare_and_jump_seq (rtx op0, rtx op1, enum rtx_code comp, LABEL_NUSES (label)++; } if (prob.initialized_p ()) - add_int_reg_note (jump, REG_BR_PROB, prob.to_reg_br_prob_base ()); + add_reg_br_prob_note (jump, prob); seq = get_insns (); end_sequence (); diff --git a/gcc/optabs.c b/gcc/optabs.c index 8fd5d91..3905132 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -4003,7 +4003,7 @@ emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, && JUMP_P (insn) && any_condjump_p (insn) && !find_reg_note (insn, REG_BR_PROB, 0)) - add_int_reg_note (insn, REG_BR_PROB, prob.to_reg_br_prob_base ()); + add_reg_br_prob_note (insn, prob); } /* Generate code to compare X with Y so that the condition codes are diff --git a/gcc/predict.c b/gcc/predict.c index 2a7dbfa..310d9b0 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -513,35 +513,11 @@ edge_predicted_by_p (edge e, enum br_predictor predictor, bool taken) return false; } -/* Return true when the probability of edge is reliable. - - The profile guessing code is good at predicting branch outcome (ie. - taken/not taken), that is predicted right slightly over 75% of time. - It is however notoriously poor on predicting the probability itself. - In general the profile appear a lot flatter (with probabilities closer - to 50%) than the reality so it is bad idea to use it to drive optimization - such as those disabling dynamic branch prediction for well predictable - branches. - - There are two exceptions - edges leading to noreturn edges and edges - predicted by number of iterations heuristics are predicted well. This macro - should be able to distinguish those, but at the moment it simply check for - noreturn heuristic that is only one giving probability over 99% or bellow - 1%. In future we might want to propagate reliability information across the - CFG if we find this information useful on multiple places. */ -static bool -probability_reliable_p (int prob) -{ - return (profile_status_for_fn (cfun) == PROFILE_READ - || (profile_status_for_fn (cfun) == PROFILE_GUESSED - && (prob <= HITRATE (1) || prob >= HITRATE (99)))); -} - /* Same predicate as above, working on edges. */ bool edge_probability_reliable_p (const_edge e) { - return e->probability.reliable_p (); + return e->probability.probably_reliable_p (); } /* Same predicate as edge_probability_reliable_p, working on notes. */ @@ -549,7 +525,8 @@ bool br_prob_note_reliable_p (const_rtx note) { gcc_assert (REG_NOTE_KIND (note) == REG_BR_PROB); - return probability_reliable_p (XINT (note, 0)); + return profile_probability::from_reg_br_prob_note + (XINT (note, 0)).probably_reliable_p (); } static void @@ -723,7 +700,8 @@ invert_br_probabilities (rtx insn) for (note = REG_NOTES (insn); note; note = XEXP (note, 1)) if (REG_NOTE_KIND (note) == REG_BR_PROB) - XINT (note, 0) = REG_BR_PROB_BASE - XINT (note, 0); + XINT (note, 0) = profile_probability::from_reg_br_prob_note + (XINT (note, 0)).invert ().to_reg_br_prob_note (); else if (REG_NOTE_KIND (note) == REG_BR_PRED) XEXP (XEXP (note, 0), 1) = GEN_INT (REG_BR_PROB_BASE - INTVAL (XEXP (XEXP (note, 0), 1))); @@ -870,6 +848,15 @@ set_even_probabilities (basic_block bb, e->probability = profile_probability::never (); } +/* Add REG_BR_PROB note to JUMP with PROB. */ + +void +add_reg_br_prob_note (rtx_insn *jump, profile_probability prob) +{ + gcc_checking_assert (JUMP_P (jump) && !find_reg_note (jump, REG_BR_PROB, 0)); + add_int_reg_note (jump, REG_BR_PROB, prob.to_reg_br_prob_note ()); +} + /* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB note if not already present. Remove now useless REG_BR_PRED notes. */ @@ -968,26 +955,26 @@ combine_predictions_for_insn (rtx_insn *insn, basic_block bb) if (!prob_note) { - add_int_reg_note (insn, REG_BR_PROB, combined_probability); + profile_probability p + = profile_probability::from_reg_br_prob_base (combined_probability); + add_reg_br_prob_note (insn, p); /* Save the prediction into CFG in case we are seeing non-degenerated conditional jump. */ if (!single_succ_p (bb)) { - BRANCH_EDGE (bb)->probability - = profile_probability::from_reg_br_prob_base (combined_probability); + BRANCH_EDGE (bb)->probability = p; FALLTHRU_EDGE (bb)->probability = BRANCH_EDGE (bb)->probability.invert (); } } else if (!single_succ_p (bb)) { - int prob = XINT (prob_note, 0); + profile_probability prob = profile_probability::from_reg_br_prob_note + (XINT (prob_note, 0)); - BRANCH_EDGE (bb)->probability - = profile_probability::from_reg_br_prob_base (prob); - FALLTHRU_EDGE (bb)->probability - = BRANCH_EDGE (bb)->probability.invert (); + BRANCH_EDGE (bb)->probability = prob; + FALLTHRU_EDGE (bb)->probability = prob.invert (); } else single_succ_edge (bb)->probability = profile_probability::always (); diff --git a/gcc/predict.h b/gcc/predict.h index 1441fa4..a6cf1ee 100644 --- a/gcc/predict.h +++ b/gcc/predict.h @@ -47,6 +47,9 @@ enum prediction TAKEN }; +/* In emit-rtl.c. */ +extern profile_probability split_branch_probability; + extern gcov_type get_hot_bb_threshold (void); extern void set_hot_bb_threshold (gcov_type); extern bool maybe_hot_count_p (struct function *, profile_count); @@ -96,4 +99,6 @@ extern void rebuild_frequencies (void); extern void report_predictor_hitrates (void); extern void force_edge_cold (edge, bool); +extern void add_reg_br_prob_note (rtx_insn *, profile_probability); + #endif /* GCC_PREDICT_H */ diff --git a/gcc/profile-count.h b/gcc/profile-count.h index aef9da3..8fd22b8 100644 --- a/gcc/profile-count.h +++ b/gcc/profile-count.h @@ -190,6 +190,23 @@ public: return RDIV (m_val * REG_BR_PROB_BASE, max_probability); } + /* Conversion to and from RTL representation of profile probabilities. */ + static profile_probability from_reg_br_prob_note (int v) + { + profile_probability ret; + ret.m_val = ((unsigned int)v) / 4; + ret.m_quality = (enum profile_quality)(v & 3); + return ret; + } + int to_reg_br_prob_note () const + { + gcc_checking_assert (initialized_p ()); + int ret = m_val * 4 + m_quality; + gcc_checking_assert (profile_probability::from_reg_br_prob_note (ret) + == *this); + return ret; + } + /* Return VAL1/VAL2. */ static profile_probability probability_in_gcov_type (gcov_type val1, gcov_type val2) diff --git a/gcc/reg-notes.def b/gcc/reg-notes.def index 8734d26..943eff4 100644 --- a/gcc/reg-notes.def +++ b/gcc/reg-notes.def @@ -99,20 +99,21 @@ REG_NOTE (DEP_OUTPUT) REG_NOTE (DEP_ANTI) REG_NOTE (DEP_CONTROL) -/* REG_BR_PROB is attached to JUMP_INSNs and CALL_INSNs. It has an +/* REG_BR_PROB is attached to JUMP_INSNs. It has an integer value (in an INT_LIST). For jumps, it is the probability - that this is a taken branch. For calls, it is the probability that - this call won't return. */ + that this is a taken branch. The integer represents a value of + profile_probability type. Use to_reg_br_prob_note and from_reg_br_prob_note + to extract the actual value. */ REG_NOTE (BR_PROB) /* Attached to a call insn; indicates that the call is malloc-like and that the pointer returned cannot alias anything else. */ REG_NOTE (NOALIAS) -/* REG_BR_PRED is attached to JUMP_INSNs and CALL_INSNSs. It contains +/* REG_BR_PRED is attached to JUMP_INSNs. It contains CONCAT of two integer value. First specifies the branch predictor that added the note, second specifies the predicted hitrate of - branch in the same format as REG_BR_PROB note uses. */ + branch in a fixed point arithmetic based on REG_BR_PROB_BASE. */ REG_NOTE (BR_PRED) /* Attached to insns that are RTX_FRAME_RELATED_P, but are too complex diff --git a/gcc/reorg.c b/gcc/reorg.c index dbd74ab..5914af6 100644 --- a/gcc/reorg.c +++ b/gcc/reorg.c @@ -840,7 +840,8 @@ mostly_true_jump (rtx jump_insn) rtx note = find_reg_note (jump_insn, REG_BR_PROB, 0); if (note) { - int prob = XINT (note, 0); + int prob = profile_probability::from_reg_br_prob_note (XINT (note, 0)) + .to_reg_br_prob_base (); if (prob >= REG_BR_PROB_BASE * 9 / 10) return 2; diff --git a/gcc/rtl.h b/gcc/rtl.h index 59da995..e63dcf0 100644 --- a/gcc/rtl.h +++ b/gcc/rtl.h @@ -2938,7 +2938,6 @@ extern rtx *find_constant_term_loc (rtx *); /* In emit-rtl.c */ extern rtx_insn *try_split (rtx, rtx_insn *, int); -extern int split_branch_probability; /* In insn-recog.c (generated by genrecog). */ extern rtx_insn *split_insns (rtx, rtx_insn *); diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 6472e11..3ede9d8 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,7 @@ +2017-07-16 Jan Hubicka + + * gcc.dg/predict-8.c: Update. + 2017-07-16 Volker Reichelt * g++.dg/cpp1z/direct-enum-init1.C: Revert special enum handling. diff --git a/gcc/testsuite/gcc.dg/predict-8.c b/gcc/testsuite/gcc.dg/predict-8.c index edead8d..e13cc00 100644 --- a/gcc/testsuite/gcc.dg/predict-8.c +++ b/gcc/testsuite/gcc.dg/predict-8.c @@ -8,4 +8,4 @@ int foo(float a, float b) { return 2; } -/* { dg-final { scan-rtl-dump-times "REG_BR_PROB 100" 1 "expand"} } */ +/* { dg-final { scan-rtl-dump-times "REG_BR_PROB 400 " 1 "expand"} } */ -- cgit v1.1