1 files changed, 112 insertions, 73 deletions

diff --git a/gcc/profile-count.h b/gcc/profile-count.h
index 0e79fd2..c893aec 100644
--- a/gcc/profile-count.h
+++ b/gcc/profile-count.h
@@ -37,19 +37,21 @@ enum profile_quality {
    */
   GUESSED_LOCAL,
 
+  /* Same as GUESSED_GLOBAL0 but global count is afdo 0.  */
+  GUESSED_GLOBAL0_AFDO,
+
+  /* Same as GUESSED_GLOBAL0 but global count is adjusted 0.  */
+  GUESSED_GLOBAL0_ADJUSTED,
+
   /* Profile was read by feedback and was 0, we used local heuristics to guess
      better.  This is the case of functions not run in profile feedback.
      Never used by probabilities.  */
   GUESSED_GLOBAL0,
 
-  /* Same as GUESSED_GLOBAL0 but global count is adjusted 0.  */
-  GUESSED_GLOBAL0_ADJUSTED,
-
   /* Profile is based on static branch prediction heuristics.  It may or may
      not reflect the reality but it can be compared interprocedurally
      (for example, we inlined function w/o profile feedback into function
-      with feedback and propagated from that).
-     Never used by probabilities.  */
+      with feedback and propagated from that).  */
   GUESSED,
 
   /* Profile was determined by autofdo.  */
@@ -151,26 +153,39 @@ class GTY((user)) profile_probability
   static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
   static const uint32_t uninitialized_probability
 		 = ((uint32_t) 1 << (n_bits - 1)) - 1;
+  /* For probibilityes quality is either UNINITIALIZED (0)
+     or greated then GUESSED.  To save bits we store it in
+     adjusted form which skips the invalid values.  */
+  static const int min_quality = GUESSED;
 
   uint32_t m_val : 29;
-  enum profile_quality m_quality : 3;
+  unsigned m_adjusted_quality : 3;
 
   friend struct profile_count;
+
+  /* Set the quality of the probability.  */
+  void
+  set_quality (profile_quality quality)
+  {
+    gcc_checking_assert (quality == UNINITIALIZED_PROFILE
+			 || (quality >= min_quality && quality <= PRECISE));
+    m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
+  }
+
 public:
-  profile_probability (): m_val (uninitialized_probability),
-    m_quality (GUESSED)
-  {}
+  profile_probability (): m_val (uninitialized_probability)
+  { set_quality (GUESSED); }
 
   profile_probability (uint32_t val, profile_quality quality):
-    m_val (val), m_quality (quality)
-  {}
+    m_val (val)
+  { set_quality (quality); }
 
   /* Named probabilities.  */
   static profile_probability never ()
     {
       profile_probability ret;
       ret.m_val = 0;
-      ret.m_quality = PRECISE;
+      ret.set_quality (PRECISE);
       return ret;
     }
 
@@ -178,7 +193,7 @@ public:
     {
       profile_probability ret;
       ret.m_val = 0;
-      ret.m_quality = GUESSED;
+      ret.set_quality (GUESSED);
       return ret;
     }
 
@@ -222,7 +237,7 @@ public:
     {
       profile_probability ret;
       ret.m_val = max_probability;
-      ret.m_quality = GUESSED;
+      ret.set_quality (GUESSED);
       return ret;
     }
 
@@ -230,7 +245,7 @@ public:
     {
       profile_probability ret;
       ret.m_val = max_probability;
-      ret.m_quality = PRECISE;
+      ret.set_quality (PRECISE);
       return ret;
     }
 
@@ -240,7 +255,7 @@ public:
     {
       profile_probability c;
       c.m_val = uninitialized_probability;
-      c.m_quality = GUESSED;
+      c.set_quality (GUESSED);
       return c;
     }
 
@@ -253,7 +268,7 @@ public:
   /* Return true if value can be trusted.  */
   bool reliable_p () const
     {
-      return m_quality >= ADJUSTED;
+      return quality () >= ADJUSTED;
     }
 
   /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
@@ -263,7 +278,7 @@ public:
       profile_probability ret;
       gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
       ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
-      ret.m_quality = GUESSED;
+      ret.set_quality (GUESSED);
       return ret;
     }
 
@@ -273,7 +288,7 @@ public:
       profile_probability ret = *this;
       if (!initialized_p ())
 	return *this;
-      ret.m_quality = ADJUSTED;
+      ret.set_quality (ADJUSTED);
       return ret;
     }
 
@@ -288,14 +303,14 @@ public:
     {
       profile_probability ret;
       ret.m_val = ((unsigned int)v) / 8;
-      ret.m_quality = (enum profile_quality)(v & 7);
+      ret.m_adjusted_quality = ((unsigned int)v) & 7;
       return ret;
     }
 
   int to_reg_br_prob_note () const
     {
       gcc_checking_assert (initialized_p ());
-      int ret = m_val * 8 + m_quality;
+      int ret = m_val * 8 + m_adjusted_quality;
       gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
       return ret;
     }
@@ -315,14 +330,14 @@ public:
 	  gcc_checking_assert (tmp <= max_probability);
 	  ret.m_val = tmp;
 	}
-      ret.m_quality = PRECISE;
+      ret.set_quality (PRECISE);
       return ret;
     }
 
   /* Basic operations.  */
   bool operator== (const profile_probability &other) const
     {
-      return m_val == other.m_val && m_quality == other.m_quality;
+      return m_val == other.m_val && quality () == other.quality ();
     }
 
   profile_probability operator+ (const profile_probability &other) const
@@ -336,7 +351,7 @@ public:
 
       profile_probability ret;
       ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
-      ret.m_quality = MIN (m_quality, other.m_quality);
+      ret.set_quality (MIN (quality (), other.quality ()));
       return ret;
     }
 
@@ -354,7 +369,7 @@ public:
       else
 	{
 	  m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
-	  m_quality = MIN (m_quality, other.m_quality);
+	  set_quality (MIN (quality (), other.quality ()));
 	}
       return *this;
     }
@@ -368,7 +383,7 @@ public:
 	return uninitialized ();
       profile_probability ret;
       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
-      ret.m_quality = MIN (m_quality, other.m_quality);
+      ret.set_quality (MIN (quality (), other.quality ()));
       return ret;
     }
 
@@ -382,7 +397,7 @@ public:
       else
 	{
 	  m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
-	  m_quality = MIN (m_quality, other.m_quality);
+	  set_quality (MIN (quality (), other.quality ()));
 	}
       return *this;
     }
@@ -396,7 +411,7 @@ public:
 	return uninitialized ();
       profile_probability ret;
       ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
-      ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+      ret.set_quality (MIN (quality (), other.quality ()));
       return ret;
     }
 
@@ -410,7 +425,7 @@ public:
       else
 	{
 	  m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
-	  m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+	  set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
 	}
       return *this;
     }
@@ -426,8 +441,8 @@ public:
       if (m_val >= other.m_val)
 	{
 	  ret.m_val = max_probability;
-          ret.m_quality = MIN (MIN (m_quality, other.m_quality),
-			       GUESSED);
+	  ret.set_quality (MIN (MIN (quality (), other.quality ()),
+				GUESSED));
 	  return ret;
 	}
       else if (!m_val)
@@ -439,7 +454,7 @@ public:
 				 other.m_val),
 			   max_probability);
 	}
-      ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+      ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
       return ret;
     }
 
@@ -451,13 +466,13 @@ public:
 	return *this = uninitialized ();
       else
 	{
-          /* If we get probability above 1, mark it as unreliable
+	  /* If we get probability above 1, mark it as unreliable
 	     and return 1. */
 	  if (m_val > other.m_val)
 	    {
 	      m_val = max_probability;
-              m_quality = MIN (MIN (m_quality, other.m_quality),
-			       GUESSED);
+	      set_quality (MIN (MIN (quality (), other.quality ()),
+				GUESSED));
 	      return *this;
 	    }
 	  else if (!m_val)
@@ -469,7 +484,7 @@ public:
 				 other.m_val),
 			   max_probability);
 	    }
-	  m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
+	  set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
 	}
       return *this;
     }
@@ -519,7 +534,7 @@ public:
   profile_probability guessed () const
     {
       profile_probability ret = *this;
-      ret.m_quality = GUESSED;
+      ret.set_quality (GUESSED);
       return ret;
     }
 
@@ -527,7 +542,7 @@ public:
   profile_probability afdo () const
     {
       profile_probability ret = *this;
-      ret.m_quality = AFDO;
+      ret.set_quality (AFDO);
       return ret;
     }
 
@@ -542,7 +557,7 @@ public:
       uint64_t tmp;
       safe_scale_64bit (m_val, num, den, &tmp);
       ret.m_val = MIN (tmp, max_probability);
-      ret.m_quality = MIN (m_quality, ADJUSTED);
+      ret.set_quality (MIN (quality (), ADJUSTED));
       return ret;
     }
 
@@ -564,8 +579,8 @@ public:
       uint64_t val;
       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
       ret.m_val = MIN (val, max_probability);
-      ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
-				     num.m_quality), den.m_quality);
+      ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
+			    num.quality ()), den.quality ()));
       return ret;
     }
 
@@ -582,12 +597,12 @@ public:
      There are two exceptions - edges leading to noreturn edges and edges
      predicted by number of iterations heuristics are predicted well.  This macro
      should be able to distinguish those, but at the moment it simply check for
-     noreturn heuristic that is only one giving probability over 99% or bellow
+     noreturn heuristic that is only one giving probability over 99% or below
      1%.  In future we might want to propagate reliability information across the
      CFG if we find this information useful on multiple places.   */
   bool probably_reliable_p () const
     {
-      if (m_quality >= ADJUSTED)
+      if (quality () >= ADJUSTED)
 	return true;
       if (!initialized_p ())
 	return false;
@@ -598,10 +613,10 @@ public:
   /* Return false if profile_probability is bogus.  */
   bool verify () const
     {
-      gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
+      gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
       if (m_val == uninitialized_probability)
-	return m_quality == GUESSED;
-      else if (m_quality < GUESSED)
+	return quality () == GUESSED;
+      else if (quality () < GUESSED)
 	return false;
       return m_val <= max_probability;
     }
@@ -656,11 +671,16 @@ public:
   /* Compute sware root.  */
   profile_probability sqrt () const;
 
-  /* Get the value of the count.  */
+  /* Get the value of the probability.  */
   uint32_t value () const { return m_val; }
 
-  /* Get the quality of the count.  */
-  enum profile_quality quality () const { return m_quality; }
+  /* Get the quality of the probability.  */
+  enum profile_quality quality () const
+  {
+    return (profile_quality) (m_adjusted_quality
+			      ? m_adjusted_quality + min_quality - 1
+			      : UNINITIALIZED_PROFILE);
+  }
 
   /* Output THIS to F.  */
   void dump (FILE *f) const;
@@ -749,11 +769,11 @@ public:
 struct GTY(()) profile_count
 {
 public:
-  /* Use 62bit to hold basic block counters.  Should be at least
+  /* Use 60bit to hold basic block counters.  Should be at least
      64bit.  Although a counter cannot be negative, we use a signed
      type to hold various extra stages.  */
 
-  static const int n_bits = 61;
+  static const int n_bits = 60;
   static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
 private:
   static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
@@ -769,7 +789,7 @@ private:
 #endif
   uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
 #undef UINT64_BIT_FIELD_ALIGN
-  enum profile_quality m_quality : 3;
+  enum profile_quality m_quality : 4;
 public:
 
   /* Return true if both values can meaningfully appear in single function
@@ -808,6 +828,14 @@ public:
       return c;
     }
 
+  static profile_count afdo_zero ()
+    {
+      profile_count c;
+      c.m_val = 0;
+      c.m_quality = AFDO;
+      return c;
+    }
+
   static profile_count guessed_zero ()
     {
       profile_count c;
@@ -853,7 +881,7 @@ public:
   /* Return true if value can be operated inter-procedurally.  */
   bool ipa_p () const
     {
-      return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
+      return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
     }
 
   /* Return true if quality of profile is precise.  */
@@ -929,7 +957,7 @@ public:
 	return *this = uninitialized ();
       else
 	{
-          gcc_checking_assert (compatible_p (other));
+	  gcc_checking_assert (compatible_p (other));
 	  uint64_t ret_val = m_val + other.m_val;
 	  m_val = MIN (ret_val, max_count);
 	  m_quality = MIN (m_quality, other.m_quality);
@@ -958,7 +986,7 @@ public:
 	return *this = uninitialized ();
       else
 	{
-          gcc_checking_assert (compatible_p (other));
+	  gcc_checking_assert (compatible_p (other));
 	  m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
 	  m_quality = MIN (m_quality, other.m_quality);
 	}
@@ -1084,18 +1112,7 @@ public:
     }
 
   /* Make counter forcibly nonzero.  */
-  profile_count force_nonzero () const
-    {
-      if (!initialized_p ())
-	return *this;
-      profile_count ret = *this;
-      if (ret.m_val == 0)
-	{
-	  ret.m_val = 1;
-          ret.m_quality = MIN (m_quality, ADJUSTED);
-	}
-      return ret;
-    }
+  profile_count force_nonzero () const;
 
   profile_count max (profile_count other) const
     {
@@ -1153,7 +1170,7 @@ public:
       safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
 			&tmp);
       ret.m_val = tmp;
-      ret.m_quality = MIN (m_quality, prob.m_quality);
+      ret.m_quality = MIN (m_quality, prob.quality ());
       return ret;
     }
 
@@ -1191,7 +1208,7 @@ public:
       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
       ret.m_val = MIN (val, max_count);
       ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
-			        num.m_quality), den.m_quality);
+			   num.m_quality), den.m_quality);
       /* Be sure that ret is not local if num is global.
 	 Also ensure that ret is not global0 when num is global.  */
       if (num.ipa_p ())
@@ -1220,6 +1237,17 @@ public:
       return ret;
     }
 
+  /* We know that profile is globally afdo 0 but keep local profile
+     if present.  */
+  profile_count global0afdo () const
+    {
+      profile_count ret = *this;
+      if (!initialized_p ())
+	return *this;
+      ret.m_quality = GUESSED_GLOBAL0_AFDO;
+      return ret;
+    }
+
   /* We know that profile is globally adjusted 0 but keep local profile
      if present.  */
   profile_count global0adjusted () const
@@ -1239,16 +1267,27 @@ public:
       return ret;
     }
 
+  /* Return THIS with quality GUESSED.  */
+  profile_count force_guessed () const
+    {
+      profile_count ret = *this;
+      gcc_checking_assert (initialized_p ());
+      ret.m_quality = GUESSED;
+      return ret;
+    }
+
   /* Return variant of profile count which is always safe to compare
      across functions.  */
   profile_count ipa () const
     {
-      if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
+      if (m_quality > GUESSED_GLOBAL0)
 	return *this;
       if (m_quality == GUESSED_GLOBAL0)
 	return zero ();
       if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
 	return adjusted_zero ();
+      if (m_quality == GUESSED_GLOBAL0_AFDO)
+	return afdo_zero ();
       return uninitialized ();
     }
 
@@ -1278,14 +1317,14 @@ public:
       if (overall.m_val < m_val)
 	{
 	  ret.m_val = profile_probability::max_probability;
-	  ret.m_quality = GUESSED;
+	  ret.set_quality (GUESSED);
 	  return ret;
 	}
       else
 	ret.m_val = RDIV (m_val * profile_probability::max_probability,
 			  overall.m_val);
-      ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
-				GUESSED), ADJUSTED);
+      ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
+				 GUESSED), ADJUSTED));
       return ret;
     }