DynamicAPInt: optimize size of structure (#97831)

Reuse the APInt::BitWidth to eliminate DynamicAPInt::HoldsLarge, cutting
the size of DynamicAPInt by four bytes. This is implemented by making
DynamicAPInt a friend of SlowDynamicAPInt and APInt, so it can directly
access SlowDynamicAPInt::Val and APInt::BitWidth.

We get a speedup of 4% with this patch.

4 files changed, 40 insertions, 11 deletions

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 6cfa6ec..108df7e 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -30,6 +30,7 @@ class StringRef;
 class hash_code;
 class raw_ostream;
 struct Align;
+class DynamicAPInt;
 
 template <typename T> class SmallVectorImpl;
 template <typename T> class ArrayRef;
@@ -1895,6 +1896,9 @@ private:
   friend struct DenseMapInfo<APInt, void>;
   friend class APSInt;
 
+  // Make DynamicAPInt a friend so it can access BitWidth directly.
+  friend DynamicAPInt;
+
   /// This constructor is used only internally for speed of construction of
   /// temporaries. It is unsafe since it takes ownership of the pointer, so it
   /// is not public.
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index f312f77..2f11f91 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -35,21 +35,23 @@ namespace llvm {
 /// We always_inline all operations; removing these results in a 1.5x
 /// performance slowdown.
 ///
-/// When HoldsLarge is true, a SlowMPInt is held in the union. If it is false,
-/// the int64_t is held. Using std::variant instead would lead to significantly
-/// worse performance.
+/// When isLarge returns true, a SlowMPInt is held in the union. If isSmall
+/// returns true, the int64_t is held. We don't have a separate field for
+/// indicating this, and instead "steal" memory from ValLarge when it is not in
+/// use because we know that the memory layout of APInt is such that BitWidth
+/// doesn't overlap with ValSmall (see static_assert_layout). Using std::variant
+/// instead would lead to significantly worse performance.
 class DynamicAPInt {
   union {
     int64_t ValSmall;
     detail::SlowDynamicAPInt ValLarge;
   };
-  unsigned HoldsLarge;
 
   LLVM_ATTRIBUTE_ALWAYS_INLINE void initSmall(int64_t O) {
     if (LLVM_UNLIKELY(isLarge()))
       ValLarge.detail::SlowDynamicAPInt::~SlowDynamicAPInt();
     ValSmall = O;
-    HoldsLarge = false;
+    ValLarge.Val.BitWidth = 0;
   }
   LLVM_ATTRIBUTE_ALWAYS_INLINE void
   initLarge(const detail::SlowDynamicAPInt &O) {
@@ -66,14 +68,17 @@ class DynamicAPInt {
       // and leak it.
       ValLarge = O;
     }
-    HoldsLarge = true;
   }
 
   LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(
       const detail::SlowDynamicAPInt &Val)
-      : ValLarge(Val), HoldsLarge(true) {}
-  LLVM_ATTRIBUTE_ALWAYS_INLINE bool isSmall() const { return !HoldsLarge; }
-  LLVM_ATTRIBUTE_ALWAYS_INLINE bool isLarge() const { return HoldsLarge; }
+      : ValLarge(Val) {}
+  LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr bool isSmall() const {
+    return ValLarge.Val.BitWidth == 0;
+  }
+  LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr bool isLarge() const {
+    return !isSmall();
+  }
   /// Get the stored value. For getSmall/Large,
   /// the stored value should be small/large.
   LLVM_ATTRIBUTE_ALWAYS_INLINE int64_t getSmall() const {
@@ -105,14 +110,17 @@ class DynamicAPInt {
 
 public:
   LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(int64_t Val)
-      : ValSmall(Val), HoldsLarge(false) {}
+      : ValSmall(Val) {
+    ValLarge.Val.BitWidth = 0;
+  }
   LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt() : DynamicAPInt(0) {}
   LLVM_ATTRIBUTE_ALWAYS_INLINE ~DynamicAPInt() {
     if (LLVM_UNLIKELY(isLarge()))
       ValLarge.detail::SlowDynamicAPInt::~SlowDynamicAPInt();
   }
   LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt(const DynamicAPInt &O)
-      : ValSmall(O.ValSmall), HoldsLarge(false) {
+      : ValSmall(O.ValSmall) {
+    ValLarge.Val.BitWidth = 0;
     if (LLVM_UNLIKELY(O.isLarge()))
       initLarge(O.ValLarge);
   }
@@ -203,6 +211,8 @@ public:
 
   friend hash_code hash_value(const DynamicAPInt &x); // NOLINT
 
+  void static_assert_layout(); // NOLINT
+
   raw_ostream &print(raw_ostream &OS) const;
   LLVM_DUMP_METHOD void dump() const;
 };
diff --git a/llvm/include/llvm/ADT/SlowDynamicAPInt.h b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
index 1678bda..cda5f39 100644
--- a/llvm/include/llvm/ADT/SlowDynamicAPInt.h
+++ b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
@@ -21,6 +21,10 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/Support/raw_ostream.h"
 
+namespace llvm {
+class DynamicAPInt;
+} // namespace llvm
+
 namespace llvm::detail {
 /// A simple class providing dynamic arbitrary-precision arithmetic. Internally,
 /// it stores an APInt, whose width is doubled whenever an overflow occurs at a
@@ -69,6 +73,9 @@ public:
   /// Overload to compute a hash_code for a SlowDynamicAPInt value.
   friend hash_code hash_value(const SlowDynamicAPInt &X); // NOLINT
 
+  // Make DynamicAPInt a friend so it can access Val directly.
+  friend DynamicAPInt;
+
   unsigned getBitWidth() const { return Val.getBitWidth(); }
 
   void print(raw_ostream &OS) const;
diff --git a/llvm/lib/Support/DynamicAPInt.cpp b/llvm/lib/Support/DynamicAPInt.cpp
index cae034c..bfcb97e 100644
--- a/llvm/lib/Support/DynamicAPInt.cpp
+++ b/llvm/lib/Support/DynamicAPInt.cpp
@@ -18,6 +18,14 @@ hash_code llvm::hash_value(const DynamicAPInt &X) {
   return detail::hash_value(X.getLarge());
 }
 
+void DynamicAPInt::static_assert_layout() {
+  constexpr size_t ValLargeOffset =
+      offsetof(DynamicAPInt, ValLarge.Val.BitWidth);
+  constexpr size_t ValSmallOffset = offsetof(DynamicAPInt, ValSmall);
+  constexpr size_t ValSmallSize = sizeof(ValSmall);
+  static_assert(ValLargeOffset >= ValSmallOffset + ValSmallSize);
+}
+
 raw_ostream &DynamicAPInt::print(raw_ostream &OS) const {
   if (isSmall())
     return OS << ValSmall;