11 files changed, 80 insertions, 35 deletions
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index d02cf98..3d22f859 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -1207,6 +1207,22 @@ LLVM_C_ABI LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
                                         LLVMTypeRef FunctionTy);
 
 /**
+ * Obtain or insert a function into a module.
+ *
+ * If a function with the specified name already exists in the module, it
+ * is returned. Otherwise, a new function is created in the module with the
+ * specified name and type and is returned.
+ *
+ * The returned value corresponds to a llvm::Function instance.
+ *
+ * @see llvm::Module::getOrInsertFunction()
+ */
+LLVM_C_ABI LLVMValueRef LLVMGetOrInsertFunction(LLVMModuleRef M,
+                                                const char *Name,
+                                                size_t NameLen,
+                                                LLVMTypeRef FunctionTy);
+
+/**
  * Obtain a Function value from a Module by its name.
  *
  * The returned value corresponds to a llvm::Function value.
diff --git a/llvm/include/llvm/ADT/EquivalenceClasses.h b/llvm/include/llvm/ADT/EquivalenceClasses.h
index 90d8948..5e8c2fa 100644
--- a/llvm/include/llvm/ADT/EquivalenceClasses.h
+++ b/llvm/include/llvm/ADT/EquivalenceClasses.h
@@ -128,7 +128,7 @@ private:
   /// ECValues, it just keeps the key as part of the value.
   DenseMap<ElemTy, ECValue *> TheMapping;
 
-  /// List of all members, used to provide a determinstic iteration order.
+  /// List of all members, used to provide a deterministic iteration order.
   SmallVector<const ECValue *> Members;
 
   mutable BumpPtrAllocator ECValueAllocator;
diff --git a/llvm/include/llvm/Analysis/HeatUtils.h b/llvm/include/llvm/Analysis/HeatUtils.h
index 179862c..2391086 100644
--- a/llvm/include/llvm/Analysis/HeatUtils.h
+++ b/llvm/include/llvm/Analysis/HeatUtils.h
@@ -23,17 +23,17 @@ class BlockFrequencyInfo;
 class Function;
 
 // Returns number of calls of calledFunction by callerFunction.
-LLVM_ABI uint64_t getNumOfCalls(Function &callerFunction,
-                                Function &calledFunction);
+LLVM_ABI uint64_t getNumOfCalls(const Function &CallerFunction,
+                                const Function &CalledFunction);
 
 // Returns the maximum frequency of a BB in a function.
 LLVM_ABI uint64_t getMaxFreq(const Function &F, const BlockFrequencyInfo *BFI);
 
 // Calculates heat color based on current and maximum frequencies.
-LLVM_ABI std::string getHeatColor(uint64_t freq, uint64_t maxFreq);
+LLVM_ABI std::string getHeatColor(uint64_t Freq, uint64_t MaxFreq);
 
 // Calculates heat color based on percent of "hotness".
-LLVM_ABI std::string getHeatColor(double percent);
+LLVM_ABI std::string getHeatColor(double Percent);
 
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 1c7d346..464f475 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -800,6 +800,7 @@ enum AttributeKindCodes {
   ATTR_KIND_SANITIZE_TYPE = 101,
   ATTR_KIND_CAPTURES = 102,
   ATTR_KIND_DEAD_ON_RETURN = 103,
+  ATTR_KIND_SANITIZE_ALLOC_TOKEN = 104,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/LiveRangeEdit.h b/llvm/include/llvm/CodeGen/LiveRangeEdit.h
index 6473138a..d0ed3ff 100644
--- a/llvm/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/llvm/include/llvm/CodeGen/LiveRangeEdit.h
@@ -75,24 +75,14 @@ private:
   /// FirstNew - Index of the first register added to NewRegs.
   const unsigned FirstNew;
 
-  /// ScannedRemattable - true when remattable values have been identified.
-  bool ScannedRemattable = false;
-
   /// DeadRemats - The saved instructions which have already been dead after
   /// rematerialization but not deleted yet -- to be done in postOptimization.
   SmallPtrSet<MachineInstr *, 32> *DeadRemats;
 
-  /// Remattable - Values defined by remattable instructions as identified by
-  /// tii.isTriviallyReMaterializable().
-  SmallPtrSet<const VNInfo *, 4> Remattable;
-
   /// Rematted - Values that were actually rematted, and so need to have their
   /// live range trimmed or entirely removed.
   SmallPtrSet<const VNInfo *, 4> Rematted;
 
-  /// scanRemattable - Identify the Parent values that may rematerialize.
-  void scanRemattable();
-
   /// foldAsLoad - If LI has a single use and a single def that can be folded as
   /// a load, eliminate the register by folding the def into the use.
   bool foldAsLoad(LiveInterval *LI, SmallVectorImpl<MachineInstr *> &Dead);
@@ -175,11 +165,6 @@ public:
 
   Register create() { return createFrom(getReg()); }
 
-  /// anyRematerializable - Return true if any parent values may be
-  /// rematerializable.  This function must be called before
-  /// canRematerializeAt is called..
-  bool anyRematerializable();
-
   /// Remat - Information needed to rematerialize at a specific location.
   struct Remat {
     const VNInfo *const ParentVNI;  // parent_'s value at the remat location.
@@ -189,9 +174,9 @@ public:
     explicit Remat(const VNInfo *ParentVNI) : ParentVNI(ParentVNI) {}
   };
 
-  /// canRematerializeAt - Determine if ParentVNI can be rematerialized at
+  /// canRematerializeAt - Determine if RM.Orig can be rematerialized at
   /// UseIdx. It is assumed that parent_.getVNINfoAt(UseIdx) == ParentVNI.
-  bool canRematerializeAt(Remat &RM, VNInfo *OrigVNI, SlotIndex UseIdx);
+  bool canRematerializeAt(Remat &RM, SlotIndex UseIdx);
 
   /// rematerializeAt - Rematerialize RM.ParentVNI into DestReg by inserting an
   /// instruction into MBB before MI. The new instruction is mapped, but
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index ef816fb..8e7d9dc 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -342,6 +342,9 @@ def SanitizeRealtime : EnumAttr<"sanitize_realtime", IntersectPreserve, [FnAttr]
 /// during a real-time sanitized function (see `sanitize_realtime`).
 def SanitizeRealtimeBlocking : EnumAttr<"sanitize_realtime_blocking", IntersectPreserve, [FnAttr]>;
 
+/// Allocation token instrumentation is on.
+def SanitizeAllocToken : EnumAttr<"sanitize_alloc_token", IntersectPreserve, [FnAttr]>;
+
 /// Speculative Load Hardening is enabled.
 ///
 /// Note that this uses the default compatibility (always compatible during
diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def
index 0603abc..74746cc 100644
--- a/llvm/include/llvm/IR/FixedMetadataKinds.def
+++ b/llvm/include/llvm/IR/FixedMetadataKinds.def
@@ -56,3 +56,4 @@ LLVM_FIXED_MD_KIND(MD_noalias_addrspace, "noalias.addrspace", 41)
 LLVM_FIXED_MD_KIND(MD_callee_type, "callee_type", 42)
 LLVM_FIXED_MD_KIND(MD_nofree, "nofree", 43)
 LLVM_FIXED_MD_KIND(MD_captures, "captures", 44)
+LLVM_FIXED_MD_KIND(MD_alloc_token, "alloc_token", 45)
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 14685ab..9f56779 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -601,11 +601,9 @@ public:
       Instruction::CastOps firstOpcode,  ///< Opcode of first cast
       Instruction::CastOps secondOpcode, ///< Opcode of second cast
       Type *SrcTy,                       ///< SrcTy of 1st cast
-      Type *MidTy,       ///< DstTy of 1st cast & SrcTy of 2nd cast
-      Type *DstTy,       ///< DstTy of 2nd cast
-      Type *SrcIntPtrTy, ///< Integer type corresponding to Ptr SrcTy, or null
-      Type *MidIntPtrTy, ///< Integer type corresponding to Ptr MidTy, or null
-      Type *DstIntPtrTy  ///< Integer type corresponding to Ptr DstTy, or null
+      Type *MidTy,         ///< DstTy of 1st cast & SrcTy of 2nd cast
+      Type *DstTy,         ///< DstTy of 2nd cast
+      const DataLayout *DL ///< Optional data layout
   );
 
   /// Return the opcode of this CastInst
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
index 773de89..ffee81d 100644
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -80,9 +80,6 @@ public:
   /// make as little assumptions as possible about the program state when
   /// RunSafely has returned false.
   LLVM_ABI bool RunSafely(function_ref<void()> Fn);
-  bool RunSafely(void (*Fn)(void*), void *UserData) {
-    return RunSafely([&]() { Fn(UserData); });
-  }
 
   /// Execute the provide callback function (with the given arguments) in
   /// a protected context which is run in another thread (optionally with a
@@ -94,10 +91,6 @@ public:
   /// propagated to the new thread as well.
   LLVM_ABI bool RunSafelyOnThread(function_ref<void()>,
                                   unsigned RequestedStackSize = 0);
-  bool RunSafelyOnThread(void (*Fn)(void*), void *UserData,
-                         unsigned RequestedStackSize = 0) {
-    return RunSafelyOnThread([&]() { Fn(UserData); }, RequestedStackSize);
-  }
 
   LLVM_ABI bool RunSafelyOnNewStack(function_ref<void()>,
                                     unsigned RequestedStackSize = 0);
diff --git a/llvm/include/llvm/Support/SpecialCaseList.h b/llvm/include/llvm/Support/SpecialCaseList.h
index 22a62ea..c2c9271 100644
--- a/llvm/include/llvm/Support/SpecialCaseList.h
+++ b/llvm/include/llvm/Support/SpecialCaseList.h
@@ -147,7 +147,9 @@ protected:
     Section(StringRef Str, unsigned FileIdx)
         : SectionStr(Str), FileIdx(FileIdx) {};
 
-    std::unique_ptr<Matcher> SectionMatcher = std::make_unique<Matcher>();
+    Section(Section &&) = default;
+
+    Matcher SectionMatcher;
     SectionEntries Entries;
     std::string SectionStr;
     unsigned FileIdx;
diff --git a/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h b/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h
new file mode 100644
index 0000000..b1391cb0
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Instrumentation/AllocToken.h
@@ -0,0 +1,46 @@
+//===- AllocToken.h - Allocation token instrumentation --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the AllocTokenPass, an instrumentation pass that
+// replaces allocation calls with ones including an allocation token.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_ALLOCTOKEN_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_ALLOCTOKEN_H
+
+#include "llvm/IR/Analysis.h"
+#include "llvm/IR/PassManager.h"
+#include <optional>
+
+namespace llvm {
+
+class Module;
+
+struct AllocTokenOptions {
+  std::optional<uint64_t> MaxTokens;
+  bool FastABI = false;
+  bool Extended = false;
+  AllocTokenOptions() = default;
+};
+
+/// A module pass that rewrites heap allocations to use token-enabled
+/// allocation functions based on various source-level properties.
+class AllocTokenPass : public PassInfoMixin<AllocTokenPass> {
+public:
+  LLVM_ABI explicit AllocTokenPass(AllocTokenOptions Opts = {});
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  static bool isRequired() { return true; }
+
+private:
+  const AllocTokenOptions Options;
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_INSTRUMENTATION_ALLOCTOKEN_H