diff options
author | Kyungwoo Lee <kyulee@meta.com> | 2024-09-29 18:28:15 -0700 |
---|---|---|
committer | Kyungwoo Lee <kyulee@meta.com> | 2024-10-07 17:45:31 -0700 |
commit | d75c88f55a8e271ab8ad6c2a8ef3383dafdf8516 (patch) | |
tree | a47c5fb3f220ebf8251bddbbd2a73586ff57c927 | |
parent | 39ac121309f5c270c8fdc97c45a845542dd78681 (diff) | |
download | llvm-users/kyulee-com/thinback2.zip llvm-users/kyulee-com/thinback2.tar.gz llvm-users/kyulee-com/thinback2.tar.bz2 |
[NFC] Refactor ThinBackendusers/kyulee-com/thinback2
- Change it to a type from a function.
- Store the parallelism in the type for the future use.
-rw-r--r-- | llvm/include/llvm/LTO/LTO.h | 76 | ||||
-rw-r--r-- | llvm/lib/LTO/LTO.cpp | 106 |
2 files changed, 104 insertions, 78 deletions
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 5c47c4d..a6b9ede 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -26,6 +26,7 @@ #include "llvm/Support/Caching.h" #include "llvm/Support/Error.h" #include "llvm/Support/StringSaver.h" +#include "llvm/Support/ThreadPool.h" #include "llvm/Support/thread.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" @@ -105,7 +106,6 @@ void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index); class LTO; struct SymbolResolution; -class ThinBackendProc; /// An input file. This is a symbol table wrapper that only exposes the /// information that an LTO client should need in order to do symbol resolution. @@ -194,13 +194,80 @@ private: } }; +using IndexWriteCallback = std::function<void(const std::string &)>; + +/// This class defines the interface to the ThinLTO backend. +class ThinBackendProc { +protected: + const Config &Conf; + ModuleSummaryIndex &CombinedIndex; + const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries; + IndexWriteCallback OnWrite; + bool ShouldEmitImportsFiles; + DefaultThreadPool BackendThreadPool; + std::optional<Error> Err; + std::mutex ErrMu; + +public: + ThinBackendProc( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, + lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles, + ThreadPoolStrategy ThinLTOParallelism) + : Conf(Conf), CombinedIndex(CombinedIndex), + ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries), + OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles), + BackendThreadPool(ThinLTOParallelism) {} + + virtual ~ThinBackendProc() = default; + virtual Error start( + unsigned Task, BitcodeModule BM, + const FunctionImporter::ImportMapTy &ImportList, + const FunctionImporter::ExportSetTy &ExportList, + const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, + MapVector<StringRef, BitcodeModule> &ModuleMap) = 0; + Error wait() { + BackendThreadPool.wait(); + if (Err) + return std::move(*Err); + return Error::success(); + } + unsigned getThreadCount() { return BackendThreadPool.getMaxConcurrency(); } + virtual bool isSensitiveToInputOrder() { return false; } + + // Write sharded indices and (optionally) imports to disk + Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, + llvm::StringRef ModulePath, + const std::string &NewModulePath) const; +}; + /// A ThinBackend defines what happens after the thin-link phase during ThinLTO. /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. -using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>( +using ThinBackendFunction = std::function<std::unique_ptr<ThinBackendProc>( const Config &C, ModuleSummaryIndex &CombinedIndex, - DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, + const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache)>; +struct ThinBackend { + ThinBackend(ThinBackendFunction Func, ThreadPoolStrategy Parallelism) + : Func(std::move(Func)), Parallelism(std::move(Parallelism)) {} + ThinBackend() = default; + + std::unique_ptr<ThinBackendProc> operator()( + const Config &Conf, ModuleSummaryIndex &CombinedIndex, + const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, + AddStreamFn AddStream, FileCache Cache) { + assert(isValid() && "Invalid backend function"); + return Func(Conf, CombinedIndex, ModuleToDefinedGVSummaries, + std::move(AddStream), std::move(Cache)); + } + ThreadPoolStrategy getParallelism() const { return Parallelism; } + bool isValid() const { return static_cast<bool>(Func); } + +private: + ThinBackendFunction Func = nullptr; + ThreadPoolStrategy Parallelism; +}; /// This ThinBackend runs the individual backend jobs in-process. /// The default value means to use one job per hardware core (not hyper-thread). @@ -210,7 +277,6 @@ using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>( /// to the same path as the input module, with suffix ".thinlto.bc" /// ShouldEmitImportsFiles is true it also writes a list of imported files to a /// similar path with ".imports" appended instead. -using IndexWriteCallback = std::function<void(const std::string &)>; ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite = nullptr, bool ShouldEmitIndexFiles = false, @@ -276,7 +342,7 @@ public: /// this constructor. /// FIXME: We do currently require the DiagHandler field to be set in Conf. /// Until that is fixed, a Config argument is required. - LTO(Config Conf, ThinBackend Backend = nullptr, + LTO(Config Conf, ThinBackend Backend = {}, unsigned ParallelCodeGenParallelismLevel = 1, LTOKind LTOMode = LTOK_Default); ~LTO(); diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index ccf1139..e1714b2 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -578,10 +578,10 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel, CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat; } -LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) - : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) { - if (!Backend) - this->Backend = +LTO::ThinLTOState::ThinLTOState(ThinBackend BackendParam) + : Backend(std::move(BackendParam)), CombinedIndex(/*HaveGVs*/ false) { + if (!Backend.isValid()) + Backend = createInProcessThinBackend(llvm::heavyweight_hardware_concurrency()); } @@ -1368,75 +1368,33 @@ SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) { return LibcallSymbols; } -/// This class defines the interface to the ThinLTO backend. -class lto::ThinBackendProc { -protected: - const Config &Conf; - ModuleSummaryIndex &CombinedIndex; - const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries; - lto::IndexWriteCallback OnWrite; - bool ShouldEmitImportsFiles; - DefaultThreadPool BackendThreadPool; - std::optional<Error> Err; - std::mutex ErrMu; +Error ThinBackendProc::emitFiles( + const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, + const std::string &NewModulePath) const { + ModuleToSummariesForIndexTy ModuleToSummariesForIndex; + GVSummaryPtrSet DeclarationSummaries; -public: - ThinBackendProc( - const Config &Conf, ModuleSummaryIndex &CombinedIndex, - const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, - lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles, - ThreadPoolStrategy ThinLTOParallelism) - : Conf(Conf), CombinedIndex(CombinedIndex), - ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries), - OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles), - BackendThreadPool(ThinLTOParallelism) {} - - virtual ~ThinBackendProc() = default; - virtual Error start( - unsigned Task, BitcodeModule BM, - const FunctionImporter::ImportMapTy &ImportList, - const FunctionImporter::ExportSetTy &ExportList, - const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR, - MapVector<StringRef, BitcodeModule> &ModuleMap) = 0; - Error wait() { - BackendThreadPool.wait(); - if (Err) - return std::move(*Err); - return Error::success(); - } - unsigned getThreadCount() { return BackendThreadPool.getMaxConcurrency(); } - virtual bool isSensitiveToInputOrder() { return false; } - - // Write sharded indices and (optionally) imports to disk - Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, - llvm::StringRef ModulePath, - const std::string &NewModulePath) const { - ModuleToSummariesForIndexTy ModuleToSummariesForIndex; - GVSummaryPtrSet DeclarationSummaries; - - std::error_code EC; - gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, - ImportList, ModuleToSummariesForIndex, - DeclarationSummaries); - - raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, - sys::fs::OpenFlags::OF_None); - if (EC) - return createFileError("cannot open " + NewModulePath + ".thinlto.bc", - EC); - - writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex, - &DeclarationSummaries); - - if (ShouldEmitImportsFiles) { - Error ImportFilesError = EmitImportsFiles( - ModulePath, NewModulePath + ".imports", ModuleToSummariesForIndex); - if (ImportFilesError) - return ImportFilesError; - } - return Error::success(); + std::error_code EC; + gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, + ImportList, ModuleToSummariesForIndex, + DeclarationSummaries); + + raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC, + sys::fs::OpenFlags::OF_None); + if (EC) + return createFileError("cannot open " + NewModulePath + ".thinlto.bc", EC); + + writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex, + &DeclarationSummaries); + + if (ShouldEmitImportsFiles) { + Error ImportFilesError = EmitImportsFiles( + ModulePath, NewModulePath + ".imports", ModuleToSummariesForIndex); + if (ImportFilesError) + return ImportFilesError; } -}; + return Error::success(); +} namespace { class InProcessThinBackend : public ThinBackendProc { @@ -1561,7 +1519,7 @@ ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism, lto::IndexWriteCallback OnWrite, bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles) { - return + auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache) { @@ -1570,6 +1528,7 @@ ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism, AddStream, Cache, OnWrite, ShouldEmitIndexFiles, ShouldEmitImportsFiles); }; + return ThinBackend(Func, Parallelism); } StringLiteral lto::getThinLTODefaultCPU(const Triple &TheTriple) { @@ -1681,7 +1640,7 @@ ThinBackend lto::createWriteIndexesThinBackend( std::string NewPrefix, std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) { - return + auto Func = [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache) { @@ -1690,6 +1649,7 @@ ThinBackend lto::createWriteIndexesThinBackend( OldPrefix, NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite); }; + return ThinBackend(Func, Parallelism); } Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, |