From a033dbbe5c43247b60869b008e67ed86ed230eaa Mon Sep 17 00:00:00 2001 From: Fred Riss Date: Wed, 19 Oct 2022 17:29:36 -0700 Subject: [Clang] Give Clang the ability to use a shared stat cache Every Clang instance uses an internal FileSystemStatCache to avoid stating the same content multiple times. However, different instances of Clang will contend for filesystem access for their initial stats during HeaderSearch or module validation. On some workloads, the time spent in the kernel in these concurrent stat calls has been measured to be over 20% of the overall compilation time. This is extremly wassteful when most of the stat calls target mostly immutable content like a SDK. This commit introduces a new tool `clang-stat-cache` able to generate an OnDiskHashmap containing the stat data for a given filesystem hierarchy. The driver part of this has been modeled after -ivfsoverlay given the similarities with what it influences. It introduces a new -ivfsstatcache driver option to instruct Clang to use a stat cache generated by `clang-stat-cache`. These stat caches are inserted at the bottom of the VFS stack (right above the real filesystem). Differential Revision: https://reviews.llvm.org/D136651 --- clang/lib/Frontend/CompilerInvocation.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) (limited to 'clang/lib/Frontend/CompilerInvocation.cpp') diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0bb9c8c..b0ef37fa 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -83,6 +83,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/Regex.h" +#include "llvm/Support/StatCacheFileSystem.h" #include "llvm/Support/VersionTuple.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" @@ -3084,6 +3085,9 @@ static void GenerateHeaderSearchArgs(HeaderSearchOptions &Opts, GenerateArg(Args, Opt, P.Prefix, SA); } + for (const std::string &F : Opts.VFSStatCacheFiles) + GenerateArg(Args, OPT_ivfsstatcache, F, SA); + for (const std::string &F : Opts.VFSOverlayFiles) GenerateArg(Args, OPT_ivfsoverlay, F, SA); } @@ -3217,6 +3221,9 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args, Opts.AddSystemHeaderPrefix( A->getValue(), A->getOption().matches(OPT_system_header_prefix)); + for (const auto *A : Args.filtered(OPT_ivfsstatcache)) + Opts.AddVFSStatCacheFile(A->getValue()); + for (const auto *A : Args.filtered(OPT_ivfsoverlay)) Opts.AddVFSOverlayFile(A->getValue()); @@ -4747,12 +4754,31 @@ clang::createVFSFromCompilerInvocation( const CompilerInvocation &CI, DiagnosticsEngine &Diags, IntrusiveRefCntPtr BaseFS) { return createVFSFromOverlayFiles(CI.getHeaderSearchOpts().VFSOverlayFiles, + CI.getHeaderSearchOpts().VFSStatCacheFiles, Diags, std::move(BaseFS)); } IntrusiveRefCntPtr clang::createVFSFromOverlayFiles( - ArrayRef VFSOverlayFiles, DiagnosticsEngine &Diags, + ArrayRef VFSOverlayFiles, + ArrayRef VFSStatCacheFiles, DiagnosticsEngine &Diags, IntrusiveRefCntPtr BaseFS) { + for (const auto &File : VFSStatCacheFiles) { + llvm::ErrorOr> Buffer = + BaseFS->getBufferForFile(File); + if (!Buffer) { + Diags.Report(diag::err_missing_vfs_stat_cache_file) << File; + continue; + } + + auto StatCache = + llvm::vfs::StatCacheFileSystem::create(std::move(*Buffer), BaseFS); + + if (errorToBool(StatCache.takeError())) + Diags.Report(diag::err_invalid_vfs_stat_cache) << File; + else + BaseFS = std::move(*StatCache); + } + if (VFSOverlayFiles.empty()) return BaseFS; -- cgit v1.1