From 9d6c43b4aed117f53167e72749b31a943941345d Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 13 Mar 2024 09:26:53 +0800 Subject: [NFC] [C++20] [Modules] [P1689] [Scanner] Don't use thread pool in P1689 per file mode (#84285) I suddenly found that the clang scan deps may use all concurrent threads to scan the files. It makes sense in the batch mode. But in P1689 per file mode, it simply wastes times and resources. This patch itself should be a NFC patch. It simply moves codes. --- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 195 +++++++++++++------------- 1 file changed, 100 insertions(+), 95 deletions(-) diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index d042fec..eaa76dd 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -867,13 +867,6 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { // Print out the dependency results to STDOUT by default. SharedStream DependencyOS(llvm::outs()); - DependencyScanningService Service(ScanMode, Format, OptimizeArgs, - EagerLoadModules); - llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads)); - std::vector> WorkerTools; - for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) - WorkerTools.push_back(std::make_unique(Service)); - std::vector Inputs = AdjustingCompilations->getAllCompileCommands(); @@ -893,102 +886,114 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { if (Format == ScanningOutputFormat::Full) FD.emplace(ModuleName.empty() ? Inputs.size() : 0); - if (Verbose) { - llvm::outs() << "Running clang-scan-deps on " << Inputs.size() - << " files using " << Pool.getMaxConcurrency() << " workers\n"; - } + auto ScanningTask = [&](DependencyScanningService &Service) { + DependencyScanningTool WorkerTool(Service); + + llvm::DenseSet AlreadySeenModules; + while (auto MaybeInputIndex = GetNextInputIndex()) { + size_t LocalIndex = *MaybeInputIndex; + const tooling::CompileCommand *Input = &Inputs[LocalIndex]; + std::string Filename = std::move(Input->Filename); + std::string CWD = std::move(Input->Directory); + + std::optional MaybeModuleName; + if (!ModuleName.empty()) + MaybeModuleName = ModuleName; + + std::string OutputDir(ModuleFilesDir); + if (OutputDir.empty()) + OutputDir = getModuleCachePath(Input->CommandLine); + auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) { + return ::lookupModuleOutput(MID, MOK, OutputDir); + }; + + // Run the tool on it. + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTool.getDependencyFile(Input->CommandLine, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else if (Format == ScanningOutputFormat::P1689) { + // It is useful to generate the make-format dependency output during + // the scanning for P1689. Otherwise the users need to scan again for + // it. We will generate the make-format dependency output if we find + // `-MF` in the command lines. + std::string MakeformatOutputPath; + std::string MakeformatOutput; + + auto MaybeRule = WorkerTool.getP1689ModuleDependencyFile( + *Input, CWD, MakeformatOutput, MakeformatOutputPath); + + if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs)) + HadErrors = true; + + if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() && + !HadErrors) { + static std::mutex Lock; + // With compilation database, we may open different files + // concurrently or we may write the same file concurrently. So we + // use a map here to allow multiple compile commands to write to the + // same file. Also we need a lock here to avoid data race. + static llvm::StringMap OSs; + std::unique_lock LockGuard(Lock); + + auto OSIter = OSs.find(MakeformatOutputPath); + if (OSIter == OSs.end()) { + std::error_code EC; + OSIter = + OSs.try_emplace(MakeformatOutputPath, MakeformatOutputPath, EC) + .first; + if (EC) + llvm::errs() << "Failed to open P1689 make format output file \"" + << MakeformatOutputPath << "\" for " << EC.message() + << "\n"; + } + + SharedStream MakeformatOS(OSIter->second); + llvm::Expected MaybeOutput(MakeformatOutput); + if (handleMakeDependencyToolResult(Filename, MaybeOutput, + MakeformatOS, Errs)) + HadErrors = true; + } + } else if (MaybeModuleName) { + auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( + *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, + LookupOutput); + if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } else { + auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies( + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); + if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, LocalIndex, + DependencyOS, Errs)) + HadErrors = true; + } + } + }; + + DependencyScanningService Service(ScanMode, Format, OptimizeArgs, + EagerLoadModules); llvm::Timer T; T.startTimer(); - for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) { - Pool.async([&, I]() { - llvm::DenseSet AlreadySeenModules; - while (auto MaybeInputIndex = GetNextInputIndex()) { - size_t LocalIndex = *MaybeInputIndex; - const tooling::CompileCommand *Input = &Inputs[LocalIndex]; - std::string Filename = std::move(Input->Filename); - std::string CWD = std::move(Input->Directory); - - std::optional MaybeModuleName; - if (!ModuleName.empty()) - MaybeModuleName = ModuleName; - - std::string OutputDir(ModuleFilesDir); - if (OutputDir.empty()) - OutputDir = getModuleCachePath(Input->CommandLine); - auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) { - return ::lookupModuleOutput(MID, MOK, OutputDir); - }; + if (Inputs.size() == 1) { + ScanningTask(Service); + } else { + llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(NumThreads)); - // Run the tool on it. - if (Format == ScanningOutputFormat::Make) { - auto MaybeFile = - WorkerTools[I]->getDependencyFile(Input->CommandLine, CWD); - if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, - Errs)) - HadErrors = true; - } else if (Format == ScanningOutputFormat::P1689) { - // It is useful to generate the make-format dependency output during - // the scanning for P1689. Otherwise the users need to scan again for - // it. We will generate the make-format dependency output if we find - // `-MF` in the command lines. - std::string MakeformatOutputPath; - std::string MakeformatOutput; - - auto MaybeRule = WorkerTools[I]->getP1689ModuleDependencyFile( - *Input, CWD, MakeformatOutput, MakeformatOutputPath); - - if (handleP1689DependencyToolResult(Filename, MaybeRule, PD, Errs)) - HadErrors = true; + if (Verbose) { + llvm::outs() << "Running clang-scan-deps on " << Inputs.size() + << " files using " << Pool.getMaxConcurrency() + << " workers\n"; + } - if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() && - !HadErrors) { - static std::mutex Lock; - // With compilation database, we may open different files - // concurrently or we may write the same file concurrently. So we - // use a map here to allow multiple compile commands to write to the - // same file. Also we need a lock here to avoid data race. - static llvm::StringMap OSs; - std::unique_lock LockGuard(Lock); - - auto OSIter = OSs.find(MakeformatOutputPath); - if (OSIter == OSs.end()) { - std::error_code EC; - OSIter = OSs.try_emplace(MakeformatOutputPath, - MakeformatOutputPath, EC) - .first; - if (EC) - llvm::errs() - << "Failed to open P1689 make format output file \"" - << MakeformatOutputPath << "\" for " << EC.message() - << "\n"; - } + for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I) + Pool.async([ScanningTask, &Service]() { ScanningTask(Service); }); - SharedStream MakeformatOS(OSIter->second); - llvm::Expected MaybeOutput(MakeformatOutput); - if (handleMakeDependencyToolResult(Filename, MaybeOutput, - MakeformatOS, Errs)) - HadErrors = true; - } - } else if (MaybeModuleName) { - auto MaybeModuleDepsGraph = WorkerTools[I]->getModuleDependencies( - *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput); - if (handleModuleResult(*MaybeModuleName, MaybeModuleDepsGraph, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; - } else { - auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); - if (handleTranslationUnitResult(Filename, MaybeTUDeps, *FD, - LocalIndex, DependencyOS, Errs)) - HadErrors = true; - } - } - }); + Pool.wait(); } - Pool.wait(); T.stopTimer(); if (PrintTiming) -- cgit v1.1