[clang][ExtractAPI] Add --emit-symbol-graph option

Add new --emit-symbol-graph=<DIR> option which generates ExtractAPI symbol
graph information of .c/.m files on regular compilation job and put them in
the provided "DIR" directory.

Reviewed By: dang

Differential Revision: https://reviews.llvm.org/D152356

2 files changed, 148 insertions, 20 deletions

diff --git a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
index c27b2d0..eb533a9 100644
--- a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
+++ b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
@@ -28,12 +28,14 @@
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendOptions.h"
+#include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Error.h"
@@ -245,6 +247,20 @@ private:
   LocationFileChecker &LCF;
 };
 
+class WrappingExtractAPIConsumer : public ASTConsumer {
+public:
+  WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)
+      : Visitor(Context, API) {}
+
+  void HandleTranslationUnit(ASTContext &Context) override {
+    // Use ExtractAPIVisitor to traverse symbol declarations in the context.
+    Visitor.TraverseDecl(Context.getTranslationUnitDecl());
+  }
+
+private:
+  ExtractAPIVisitor<> Visitor;
+};
+
 class ExtractAPIConsumer : public ASTConsumer {
 public:
   ExtractAPIConsumer(ASTContext &Context,
@@ -263,9 +279,8 @@ private:
 
 class MacroCallback : public PPCallbacks {
 public:
-  MacroCallback(const SourceManager &SM, LocationFileChecker &LCF, APISet &API,
-                Preprocessor &PP)
-      : SM(SM), LCF(LCF), API(API), PP(PP) {}
+  MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)
+      : SM(SM), API(API), PP(PP) {}
 
   void MacroDefined(const Token &MacroNameToken,
                     const MacroDirective *MD) override {
@@ -305,7 +320,7 @@ public:
       if (PM.MD->getMacroInfo()->isUsedForHeaderGuard())
         continue;
 
-      if (!LCF(PM.MacroNameToken.getLocation()))
+      if (!shouldMacroBeIncluded(PM))
         continue;
 
       StringRef Name = PM.MacroNameToken.getIdentifierInfo()->getName();
@@ -323,7 +338,7 @@ public:
     PendingMacros.clear();
   }
 
-private:
+protected:
   struct PendingMacro {
     Token MacroNameToken;
     const MacroDirective *MD;
@@ -332,18 +347,58 @@ private:
         : MacroNameToken(MacroNameToken), MD(MD) {}
   };
 
+  virtual bool shouldMacroBeIncluded(const PendingMacro &PM) { return true; }
+
   const SourceManager &SM;
-  LocationFileChecker &LCF;
   APISet &API;
   Preprocessor &PP;
   llvm::SmallVector<PendingMacro> PendingMacros;
 };
 
+class APIMacroCallback : public MacroCallback {
+public:
+  APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,
+                   LocationFileChecker &LCF)
+      : MacroCallback(SM, API, PP), LCF(LCF) {}
+
+  bool shouldMacroBeIncluded(const PendingMacro &PM) override {
+    // Do not include macros from external files
+    return LCF(PM.MacroNameToken.getLocation());
+  }
+
+private:
+  LocationFileChecker &LCF;
+};
+
 } // namespace
 
+void ExtractAPIActionBase::ImplEndSourceFileAction() {
+  if (!OS)
+    return;
+
+  // Setup a SymbolGraphSerializer to write out collected API information in
+  // the Symbol Graph format.
+  // FIXME: Make the kind of APISerializer configurable.
+  SymbolGraphSerializer SGSerializer(*API, IgnoresList);
+  SGSerializer.serialize(*OS);
+  OS.reset();
+}
+
+std::unique_ptr<raw_pwrite_stream>
+ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) {
+  std::unique_ptr<raw_pwrite_stream> OS;
+  OS = CI.createDefaultOutputFile(/*Binary=*/false, InFile,
+                                  /*Extension=*/"json",
+                                  /*RemoveFileOnSignal=*/false);
+  if (!OS)
+    return nullptr;
+  return OS;
+}
+
 std::unique_ptr<ASTConsumer>
 ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   OS = CreateOutputFile(CI, InFile);
+
   if (!OS)
     return nullptr;
 
@@ -357,8 +412,8 @@ ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 
   auto LCF = std::make_unique<LocationFileChecker>(CI, KnownInputFiles);
 
-  CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
-      CI.getSourceManager(), *LCF, *API, CI.getPreprocessor()));
+  CI.getPreprocessor().addPPCallbacks(std::make_unique<APIMacroCallback>(
+      CI.getSourceManager(), *API, CI.getPreprocessor(), *LCF));
 
   // Do not include location in anonymous decls.
   PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
@@ -438,23 +493,88 @@ bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
   return true;
 }
 
-void ExtractAPIAction::EndSourceFileAction() {
+void ExtractAPIAction::EndSourceFileAction() { ImplEndSourceFileAction(); }
+
+std::unique_ptr<ASTConsumer>
+WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,
+                                            StringRef InFile) {
+  auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+  if (!OtherConsumer)
+    return nullptr;
+
+  CreatedASTConsumer = true;
+
+  OS = CreateOutputFile(CI, InFile);
   if (!OS)
-    return;
+    return nullptr;
 
-  // Setup a SymbolGraphSerializer to write out collected API information in
-  // the Symbol Graph format.
-  // FIXME: Make the kind of APISerializer configurable.
-  SymbolGraphSerializer SGSerializer(*API, IgnoresList);
-  SGSerializer.serialize(*OS);
-  OS.reset();
+  auto ProductName = CI.getFrontendOpts().ProductName;
+
+  // Now that we have enough information about the language options and the
+  // target triple, let's create the APISet before anyone uses it.
+  API = std::make_unique<APISet>(
+      CI.getTarget().getTriple(),
+      CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), ProductName);
+
+  CI.getPreprocessor().addPPCallbacks(std::make_unique<MacroCallback>(
+      CI.getSourceManager(), *API, CI.getPreprocessor()));
+
+  // Do not include location in anonymous decls.
+  PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
+  Policy.AnonymousTagLocations = false;
+  CI.getASTContext().setPrintingPolicy(Policy);
+
+  if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
+    llvm::handleAllErrors(
+        APIIgnoresList::create(CI.getFrontendOpts().ExtractAPIIgnoresFileList,
+                               CI.getFileManager())
+            .moveInto(IgnoresList),
+        [&CI](const IgnoresFileNotFound &Err) {
+          CI.getDiagnostics().Report(
+              diag::err_extract_api_ignores_file_not_found)
+              << Err.Path;
+        });
+  }
+
+  auto WrappingConsumer =
+      std::make_unique<WrappingExtractAPIConsumer>(CI.getASTContext(), *API);
+  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+  Consumers.push_back(std::move(OtherConsumer));
+  Consumers.push_back(std::move(WrappingConsumer));
+
+  return std::make_unique<MultiplexConsumer>(std::move(Consumers));
+}
+
+void WrappingExtractAPIAction::EndSourceFileAction() {
+  // Invoke wrapped action's method.
+  WrapperFrontendAction::EndSourceFileAction();
+
+  if (CreatedASTConsumer) {
+    ImplEndSourceFileAction();
+  }
 }
 
 std::unique_ptr<raw_pwrite_stream>
-ExtractAPIAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) {
-  std::unique_ptr<raw_pwrite_stream> OS =
-      CI.createDefaultOutputFile(/*Binary=*/false, InFile, /*Extension=*/"json",
-                                 /*RemoveFileOnSignal=*/false);
+WrappingExtractAPIAction::CreateOutputFile(CompilerInstance &CI,
+                                           StringRef InFile) {
+  std::unique_ptr<raw_pwrite_stream> OS;
+  std::string OutputDir = CI.getFrontendOpts().SymbolGraphOutputDir;
+
+  // The symbol graphs need to be generated as a side effect of regular
+  // compilation so the output should be dumped in the directory provided with
+  // the command line option.
+  llvm::SmallString<128> OutFilePath(OutputDir);
+  auto Seperator = llvm::sys::path::get_separator();
+  auto Infilename = llvm::sys::path::filename(InFile);
+  OutFilePath.append({Seperator, Infilename});
+  llvm::sys::path::replace_extension(OutFilePath, "json");
+  // StringRef outputFilePathref = *OutFilePath;
+
+  // don't use the default output file
+  OS = CI.createOutputFile(/*OutputPath=*/OutFilePath, /*Binary=*/false,
+                           /*RemoveFileOnSignal=*/true,
+                           /*UseTemporary=*/true,
+                           /*CreateMissingDirectories=*/true);
   if (!OS)
     return nullptr;
   return OS;
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 47157ca..310f677 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -178,6 +178,14 @@ CreateFrontendAction(CompilerInstance &CI) {
   }
 #endif
 
+  // Wrap the base FE action in an extract api action to generate
+  // symbol graph as a biproduct of comilation ( enabled with
+  // --emit-symbol-graph option )
+  if (!FEOpts.SymbolGraphOutputDir.empty()) {
+    CI.getCodeGenOpts().ClearASTBeforeBackend = false;
+    Act = std::make_unique<WrappingExtractAPIAction>(std::move(Act));
+  }
+
   // If there are any AST files to merge, create a frontend action
   // adaptor to perform the merge.
   if (!FEOpts.ASTMergeFiles.empty())