Merge pull request #3258 from apple/jan_svoboda/cherry_pick

[clang][deps][tooling] Cherry-pick recent dependency scanning commits

Author: jansvoboda11

clang/include/clang/Frontend/CompilerInvocation.h

@@ -51,6 +51,11 @@ class HeaderSearchOptions;
 class PreprocessorOptions;
 class TargetOptions;
 
+// This lets us create the DiagnosticsEngine with a properly-filled-out
+// DiagnosticOptions instance.
+std::unique_ptr<DiagnosticOptions>
+CreateAndPopulateDiagOpts(ArrayRef<const char *> Argv);
+
 /// Fill out Opts based on the options given in Args.
 ///
 /// Args must have been created from the OptTable returned by

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -83,8 +83,8 @@ class DependencyScanningTool {
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, dependency file contents otherwise.
   llvm::Expected<std::string>
-  getDependencyFile(const tooling::CompilationDatabase &Compilations,
-                    StringRef CWD, llvm::Optional<StringRef> ModuleName = None);
+  getDependencyFile(const std::vector<std::string> &CommandLine, StringRef CWD,
+                    llvm::Optional<StringRef> ModuleName = None);
 
   /// Collect the full module dependency graph for the input, ignoring any
   /// modules which have already been seen. If \p ModuleName isn't empty, this
@@ -99,7 +99,7 @@ class DependencyScanningTool {
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
   llvm::Expected<FullDependenciesResult>
-  getFullDependencies(const tooling::CompilationDatabase &Compilations,
+  getFullDependencies(const std::vector<std::string> &CommandLine,
                       StringRef CWD, const llvm::StringSet<> &AlreadySeen,
                       llvm::Optional<StringRef> ModuleName = None);
 

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -14,7 +14,6 @@
 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
-#include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
@@ -30,24 +29,6 @@ namespace dependencies {
 
 class DependencyScanningWorkerFilesystem;
 
-/// Compilation database that holds and reports a single compile command.
-class SingleCommandCompilationDatabase : public CompilationDatabase {
-  CompileCommand Command;
-
-public:
-  SingleCommandCompilationDatabase(CompileCommand Cmd)
-      : Command(std::move(Cmd)) {}
-
-  std::vector<CompileCommand>
-  getCompileCommands(StringRef FilePath) const override {
-    return {Command};
-  }
-
-  std::vector<CompileCommand> getAllCompileCommands() const override {
-    return {Command};
-  }
-};
-
 class DependencyConsumer {
 public:
   virtual ~DependencyConsumer() {}
@@ -74,28 +55,15 @@ class DependencyScanningWorker {
 public:
   DependencyScanningWorker(DependencyScanningService &Service);
 
-  /// Run the dependency scanning tool for a given clang driver invocation, and
-  /// report the discovered dependencies to the provided consumer. If
-  /// \p ModuleName isn't empty, this function reports the dependencies of
-  /// module \p ModuleName.
-  ///
-  /// \returns A \c StringError with the diagnostic output if clang errors
-  /// occurred, success otherwise.
-  llvm::Error computeDependenciesForClangInvocation(
-      StringRef WorkingDirectory, ArrayRef<std::string> Arguments,
-      DependencyConsumer &Consumer,
-      llvm::Optional<StringRef> ModuleName = None);
-
-  /// Run the dependency scanning tool for a given clang driver invocation (as
-  /// specified for the given Input in the CDB), and report the discovered
-  /// dependencies to the provided consumer. If \p ModuleName isn't empty, this
-  /// function reports the dependencies of module \p ModuleName.
+  /// Run the dependency scanning tool for a given clang driver command-line,
+  /// and report the discovered dependencies to the provided consumer. If \p
+  /// ModuleName isn't empty, this function reports the dependencies of module
+  /// \p ModuleName.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, success otherwise.
-  llvm::Error computeDependencies(const std::string &Input,
-                                  StringRef WorkingDirectory,
-                                  const CompilationDatabase &CDB,
+  llvm::Error computeDependencies(StringRef WorkingDirectory,
+                                  const std::vector<std::string> &CommandLine,
                                   DependencyConsumer &Consumer,
                                   llvm::Optional<StringRef> ModuleName = None);
 
@@ -104,11 +72,13 @@ class DependencyScanningWorker {
   llvm::StringSet<> AlreadySeen;
 
 private:
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
   std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
+  /// The physical filesystem overlaid by `InMemoryFS`.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
+  /// The in-memory filesystem laid on top the physical filesystem in `RealFS`.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.

clang/include/clang/Tooling/Tooling.h

@@ -268,11 +268,17 @@ class ToolInvocation {
 
   ~ToolInvocation();
 
-  /// Set a \c DiagnosticConsumer to use during parsing.
+  /// Set a \c DiagnosticConsumer to use during driver command-line parsing and
+  /// the action invocation itself.
   void setDiagnosticConsumer(DiagnosticConsumer *DiagConsumer) {
     this->DiagConsumer = DiagConsumer;
   }
 
+  /// Set a \c DiagnosticOptions to use during driver command-line parsing.
+  void setDiagnosticOptions(DiagnosticOptions *DiagOpts) {
+    this->DiagOpts = DiagOpts;
+  }
+
   /// Run the clang invocation.
   ///
   /// \returns True if there were no errors during execution.
@@ -290,6 +296,7 @@ class ToolInvocation {
   FileManager *Files;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
   DiagnosticConsumer *DiagConsumer = nullptr;
+  DiagnosticOptions *DiagOpts = nullptr;
 };
 
 /// Utility to run a FrontendAction over a set of files.

clang/lib/Frontend/CompilerInvocation.cpp

@@ -2355,6 +2355,19 @@ void CompilerInvocation::GenerateDiagnosticArgs(
   }
 }
 
+std::unique_ptr<DiagnosticOptions>
+clang::CreateAndPopulateDiagOpts(ArrayRef<const char *> Argv) {
+  auto DiagOpts = std::make_unique<DiagnosticOptions>();
+  unsigned MissingArgIndex, MissingArgCount;
+  InputArgList Args = getDriverOptTable().ParseArgs(
+      Argv.slice(1), MissingArgIndex, MissingArgCount);
+  // We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
+  // Any errors that would be diagnosed here will also be diagnosed later,
+  // when the DiagnosticsEngine actually exists.
+  (void)ParseDiagnosticArgs(*DiagOpts, Args);
+  return DiagOpts;
+}
+
 bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
                                 DiagnosticsEngine *Diags,
                                 bool DefaultDiagColor) {

clang/lib/Interpreter/Interpreter.cpp

@@ -147,15 +147,10 @@ IncrementalCompilerBuilder::create(std::vector<const char *> &ClangArgv) {
   // Buffer diagnostics from argument parsing so that we can output them using a
   // well formed diagnostic object.
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
+      CreateAndPopulateDiagOpts(ClangArgv);
   TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
   DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
-  unsigned MissingArgIndex, MissingArgCount;
-  const llvm::opt::OptTable &Opts = driver::getDriverOptTable();
-  llvm::opt::InputArgList ParsedArgs =
-      Opts.ParseArgs(ArrayRef<const char *>(ClangArgv).slice(1),
-                     MissingArgIndex, MissingArgCount);
-  ParseDiagnosticArgs(*DiagOpts, ParsedArgs, &Diags);
 
   driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0],
                         llvm::sys::getProcessTriple(), Diags);

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -50,7 +50,7 @@ DependencyScanningTool::DependencyScanningTool(
     : Worker(Service) {}
 
 llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
-    const tooling::CompilationDatabase &Compilations, StringRef CWD,
+    const std::vector<std::string> &CommandLine, StringRef CWD,
     llvm::Optional<StringRef> ModuleName) {
   /// Prints out all of the gathered dependencies into a string.
   class MakeDependencyPrinterConsumer : public DependencyConsumer {
@@ -103,18 +103,9 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
     std::vector<std::string> Dependencies;
   };
 
-  // We expect a single command here because if a source file occurs multiple
-  // times in the original CDB, then `computeDependencies` would run the
-  // `DependencyScanningAction` once for every time the input occured in the
-  // CDB. Instead we split up the CDB into single command chunks to avoid this
-  // behavior.
-  assert(Compilations.getAllCompileCommands().size() == 1 &&
-         "Expected a compilation database with a single command!");
-  std::string Input = Compilations.getAllCompileCommands().front().Filename;
-
   MakeDependencyPrinterConsumer Consumer;
-  auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer,
-                                           ModuleName);
+  auto Result =
+      Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
     return std::move(Result);
   std::string Output;
@@ -124,7 +115,7 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
 
 llvm::Expected<FullDependenciesResult>
 DependencyScanningTool::getFullDependencies(
-    const tooling::CompilationDatabase &Compilations, StringRef CWD,
+    const std::vector<std::string> &CommandLine, StringRef CWD,
     const llvm::StringSet<> &AlreadySeen,
     llvm::Optional<StringRef> ModuleName) {
   class FullDependencyPrinterConsumer : public DependencyConsumer {
@@ -189,18 +180,9 @@ DependencyScanningTool::getFullDependencies(
     const llvm::StringSet<> &AlreadySeen;
   };
 
-  // We expect a single command here because if a source file occurs multiple
-  // times in the original CDB, then `computeDependencies` would run the
-  // `DependencyScanningAction` once for every time the input occured in the
-  // CDB. Instead we split up the CDB into single command chunks to avoid this
-  // behavior.
-  assert(Compilations.getAllCompileCommands().size() == 1 &&
-         "Expected a compilation database with a single command!");
-  std::string Input = Compilations.getAllCompileCommands().front().Filename;
-
   FullDependencyPrinterConsumer Consumer(AlreadySeen);
-  llvm::Error Result = Worker.computeDependencies(Input, CWD, Compilations,
-                                                  Consumer, ModuleName);
+  llvm::Error Result =
+      Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
     return std::move(Result);
   return Consumer.getFullDependencies();