Merge pull request #9573 from swiftlang/jan_svoboda/stable-20240723-scanner-perf-2

🍒 [clang][deps][modules] Speed up dependency scanning (2)

Author: jansvoboda11

clang/include/clang/AST/ASTContext.h

@@ -476,14 +476,6 @@ class ASTContext : public RefCountedBase<ASTContext> {
   llvm::StringMap<const Module *> PrimaryModuleNameMap;
   llvm::DenseMap<const Module *, const Module *> SameModuleLookupSet;
 
-  /// The include tree that is being built, if any.
-  /// See \c FrontendOptions::CASIncludeTreeID.
-  std::optional<std::string> CASIncludeTreeID;
-
-  /// The cas-fs tree that is being built, if any.
-  /// See \c FileSystemOptions::CASFileSystemRootID.
-  std::optional<std::string> CASFileSystemRootID;
-
   static constexpr unsigned ConstantArrayTypesLog2InitSize = 8;
   static constexpr unsigned GeneralTypesLog2InitSize = 9;
   static constexpr unsigned FunctionProtoTypesLog2InitSize = 12;
@@ -1123,20 +1115,6 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// a module fragment or a module unit but not a C++20 module.
   bool isInSameModule(const Module *M1, const Module *M2);
 
-  std::optional<std::string> getCASIncludeTreeID() const {
-    return CASIncludeTreeID;
-  }
-  void setCASIncludeTreeID(std::string ID) {
-    CASIncludeTreeID = std::move(ID);
-  }
-
-  std::optional<std::string> getCASFileSystemRootID() const {
-    return CASFileSystemRootID;
-  }
-  void setCASFileSystemRootID(std::string ID) {
-    CASFileSystemRootID = std::move(ID);
-  }
-
   TranslationUnitDecl *getTranslationUnitDecl() const {
     return TUDecl->getMostRecentDecl();
   }

clang/include/clang/Lex/HeaderSearchOptions.h

@@ -255,6 +255,10 @@ class HeaderSearchOptions {
   LLVM_PREFERRED_TYPE(bool)
   unsigned ModulesHashContent : 1;
 
+  /// Whether AST files should only contain the preprocessor information.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned ModulesSerializeOnlyPreprocessor : 1;
+
   /// Whether we should include all things that could impact the module in the
   /// hash.
   ///
@@ -282,6 +286,7 @@ class HeaderSearchOptions {
         ModulesSkipHeaderSearchPaths(false),
         ModulesSkipPragmaDiagnosticMappings(false),
         ModulesPruneNonAffectingModuleMaps(true), ModulesHashContent(false),
+        ModulesSerializeOnlyPreprocessor(false),
         ModulesStrictContextHash(false), ModulesIncludeVFSUsage(false) {}
 
   /// AddPath - Add the \p Path path to the specified \p Group list.

clang/include/clang/Lex/Preprocessor.h

@@ -2428,6 +2428,28 @@ class Preprocessor {
            !IsAtImport;
   }
 
+private:
+  /// The include tree that is being built, if any.
+  /// See \c FrontendOptions::CASIncludeTreeID.
+  std::optional<std::string> CASIncludeTreeID;
+
+  /// The cas-fs tree that is being built, if any.
+  /// See \c FileSystemOptions::CASFileSystemRootID.
+  std::optional<std::string> CASFileSystemRootID;
+
+public:
+  std::optional<std::string> getCASIncludeTreeID() const {
+    return CASIncludeTreeID;
+  }
+  void setCASIncludeTreeID(std::string ID) { CASIncludeTreeID = std::move(ID); }
+
+  std::optional<std::string> getCASFileSystemRootID() const {
+    return CASFileSystemRootID;
+  }
+  void setCASFileSystemRootID(std::string ID) {
+    CASFileSystemRootID = std::move(ID);
+  }
+
   /// Allocate a new MacroInfo object with the provided SourceLocation.
   MacroInfo *AllocateMacroInfo(SourceLocation L);
 

clang/include/clang/Serialization/ASTRecordWriter.h

@@ -60,8 +60,9 @@ class ASTRecordWriter
 
 public:
   /// Construct a ASTRecordWriter that uses the default encoding scheme.
-  ASTRecordWriter(ASTWriter &W, ASTWriter::RecordDataImpl &Record)
-      : DataStreamBasicWriter(W.getASTContext()), Writer(&W), Record(&Record) {}
+  ASTRecordWriter(ASTContext &Context, ASTWriter &W,
+                  ASTWriter::RecordDataImpl &Record)
+      : DataStreamBasicWriter(Context), Writer(&W), Record(&Record) {}
 
   /// Construct a ASTRecordWriter that uses the same encoding scheme as another
   /// ASTRecordWriter.
@@ -208,7 +209,7 @@ class ASTRecordWriter
 
   /// Emit a reference to a type.
   void AddTypeRef(QualType T) {
-    return Writer->AddTypeRef(T, *Record);
+    return Writer->AddTypeRef(getASTContext(), T, *Record);
   }
   void writeQualType(QualType T) {
     AddTypeRef(T);

clang/include/clang/Serialization/ASTWriter.h

@@ -119,9 +119,6 @@ class ASTWriter : public ASTDeserializationListener,
   /// The PCM manager which manages memory buffers for pcm files.
   InMemoryModuleCache &ModuleCache;
 
-  /// The ASTContext we're writing.
-  ASTContext *Context = nullptr;
-
   /// The preprocessor we're writing.
   Preprocessor *PP = nullptr;
 
@@ -537,55 +534,55 @@ class ASTWriter : public ASTDeserializationListener,
   unsigned getSubmoduleID(Module *Mod);
 
   /// Write the given subexpression to the bitstream.
-  void WriteSubStmt(Stmt *S);
+  void WriteSubStmt(ASTContext &Context, Stmt *S);
 
   void WriteBlockInfoBlock();
-  void WriteControlBlock(Preprocessor &PP, ASTContext &Context,
-                         StringRef isysroot);
+  void WriteControlBlock(Preprocessor &PP, StringRef isysroot);
 
   /// Write out the signature and diagnostic options, and return the signature.
-  void writeUnhashedControlBlock(Preprocessor &PP, ASTContext &Context);
+  void writeUnhashedControlBlock(Preprocessor &PP);
   ASTFileSignature backpatchSignature();
 
   /// Calculate hash of the pcm content.
   std::pair<ASTFileSignature, ASTFileSignature> createSignature() const;
   ASTFileSignature createSignatureForNamedModule() const;
 
   void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts);
-  void WriteSourceManagerBlock(SourceManager &SourceMgr,
-                               const Preprocessor &PP);
+  void WriteSourceManagerBlock(SourceManager &SourceMgr);
   void WritePreprocessor(const Preprocessor &PP, bool IsModule);
   void WriteHeaderSearch(const HeaderSearch &HS);
   void WritePreprocessorDetail(PreprocessingRecord &PPRec,
                                uint64_t MacroOffsetsBase);
-  void WriteSubmodules(Module *WritingModule);
+  void WriteSubmodules(Module *WritingModule, ASTContext *Context);
 
   void WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag,
                                      bool isModule);
 
   unsigned TypeExtQualAbbrev = 0;
   void WriteTypeAbbrevs();
-  void WriteType(QualType T);
+  void WriteType(ASTContext &Context, QualType T);
 
   bool isLookupResultExternal(StoredDeclsList &Result, DeclContext *DC);
 
-  void GenerateNameLookupTable(const DeclContext *DC,
+  void GenerateNameLookupTable(ASTContext &Context, const DeclContext *DC,
                                llvm::SmallVectorImpl<char> &LookupTable);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context,
                                         const DeclContext *DC);
   uint64_t WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC);
   void WriteTypeDeclOffsets();
   void WriteFileDeclIDsMap();
-  void WriteComments();
+  void WriteComments(ASTContext &Context);
   void WriteSelectors(Sema &SemaRef);
   void WriteReferencedSelectorsPool(Sema &SemaRef);
-  void WriteIdentifierTable(Preprocessor &PP, IdentifierResolver &IdResolver,
+  void WriteIdentifierTable(Preprocessor &PP, IdentifierResolver *IdResolver,
                             bool IsModule);
   void WriteDeclAndTypes(ASTContext &Context);
   void PrepareWritingSpecialDecls(Sema &SemaRef);
   void WriteSpecialDeclRecords(Sema &SemaRef);
-  void WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord);
-  void WriteDeclContextVisibleUpdate(const DeclContext *DC);
+  void WriteDeclUpdatesBlocks(ASTContext &Context,
+                              RecordDataImpl &OffsetsRecord);
+  void WriteDeclContextVisibleUpdate(ASTContext &Context,
+                                     const DeclContext *DC);
   void WriteFPPragmaOptions(const FPOptionsOverride &Opts);
   void WriteOpenCLExtensions(Sema &SemaRef);
   void WriteCUDAPragmas(Sema &SemaRef);
@@ -634,7 +631,7 @@ class ASTWriter : public ASTDeserializationListener,
   void WriteDeclAbbrevs();
   void WriteDecl(ASTContext &Context, Decl *D);
 
-  ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot,
+  ASTFileSignature WriteASTCore(Sema *SemaPtr, StringRef isysroot,
                                 Module *WritingModule);
 
 public:
@@ -647,22 +644,20 @@ class ASTWriter : public ASTDeserializationListener,
             bool GeneratingReducedBMI = false);
   ~ASTWriter() override;
 
-  ASTContext &getASTContext() const {
-    assert(Context && "requested AST context when not writing AST");
-    return *Context;
-  }
-
   const LangOptions &getLangOpts() const;
 
   /// Get a timestamp for output into the AST file. The actual timestamp
   /// of the specified file may be ignored if we have been instructed to not
   /// include timestamps in the output file.
   time_t getTimestampForOutput(const FileEntry *E) const;
 
-  /// Write a precompiled header for the given semantic analysis.
+  /// Write a precompiled header or a module with the AST produced by the
+  /// \c Sema object, or a dependency scanner module with the preprocessor state
+  /// produced by the \c Preprocessor object.
   ///
-  /// \param SemaRef a reference to the semantic analysis object that processed
-  /// the AST to be written into the precompiled header.
+  /// \param Subject The \c Sema object that processed the AST to be written, or
+  /// in the case of a dependency scanner module the \c Preprocessor that holds
+  /// the state.
   ///
   /// \param WritingModule The module that we are writing. If null, we are
   /// writing a precompiled header.
@@ -673,8 +668,9 @@ class ASTWriter : public ASTDeserializationListener,
   ///
   /// \return the module signature, which eventually will be a hash of
   /// the module but currently is merely a random 32-bit number.
-  ASTFileSignature WriteAST(Sema &SemaRef, StringRef OutputFile,
-                            Module *WritingModule, StringRef isysroot,
+  ASTFileSignature WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject,
+                            StringRef OutputFile, Module *WritingModule,
+                            StringRef isysroot,
                             bool ShouldCacheASTInMemory = false);
 
   /// Emit a token.
@@ -717,10 +713,10 @@ class ASTWriter : public ASTDeserializationListener,
   uint32_t getMacroDirectivesOffset(const IdentifierInfo *Name);
 
   /// Emit a reference to a type.
-  void AddTypeRef(QualType T, RecordDataImpl &Record);
+  void AddTypeRef(ASTContext &Context, QualType T, RecordDataImpl &Record);
 
   /// Force a type to be emitted and get its ID.
-  serialization::TypeID GetOrCreateTypeID(QualType T);
+  serialization::TypeID GetOrCreateTypeID(ASTContext &Context, QualType T);
 
   /// Find the first local declaration of a given local redeclarable
   /// decl.
@@ -921,9 +917,9 @@ class PCHGenerator : public SemaConsumer {
   void anchor() override;
 
   Preprocessor &PP;
+  llvm::PointerUnion<Sema *, Preprocessor *> Subject;
   std::string OutputFile;
   std::string isysroot;
-  Sema *SemaPtr;
   std::shared_ptr<PCHBuffer> Buffer;
   llvm::BitstreamWriter Stream;
   ASTWriter Writer;
@@ -938,9 +934,7 @@ class PCHGenerator : public SemaConsumer {
   bool isComplete() const { return Buffer->IsComplete; }
   PCHBuffer *getBufferPtr() { return Buffer.get(); }
   StringRef getOutputFile() const { return OutputFile; }
-  DiagnosticsEngine &getDiagnostics() const {
-    return SemaPtr->getDiagnostics();
-  }
+  DiagnosticsEngine &getDiagnostics() const;
   Preprocessor &getPreprocessor() { return PP; }
 
   virtual Module *getEmittingModule(ASTContext &Ctx);
@@ -956,7 +950,7 @@ class PCHGenerator : public SemaConsumer {
                bool GeneratingReducedBMI = false);
   ~PCHGenerator() override;
 
-  void InitializeSema(Sema &S) override { SemaPtr = &S; }
+  void InitializeSema(Sema &S) override;
   void HandleTranslationUnit(ASTContext &Ctx) override;
   void HandleVTable(CXXRecordDecl *RD) override { Writer.handleVTable(RD); }
   ASTMutationListener *GetASTMutationListener() override;

clang/include/clang/Serialization/ModuleFile.h

@@ -62,15 +62,20 @@ enum ModuleKind {
 
 /// The input file info that has been loaded from an AST file.
 struct InputFileInfo {
-  std::string FilenameAsRequested;
-  std::string Filename;
+  StringRef UnresolvedImportedFilenameAsRequested;
+  StringRef UnresolvedImportedFilename;
+
   uint64_t ContentHash;
   off_t StoredSize;
   time_t StoredTime;
   bool Overridden;
   bool Transient;
   bool TopLevel;
   bool ModuleMap;
+
+  bool isValid() const {
+    return !UnresolvedImportedFilenameAsRequested.empty();
+  }
 };
 
 /// The input file that has been loaded from this AST file, along with

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -123,10 +123,6 @@ struct ModuleDeps {
   /// additionally appear in \c FileDeps as a dependency.
   std::string ClangModuleMapFile;
 
-  /// A collection of absolute paths to files that this module directly depends
-  /// on, not including transitive dependencies.
-  llvm::StringSet<> FileDeps;
-
   /// A collection of absolute paths to module map files that this module needs
   /// to know about. The ordering is significant.
   std::vector<std::string> ModuleMapFileDeps;
@@ -155,13 +151,25 @@ struct ModuleDeps {
   /// an entity from this module is used.
   llvm::SmallVector<Module::LinkLibrary, 2> LinkLibraries;
 
+  /// Invokes \c Cb for all file dependencies of this module. Each provided
+  /// \c StringRef is only valid within the individual callback invocation.
+  void forEachFileDep(llvm::function_ref<void(StringRef)> Cb) const;
+
   /// Get (or compute) the compiler invocation that can be used to build this
   /// module. Does not include argv[0].
   const std::vector<std::string> &getBuildArguments();
 
 private:
+  friend class ModuleDepCollector;
   friend class ModuleDepCollectorPP;
 
+  /// The base directory for relative paths in \c FileDeps.
+  std::string FileDepsBaseDir;
+
+  /// A collection of paths to files that this module directly depends on, not
+  /// including transitive dependencies.
+  std::vector<std::string> FileDeps;
+
   std::variant<std::monostate, CowCompilerInvocation, std::vector<std::string>>
       BuildInfo;
 };

clang/lib/Frontend/ASTUnit.cpp

@@ -2358,7 +2358,7 @@ bool ASTUnit::Save(StringRef File) {
 
 static bool serializeUnit(ASTWriter &Writer, SmallVectorImpl<char> &Buffer,
                           Sema &S, raw_ostream &OS) {
-  Writer.WriteAST(S, std::string(), nullptr, "");
+  Writer.WriteAST(&S, std::string(), nullptr, "");
 
   // Write the generated bitstream to "Out".
   if (!Buffer.empty())

clang/lib/Frontend/FrontendAction.cpp

@@ -1208,7 +1208,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     }
 
     if (!CI.getFrontendOpts().CASIncludeTreeID.empty())
-      CI.getASTContext().setCASIncludeTreeID(
+      CI.getPreprocessor().setCASIncludeTreeID(
           CI.getFrontendOpts().CASIncludeTreeID);
 
     CI.setASTConsumer(std::move(Consumer));