Merge pull request swiftlang#29463 from davidungar/PBP-2-21-ICNR-on

[DNM, Incremental: Type-body-fingerprints on-by-default]

Author: David Ungar

include/swift/AST/ASTTypeIDZone.def

@@ -53,3 +53,4 @@ SWIFT_TYPEID_NAMED(SourceFile *, SourceFile)
 SWIFT_TYPEID_NAMED(TypeAliasDecl *, TypeAliasDecl)
 SWIFT_TYPEID_NAMED(ValueDecl *, ValueDecl)
 SWIFT_TYPEID_NAMED(VarDecl *, VarDecl)
+SWIFT_TYPEID(FingerprintAndMembers)

include/swift/AST/ASTTypeIDs.h

@@ -61,6 +61,7 @@ struct TypePair;
 struct TypeWitnessAndDecl;
 enum class AncestryFlags : uint8_t;
 enum class ImplicitMemberAction : uint8_t;
+struct FingerprintAndMembers;
 
 // Define the AST type zone (zone 1)
 #define SWIFT_TYPEID_ZONE AST

include/swift/AST/DeclContext.h

@@ -802,6 +802,12 @@ class IterableDeclContext {
   // Some Decls are IterableDeclContexts, but not all.
   static bool classof(const Decl *D);
 
+  /// Return a hash of all tokens in the body for dependency analysis, if
+  /// available.
+  Optional<std::string> getBodyFingerprint() const;
+
+  bool areDependenciesUsingTokenHashesForTypeBodies() const;
+
 private:
   /// Add a member to the list for iteration purposes, but do not notify the
   /// subclass that we have done so.

include/swift/AST/DiagnosticsFrontend.def

@@ -94,6 +94,8 @@ ERROR(error_unknown_arg,none,
       "unknown argument: '%0'", (StringRef))
 ERROR(error_invalid_arg_value,none,
       "invalid value '%1' in '%0'", (StringRef, StringRef))
+WARNING(warning_type_fingerprints_require_fine_grained_dependencies,none,
+      "Type fingerprints require fine-grained dependencies", ())
 WARNING(warning_cannot_multithread_batch_mode,none,
       "ignoring -num-threads argument; cannot multithread batch mode", ())
 ERROR(error_unsupported_option_argument,none,

include/swift/AST/FineGrainedDependencies.h

@@ -15,6 +15,7 @@
 
 #include "swift/Basic/Debug.h"
 #include "swift/Basic/LLVM.h"
+#include "swift/Basic/NullablePtr.h"
 #include "swift/Basic/Range.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/Support/MD5.h"
@@ -139,6 +140,11 @@ template <typename Key1, typename Key2, typename Value> class TwoStageMap {
     return iter2 == iter->second.end() ? None : Optional<Value>(iter2->second);
   }
 
+  NullablePtr<const InnerMap> find(const Key1 &k1) const {
+    auto iter = map.find(k1);
+    return iter == map.end() ? nullptr : &iter->second;
+  }
+
   /// The sought value must be present.
   Value findAndErase(const Key1 &k1, const Key2 &k2) {
     auto &submap = map[k1];
@@ -237,6 +243,20 @@ class BiIndexedTwoStageMap {
   }
   Optional<Value> find(const Key2 &k2, Key1 &k1) const { return find(k1, k2); }
 
+  /// Return the submap for a given Key1. May create one, after the fashion of
+  /// the standard libary.
+  const Key2Map &operator[](const Key1 &k1) { return map1[k1]; }
+  /// Return the submap for a given Key2. May create one, after the fashion of
+  /// the standard libary.
+  const Key1Map &operator[](const Key2 &k2) { return map2[k2]; }
+
+  NullablePtr<const Key2Map> find(const Key1 &k1) const {
+    return map1.find(k1);
+  }
+  NullablePtr<const Key1Map> find(const Key2 &k2) const {
+    return map2.find(k2);
+  }
+
   /// Element must be present.
   /// Return the erased value.
   Value findAndErase(const Key1 &k1, const Key2 &k2) {
@@ -250,12 +270,6 @@ class BiIndexedTwoStageMap {
   Value findAndErase(const Key2 &k2, const Key1 &k1) {
     return findAndErase(k1, k2);
   }
-  /// Return the submap for a given Key1. May create one, after the fashion of
-  /// the standard libary.
-  const Key2Map &operator[](const Key1 &k1) { return map1[k1]; }
-  /// Return the submap for a given Key2. May create one, after the fashion of
-  /// the standard libary.
-  const Key1Map &operator[](const Key2 &k2) { return map2[k2]; }
 
   /// Invoke \p fn on each Key2 and Value matching (\p k1, *)
   void forEachValueMatching(
@@ -500,8 +514,7 @@ class DependencyKey {
   template <NodeKind kind>
   static DependencyKey createDependedUponKey(StringRef);
 
-  static DependencyKey
-  createTransitiveKeyForWholeSourceFile(StringRef swiftDeps);
+  static DependencyKey createKeyForWholeSourceFile(StringRef swiftDeps);
 
   std::string humanReadableName() const;
 
@@ -787,6 +800,12 @@ class SourceFileDepGraph {
 
   static constexpr char noncascadingOrPrivatePrefix = '#';
 
+  static std::string noncascading(std::string name);
+
+  LLVM_ATTRIBUTE_UNUSED
+  static std::string privatize(std::string name);
+
+
   /// Nodes are owned by the graph.
   ~SourceFileDepGraph() {
     forEachNode([&](SourceFileDepGraphNode *n) { delete n; });

include/swift/AST/ParseRequests.h

@@ -27,21 +27,31 @@ template<typename Request>
 void reportEvaluatedRequest(UnifiedStatsReporter &stats,
                             const Request &request);
 
+struct FingerprintAndMembers {
+  Optional<std::string> fingerprint = None;
+  ArrayRef<Decl *> members = {};
+  bool operator==(const FingerprintAndMembers &x) const {
+    return fingerprint == x.fingerprint && members == x.members;
+  }
+};
+
+void simple_display(llvm::raw_ostream &out, const FingerprintAndMembers &value);
+
 /// Parse the members of a nominal type declaration or extension.
-class ParseMembersRequest :
-    public SimpleRequest<ParseMembersRequest,
-                         ArrayRef<Decl *>(IterableDeclContext *),
-                         CacheKind::Cached>
-{
+/// Return a fingerprint and the members.
+class ParseMembersRequest
+    : public SimpleRequest<ParseMembersRequest,
+                           FingerprintAndMembers(IterableDeclContext *),
+                           CacheKind::Cached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
   // Evaluation.
-  ArrayRef<Decl *> evaluate(Evaluator &evaluator,
-                            IterableDeclContext *idc) const;
+  FingerprintAndMembers evaluate(Evaluator &evaluator,
+                                 IterableDeclContext *idc) const;
 
 public:
   // Caching

include/swift/AST/ParseTypeIDZone.def

@@ -15,7 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 SWIFT_REQUEST(Parse, ParseMembersRequest,
-              ArrayRef<Decl *>(IterableDeclContext *), Cached, NoLocationInfo)
+              FingerprintAndMembers(IterableDeclContext *), Cached, NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
               BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -423,12 +423,8 @@ class SourceFile final : public FileUnit {
     return InterfaceHash.hasValue();
   }
 
-  void recordInterfaceToken(StringRef token) {
-    assert(!token.empty());
-    InterfaceHash->update(token);
-    // Add null byte to separate tokens.
-    uint8_t a[1] = {0};
-    InterfaceHash->update(a);
+  NullablePtr<llvm::MD5> getInterfaceHashPtr() {
+    return InterfaceHash ? InterfaceHash.getPointer() : nullptr;
   }
 
   void getInterfaceHash(llvm::SmallString<32> &str) const {

include/swift/Basic/LangOptions.h

@@ -298,9 +298,14 @@ namespace swift {
 
     /// Emit the newer, finer-grained swiftdeps file. Eventually will support
     /// faster rebuilds.
-    /// The initializer here sets the default for the frontend and driver.
     bool EnableFineGrainedDependencies = true;
 
+    /// Instead of hashing tokens inside of NominalType and ExtensionBodies into
+    /// the interface hash, hash them into per-iterable-decl-context
+    /// fingerprints. Fine-grained dependency types won't dirty every provides
+    /// in a file when the user adds a member to, e.g., a struct.
+    bool EnableTypeFingerprints = true;
+
     /// When using fine-grained dependencies, emit dot files for every swiftdeps
     /// file.
     bool EmitFineGrainedDependencySourcefileDotFiles = false;

include/swift/Basic/Statistic.h

@@ -69,6 +69,7 @@ class SourceFile;
 class SourceManager;
 class Stmt;
 class TypeRepr;
+struct FingerprintAndMembers;
 
 // There are a handful of cases where the swift compiler can introduce
 // counter-measurement noise via nondeterminism, especially via

include/swift/Driver/Action.h

@@ -173,17 +173,20 @@ class CompileJobAction : public JobAction {
   /// if 0 or >1 such inputs exist, return nullptr.
   const InputAction *findSingleSwiftInput() const {
     auto Inputs = getInputs();
-    auto isSwiftInput = [](const Action *A) -> const InputAction* {
-      if (auto const *S = dyn_cast<InputAction>(A))
-        return S->getType() == file_types::TY_Swift ? S : nullptr;
-      return nullptr;
-    };
-    const auto loc1 = std::find_if(Inputs.begin(), Inputs.end(), isSwiftInput);
-    if (loc1 == Inputs.end())
-      return nullptr; // none found
-    // Ensure uniqueness
-    const auto loc2 = std::find_if(loc1 + 1, Inputs.end(), isSwiftInput);
-    return loc2 == Inputs.end() ? dyn_cast<InputAction>(*loc1) : nullptr;
+    const InputAction *IA = nullptr;
+    for (auto const *I : Inputs) {
+      if (auto const *S = dyn_cast<InputAction>(I)) {
+        if (S->getType() == file_types::TY_Swift) {
+          if (IA == nullptr) {
+            IA = S;
+          } else {
+            // Already found one, two is too many.
+            return nullptr;
+          }
+        }
+      }
+    }
+    return IA;
   }
 };
 

include/swift/Driver/CoarseGrainedDependencyGraph.h

@@ -150,7 +150,7 @@ class CoarseGrainedDependencyGraphImpl {
   LoadResult loadFromString(const void *node, StringRef data);
   LoadResult loadFromPath(const void *node, StringRef path);
 
-  void addIndependentNode(const void *node) {
+  void registerJob(const void *node) {
     bool newlyInserted = Provides.insert({node, {}}).second;
     assert(newlyInserted && "node is already in graph");
     (void)newlyInserted;
@@ -260,8 +260,8 @@ class CoarseGrainedDependencyGraph : public CoarseGrainedDependencyGraphImpl {
   /// Adds \p node to the dependency graph without any connections.
   ///
   /// This can be used for new nodes that may be updated later.
-  void addIndependentNode(T node) {
-    return CoarseGrainedDependencyGraphImpl::addIndependentNode(
+  void registerJob(T node) {
+    return CoarseGrainedDependencyGraphImpl::registerJob(
         Traits::getAsVoidPointer(node));
   }
 

include/swift/Driver/Compilation.h

@@ -279,14 +279,17 @@ class Compilation {
   /// faster rebuilds.
   const bool EnableFineGrainedDependencies;
 
+  /// Is the parser recording token hashes for each type body?
+  const bool EnableTypeFingerprints;
+
   /// Helpful for debugging, but slows down the driver. So, only turn on when
   /// needed.
   const bool VerifyFineGrainedDependencyGraphAfterEveryImport;
   /// Helpful for debugging, but slows down the driver. So, only turn on when
   /// needed.
   const bool EmitFineGrainedDependencyDotFileAfterEveryImport;
 
-  /// Experiment with inter-file dependencies
+  /// Experiment with intrafile dependencies
   const bool FineGrainedDependenciesIncludeIntrafileOnes;
 
   /// Experiment with source-range-based dependencies
@@ -328,8 +331,10 @@ class Compilation {
               bool ShowDriverTimeCompilation = false,
               std::unique_ptr<UnifiedStatsReporter> Stats = nullptr,
               bool OnlyOneDependencyFile = false,
-              bool EnableFineGrainedDependencies =
-                LangOptions().EnableFineGrainedDependencies,
+              bool EnableFineGrainedDependencies
+                = LangOptions().EnableFineGrainedDependencies,
+              bool EnableTypeFingerprints =
+                LangOptions().EnableTypeFingerprints,
               bool VerifyFineGrainedDependencyGraphAfterEveryImport = false,
               bool EmitFineGrainedDependencyDotFileAfterEveryImport = false,
               bool FineGrainedDependenciesIncludeIntrafileOnes = false,
@@ -398,6 +403,8 @@ class Compilation {
     return EnableFineGrainedDependencies;
   }
 
+  bool getEnableTypeFingerprints() const { return EnableTypeFingerprints; }
+
   bool getVerifyFineGrainedDependencyGraphAfterEveryImport() const {
     return VerifyFineGrainedDependencyGraphAfterEveryImport;
   }