[CAS] Add caching diagnostic processor

Add a CachedDiagnosticsProcessor that is a DiagConsumer can capture all
the diagnostics during a compilation, serialized them into CAS with a
format that can be replayed without re-compiling.

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -494,6 +494,9 @@ ERROR(error_create_cas, none, "failed to create CAS '%0' (%1)", (StringRef, Stri
 ERROR(error_invalid_cas_id, none, "invalid CASID '%0' (%1)", (StringRef, StringRef))
 ERROR(error_cas, none, "CAS error encountered: %0", (StringRef))
 
+ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef))
+ERROR(error_replay_cached_diag, none, "failed to replay cached diagnostics: %0", (StringRef))
+
 // Dependency Verifier Diagnostics
 ERROR(missing_member_dependency,none,
       "expected "

include/swift/Frontend/CachedDiagnostics.h

@@ -0,0 +1,56 @@
+//===--- CachedDiagnostics.h - Cached Diagnostics ---------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the CachedDiagnosticConsumer class, which
+//  caches the diagnostics which can be replayed with other DiagnosticConumers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_CACHEDDIAGNOSTICS_H
+#define SWIFT_CACHEDDIAGNOSTICS_H
+
+#include "llvm/Support/Error.h"
+
+namespace swift {
+
+class CompilerInstance;
+class DiagnosticEngine;
+class SourceManager;
+class FrontendInputsAndOutputs;
+
+class CachingDiagnosticsProcessor {
+public:
+  CachingDiagnosticsProcessor(CompilerInstance &Instance);
+  ~CachingDiagnosticsProcessor();
+
+  /// Start capturing all the diagnostics from DiagnosticsEngine.
+  void startCaptureDiagnostics();
+  /// End capturing all the diagnostics from DiagnosticsEngine.
+  void endCaptureDiagnostics();
+
+  /// Disable catpure.
+  void setDisableCapture();
+
+  /// Emit serialized diagnostics into output stream.
+  llvm::Error serializeEmittedDiagnostics(llvm::raw_ostream &os);
+
+  /// Used to replay the previously cached diagnostics, after a cache hit.
+  llvm::Error replayCachedDiagnostics(llvm::StringRef Buffer);
+
+private:
+  class Implementation;
+  Implementation& Impl;
+};
+
+}
+
+#endif

include/swift/Frontend/Frontend.h

@@ -30,6 +30,7 @@
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/ClangImporter/ClangImporter.h"
+#include "swift/Frontend/CachedDiagnostics.h"
 #include "swift/Frontend/DiagnosticVerifier.h"
 #include "swift/Frontend/FrontendOptions.h"
 #include "swift/Frontend/ModuleInterfaceSupport.h"
@@ -462,6 +463,7 @@ class CompilerInstance {
   std::unique_ptr<ASTContext> Context;
   std::unique_ptr<Lowering::TypeConverter> TheSILTypes;
   std::unique_ptr<DiagnosticVerifier> DiagVerifier;
+  std::unique_ptr<CachingDiagnosticsProcessor> CDP;
 
   /// A cache describing the set of inter-module dependencies that have been queried.
   /// Null if not present.
@@ -548,6 +550,9 @@ class CompilerInstance {
   Optional<llvm::cas::ObjectRef> getCompilerBaseKey() const {
     return CompileJobBaseKey;
   }
+  CachingDiagnosticsProcessor *getCachingDiagnosticsProcessor() const {
+    return CDP.get();
+  }
 
   ASTContext &getASTContext() { return *Context; }
   const ASTContext &getASTContext() const { return *Context; }
@@ -677,6 +682,7 @@ class CompilerInstance {
   void setupDependencyTrackerIfNeeded();
   bool setupCASIfNeeded(ArrayRef<const char *> Args);
   void setupOutputBackend();
+  void setupCachingDiagnosticsProcessorIfNeeded();
 
   /// \return false if successful, true on error.
   bool setupDiagnosticVerifierIfNeeded();

lib/DriverTool/swift_cache_tool_main.cpp

@@ -42,7 +42,8 @@ enum class SwiftCacheToolAction {
   Invalid,
   PrintBaseKey,
   PrintOutputKeys,
-  ValidateOutputs
+  ValidateOutputs,
+  RenderDiags
 };
 
 struct OutputEntry {
@@ -134,11 +135,13 @@ class SwiftCacheToolInvocation {
               .Case("print-base-key", SwiftCacheToolAction::PrintBaseKey)
               .Case("print-output-keys", SwiftCacheToolAction::PrintOutputKeys)
               .Case("validate-outputs", SwiftCacheToolAction::ValidateOutputs)
+              .Case("render-diags", SwiftCacheToolAction::RenderDiags)
               .Default(SwiftCacheToolAction::Invalid);
 
     if (ActionKind == SwiftCacheToolAction::Invalid) {
-      llvm::errs() << "Invalid option specified for -cache-tool-action: "
-                   << "use print-base-key|print-output-keys|validate-outputs\n";
+      llvm::errs()
+          << "Invalid option specified for -cache-tool-action: "
+          << "print-base-key|print-output-keys|validate-outputs|render-diags\n";
       return 1;
     }
 
@@ -153,6 +156,8 @@ class SwiftCacheToolInvocation {
       return printOutputKeys();
     case SwiftCacheToolAction::ValidateOutputs:
       return validateOutputs();
+    case SwiftCacheToolAction::RenderDiags:
+      return renderDiags();
     case SwiftCacheToolAction::Invalid:
       return 0; // No action. Probably just print help. Return.
     }
@@ -202,6 +207,12 @@ class SwiftCacheToolInvocation {
       return true;
     }
 
+    // Disable diagnostic caching from this fake instance.
+    if (auto *CDP = Instance.getCachingDiagnosticsProcessor()) {
+      CDP->endCaptureDiagnostics();
+      CDP->setDisableCapture();
+    }
+
     return false;
   }
 
@@ -233,6 +244,7 @@ class SwiftCacheToolInvocation {
 
   int printOutputKeys();
   int validateOutputs();
+  int renderDiags();
 };
 
 } // end anonymous namespace
@@ -283,6 +295,10 @@ int SwiftCacheToolInvocation::printOutputKeys() {
       Invocation.getFrontendOptions().InputsAndOutputs.getAllInputs(),
       addFromInputFile);
 
+  // Add diagnostics file.
+  addOutputKey("<cached-diagnostics>", file_types::ID::TY_CachedDiagnostics,
+               "<cached-diagnostics>");
+
   if (hasError)
     return 1;
 
@@ -301,6 +317,26 @@ int SwiftCacheToolInvocation::printOutputKeys() {
   return 0;
 }
 
+static llvm::Expected<llvm::json::Array>
+readOutputEntriesFromFile(StringRef Path) {
+  auto JSONContent = llvm::MemoryBuffer::getFile(Path);
+  if (!JSONContent)
+    return llvm::createStringError(JSONContent.getError(),
+                                   "failed to read input file");
+
+  auto JSONValue = llvm::json::parse((*JSONContent)->getBuffer());
+  if (!JSONValue)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "failed to parse input file as JSON");
+
+  auto Keys = JSONValue->getAsArray();
+  if (!Keys)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid JSON format for input file");
+
+  return *Keys;
+}
+
 int SwiftCacheToolInvocation::validateOutputs() {
   auto DB = llvm::cas::createOnDiskUnifiedCASDatabases(CASPath);
   if (!DB)
@@ -310,22 +346,10 @@ int SwiftCacheToolInvocation::validateOutputs() {
   Instance.getDiags().addConsumer(PDC);
 
   auto validateCacheKeysFromFile = [&](const std::string &Path) {
-    auto JSONContent = llvm::MemoryBuffer::getFile(Path);
-    if (!JSONContent) {
-      llvm::errs() << "failed to read " << Path << ": "
-                   << JSONContent.getError().message() << "\n";
-      return true;
-    }
-    auto JSONValue = llvm::json::parse((*JSONContent)->getBuffer());
-    if (!JSONValue) {
-      llvm::errs() << "failed to parse " << Path << ": "
-                   << toString(JSONValue.takeError()) << "\n";
-      return true;
-    }
-
-    auto Keys = JSONValue->getAsArray();
+    auto Keys = readOutputEntriesFromFile(Path);
     if (!Keys) {
-      llvm::errs() << "invalid keys format in " << Path << "\n";
+      llvm::errs() << "cannot read file " << Path << ": "
+                   << toString(Keys.takeError()) << "\n";
       return true;
     }
 
@@ -350,6 +374,51 @@ int SwiftCacheToolInvocation::validateOutputs() {
   return llvm::any_of(Inputs, validateCacheKeysFromFile);
 }
 
+int SwiftCacheToolInvocation::renderDiags() {
+  if (setupCompiler())
+    return 1;
+
+  auto *CDP = Instance.getCachingDiagnosticsProcessor();
+  if (!CDP) {
+    llvm::errs() << "provided commandline doesn't support cached diagnostics\n";
+    return 1;
+  }
+
+  auto renderDiagsFromFile = [&](const std::string &Path) {
+    auto Keys = readOutputEntriesFromFile(Path);
+    if (!Keys) {
+      llvm::errs() << "cannot read file " << Path << ": "
+                   << toString(Keys.takeError()) << "\n";
+      return true;
+    }
+
+    for (const auto& Entry : *Keys) {
+      if (auto *Obj = Entry.getAsObject()) {
+        if (auto Kind = Obj->getString("OutputKind")) {
+          if (*Kind != "cached-diagnostics")
+            continue;
+        }
+        if (auto Key = Obj->getString("CacheKey")) {
+          if (auto Buffer = loadCachedCompileResultFromCacheKey(
+                  Instance.getObjectStore(), Instance.getActionCache(),
+                  Instance.getDiags(), *Key)) {
+            if (auto E = CDP->replayCachedDiagnostics(Buffer->getBuffer())) {
+              llvm::errs() << "failed to replay cache: "
+                           << toString(std::move(E)) << "\n";
+              return true;
+            }
+            return false;
+          }
+        }
+      }
+    }
+    llvm::errs() << "cannot locate cached diagnostics in file\n";
+    return true;
+  };
+
+  return llvm::any_of(Inputs, renderDiagsFromFile);
+}
+
 int swift_cache_tool_main(ArrayRef<const char *> Args, const char *Argv0,
                           void *MainAddr) {
   INITIALIZE_LLVM();

lib/Frontend/CMakeLists.txt

@@ -3,6 +3,7 @@ add_swift_host_library(swiftFrontend STATIC
   ArgsToFrontendInputsConverter.cpp
   ArgsToFrontendOptionsConverter.cpp
   ArgsToFrontendOutputsConverter.cpp
+  CachedDiagnostics.cpp
   CachingUtils.cpp
   CompileJobCacheKey.cpp
   CompilerInvocation.cpp