Delegate Codegen to IncrementalParser

Author: anutosh491

clang/include/clang/Interpreter/Interpreter.h

@@ -36,7 +36,6 @@ class ThreadSafeContext;
 namespace clang {
 
 class CompilerInstance;
-class CodeGenerator;
 class CXXRecordDecl;
 class Decl;
 class IncrementalExecutor;
@@ -109,10 +108,6 @@ class Interpreter {
   // printing happens, it's in an invalid state.
   Value LastValue;
 
-  /// When CodeGen is created the first llvm::Module gets cached in many places
-  /// and we must keep it alive.
-  std::unique_ptr<llvm::Module> CachedInCodeGenModule;
-
   /// Compiler instance performing the incremental compilation.
   std::unique_ptr<CompilerInstance> CI;
 
@@ -179,12 +174,6 @@ class Interpreter {
   llvm::Expected<Expr *> ExtractValueFromExpr(Expr *E);
   llvm::Expected<llvm::orc::ExecutorAddr> CompileDtorCall(CXXRecordDecl *CXXRD);
 
-  CodeGenerator *getCodeGen(IncrementalAction *Action = nullptr) const;
-  std::unique_ptr<llvm::Module> GenModule(IncrementalAction *Action = nullptr);
-  PartialTranslationUnit &RegisterPTU(TranslationUnitDecl *TU,
-                                      std::unique_ptr<llvm::Module> M = {},
-                                      IncrementalAction *Action = nullptr);
-
   // A cache for the compiled destructors used to for de-allocation of managed
   // clang::Values.
   llvm::DenseMap<CXXRecordDecl *, llvm::orc::ExecutorAddr> Dtors;

clang/lib/Interpreter/CMakeLists.txt

@@ -22,6 +22,7 @@ endif()
 add_clang_library(clangInterpreter
   DeviceOffload.cpp
   CodeCompletion.cpp
+  IncrementalAction.cpp
   IncrementalExecutor.cpp
   IncrementalParser.cpp
   Interpreter.cpp

clang/lib/Interpreter/DeviceOffload.cpp

@@ -26,10 +26,10 @@ namespace clang {
 
 IncrementalCUDADeviceParser::IncrementalCUDADeviceParser(
     std::unique_ptr<CompilerInstance> DeviceInstance,
-    CompilerInstance &HostInstance,
+    CompilerInstance &HostInstance, IncrementalAction *DeviceAct,
     llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS,
-    llvm::Error &Err, const std::list<PartialTranslationUnit> &PTUs)
-    : IncrementalParser(*DeviceInstance, Err), PTUs(PTUs), VFS(FS),
+    llvm::Error &Err, std::list<PartialTranslationUnit> &PTUs)
+    : IncrementalParser(*DeviceInstance, DeviceAct, Err, PTUs), VFS(FS),
       CodeGenOpts(HostInstance.getCodeGenOpts()),
       TargetOpts(DeviceInstance->getTargetOpts()) {
   if (Err)

clang/lib/Interpreter/DeviceOffload.h

@@ -22,16 +22,16 @@ struct PartialTranslationUnit;
 class CompilerInstance;
 class CodeGenOptions;
 class TargetOptions;
+class IncrementalAction;
 
 class IncrementalCUDADeviceParser : public IncrementalParser {
-  const std::list<PartialTranslationUnit> &PTUs;
 
 public:
   IncrementalCUDADeviceParser(
       std::unique_ptr<CompilerInstance> DeviceInstance,
-      CompilerInstance &HostInstance,
+      CompilerInstance &HostInstance, IncrementalAction *DeviceAct,
       llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> VFS,
-      llvm::Error &Err, const std::list<PartialTranslationUnit> &PTUs);
+      llvm::Error &Err, std::list<PartialTranslationUnit> &PTUs);
 
   // Generate PTX for the last PTU.
   llvm::Expected<llvm::StringRef> GeneratePTX();

clang/lib/Interpreter/IncrementalAction.cpp

@@ -0,0 +1,110 @@
+//===--- IncrementalAction.h - Incremental Frontend Action -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "IncrementalAction.h"
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/CodeGen/CodeGenAction.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendOptions.h"
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Interpreter/Interpreter.h"
+#include "clang/Sema/Sema.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+
+namespace clang {
+IncrementalAction::IncrementalAction(CompilerInstance &CI,
+                                     llvm::LLVMContext &LLVMCtx,
+                                     llvm::Error &Err, Interpreter &I,
+                                     std::unique_ptr<ASTConsumer> Consumer)
+    : WrapperFrontendAction([&]() {
+        llvm::ErrorAsOutParameter EAO(&Err);
+        std::unique_ptr<FrontendAction> Act;
+        switch (CI.getFrontendOpts().ProgramAction) {
+        default:
+          Err = llvm::createStringError(
+              std::errc::state_not_recoverable,
+              "Driver initialization failed. "
+              "Incremental mode for action %d is not supported",
+              CI.getFrontendOpts().ProgramAction);
+          return Act;
+        case frontend::ASTDump:
+        case frontend::ASTPrint:
+        case frontend::ParseSyntaxOnly:
+          Act = CreateFrontendAction(CI);
+          break;
+        case frontend::PluginAction:
+        case frontend::EmitAssembly:
+        case frontend::EmitBC:
+        case frontend::EmitObj:
+        case frontend::PrintPreprocessedInput:
+        case frontend::EmitLLVMOnly:
+          Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
+          break;
+        }
+        return Act;
+      }()),
+      Interp(I), Consumer(std::move(Consumer)) {}
+
+std::unique_ptr<ASTConsumer>
+IncrementalAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
+  std::unique_ptr<ASTConsumer> C =
+      WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+
+  if (Consumer) {
+    std::vector<std::unique_ptr<ASTConsumer>> Cs;
+    Cs.push_back(std::move(Consumer));
+    Cs.push_back(std::move(C));
+    return std::make_unique<MultiplexConsumer>(std::move(Cs));
+  }
+
+  return std::make_unique<InProcessPrintingASTConsumer>(std::move(C), Interp);
+}
+
+void IncrementalAction::ExecuteAction() {
+  WrapperFrontendAction::ExecuteAction();
+  getCompilerInstance().getSema().CurContext = nullptr;
+}
+
+void IncrementalAction::EndSourceFile() {
+  if (IsTerminating && getWrapped())
+    WrapperFrontendAction::EndSourceFile();
+}
+
+void IncrementalAction::FinalizeAction() {
+  assert(!IsTerminating && "Already finalized!");
+  IsTerminating = true;
+  EndSourceFile();
+}
+
+InProcessPrintingASTConsumer::InProcessPrintingASTConsumer(
+    std::unique_ptr<ASTConsumer> C, Interpreter &I)
+    : MultiplexConsumer(std::move(C)), Interp(I) {}
+
+bool InProcessPrintingASTConsumer::HandleTopLevelDecl(DeclGroupRef DGR) {
+  if (DGR.isNull())
+    return true;
+
+  for (Decl *D : DGR)
+    if (auto *TLSD = llvm::dyn_cast<TopLevelStmtDecl>(D))
+      if (TLSD && TLSD->isSemiMissing()) {
+        auto ExprOrErr =
+            Interp.ExtractValueFromExpr(cast<Expr>(TLSD->getStmt()));
+        if (llvm::Error E = ExprOrErr.takeError()) {
+          llvm::logAllUnhandledErrors(std::move(E), llvm::errs(),
+                                      "Value printing failed: ");
+          return false; // abort parsing
+        }
+        TLSD->setStmt(*ExprOrErr);
+      }
+
+  return MultiplexConsumer::HandleTopLevelDecl(DGR);
+}
+
+} // namespace clang

clang/lib/Interpreter/IncrementalAction.h

@@ -0,0 +1,68 @@
+//===--- IncrementalAction.h - Incremental Frontend Action -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_INCREMENTALACTION_H
+#define LLVM_CLANG_INTERPRETER_INCREMENTALACTION_H
+
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/MultiplexConsumer.h"
+
+namespace clang {
+
+class Interpreter;
+
+/// A custom action enabling the incremental processing functionality.
+///
+/// The usual \p FrontendAction expects one call to ExecuteAction and once it
+/// sees a call to \p EndSourceFile it deletes some of the important objects
+/// such as \p Preprocessor and \p Sema assuming no further input will come.
+///
+/// \p IncrementalAction ensures it keep its underlying action's objects alive
+/// as long as the \p IncrementalParser needs them.
+///
+class IncrementalAction : public WrapperFrontendAction {
+private:
+  bool IsTerminating = false;
+  Interpreter &Interp;
+  std::unique_ptr<ASTConsumer> Consumer;
+
+public:
+  IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
+                    llvm::Error &Err, Interpreter &I,
+                    std::unique_ptr<ASTConsumer> Consumer = nullptr);
+
+  FrontendAction *getWrapped() const { return WrappedAction.get(); }
+
+  TranslationUnitKind getTranslationUnitKind() override {
+    return TU_Incremental;
+  }
+
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override;
+
+  void ExecuteAction() override;
+
+  // Do not terminate after processing the input. This allows us to keep various
+  // clang objects alive and to incrementally grow the current TU.
+  void EndSourceFile() override;
+
+  void FinalizeAction();
+};
+
+class InProcessPrintingASTConsumer final : public MultiplexConsumer {
+  Interpreter &Interp;
+
+public:
+  InProcessPrintingASTConsumer(std::unique_ptr<ASTConsumer> C, Interpreter &I);
+
+  bool HandleTopLevelDecl(DeclGroupRef DGR) override;
+};
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_INTERPRETER_INCREMENTALACTION_H

clang/lib/Interpreter/IncrementalParser.cpp

@@ -11,27 +11,36 @@
 //===----------------------------------------------------------------------===//
 
 #include "IncrementalParser.h"
+#include "IncrementalAction.h"
 
 #include "clang/AST/DeclContextInternals.h"
+#include "clang/CodeGen/CodeGenAction.h"
+#include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendAction.h"
 #include "clang/Interpreter/PartialTranslationUnit.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Sema/Sema.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/Error.h"
 
 #include <sstream>
 
+#define DEBUG_TYPE "clang-repl"
+
 namespace clang {
 
 // IncrementalParser::IncrementalParser() {}
 
 IncrementalParser::IncrementalParser(CompilerInstance &Instance,
-                                     llvm::Error &Err)
-    : S(Instance.getSema()) {
+                                     IncrementalAction *Act, llvm::Error &Err,
+                                     std::list<PartialTranslationUnit> &PTUs)
+    : S(Instance.getSema()), CI(Instance), Act(Act), PTUs(PTUs) {
   llvm::ErrorAsOutParameter EAO(&Err);
   Consumer = &S.getASTConsumer();
-  P.reset(new Parser(S.getPreprocessor(), S, /*SkipBodies=*/false));
+  P = std::make_unique<Parser>(S.getPreprocessor(), S, /*SkipBodies=*/false);
   P->Initialize();
 }
 
@@ -185,4 +194,57 @@ void IncrementalParser::CleanUpPTU(TranslationUnitDecl *MostRecentTU) {
   }
 }
 
+PartialTranslationUnit &
+IncrementalParser::RegisterPTU(TranslationUnitDecl *TU,
+                               std::unique_ptr<llvm::Module> M /*={}*/) {
+  PTUs.emplace_back(PartialTranslationUnit());
+  PartialTranslationUnit &LastPTU = PTUs.back();
+  LastPTU.TUPart = TU;
+
+  if (!M)
+    M = GenModule();
+
+  assert((!getCodeGen() || M) && "Must have a llvm::Module at this point");
+
+  LastPTU.TheModule = std::move(M);
+  LLVM_DEBUG(llvm::dbgs() << "compile-ptu " << PTUs.size() - 1
+                          << ": [TU=" << LastPTU.TUPart);
+  if (LastPTU.TheModule)
+    LLVM_DEBUG(llvm::dbgs() << ", M=" << LastPTU.TheModule.get() << " ("
+                            << LastPTU.TheModule->getName() << ")");
+  LLVM_DEBUG(llvm::dbgs() << "]\n");
+  return LastPTU;
+}
+
+std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
+  static unsigned ID = 0;
+  if (CodeGenerator *CG = getCodeGen()) {
+    // Clang's CodeGen is designed to work with a single llvm::Module. In many
+    // cases for convenience various CodeGen parts have a reference to the
+    // llvm::Module (TheModule or Module) which does not change when a new
+    // module is pushed. However, the execution engine wants to take ownership
+    // of the module which does not map well to CodeGen's design. To work this
+    // around we created an empty module to make CodeGen happy. We should make
+    // sure it always stays empty.
+    assert(((!CachedInCodeGenModule ||
+             !CI.getPreprocessorOpts().Includes.empty()) ||
+            (CachedInCodeGenModule->empty() &&
+             CachedInCodeGenModule->global_empty() &&
+             CachedInCodeGenModule->alias_empty() &&
+             CachedInCodeGenModule->ifunc_empty())) &&
+           "CodeGen wrote to a readonly module");
+    std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
+    CG->StartModule("incr_module_" + std::to_string(ID++), M->getContext());
+    return M;
+  }
+  return nullptr;
+}
+
+CodeGenerator *IncrementalParser::getCodeGen() const {
+  FrontendAction *WrappedAct = Act->getWrapped();
+  if (!WrappedAct->hasIRSupport())
+    return nullptr;
+  return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
+}
+
 } // end namespace clang