[ClangRepl] Reland Semanic Code Completion (#75556)

capfredf · web-flow · commit 35b366ace73d · 2023-12-19T22:53:02.000+02:00
This patch contains changes from 002d471, in addition to a bug fix that added a virtual destructor to `CompletionContextHandler` The original changes in the orginal commit piggybacks on clang's semantic modules to enable semantic completion. In particular, we use `CodeCompletionContext` to differentiate two types of code completion. We also extract the relevant type information from it.
diff --git a/clang/include/clang/Interpreter/CodeCompletion.h b/clang/include/clang/Interpreter/CodeCompletion.h
@@ -23,8 +23,27 @@ namespace clang {
 class CodeCompletionResult;
 class CompilerInstance;
 
-void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
-                  unsigned Line, unsigned Col, const CompilerInstance *ParentCI,
-                  std::vector<std::string> &CCResults);
+struct ReplCodeCompleter {
+  ReplCodeCompleter() = default;
+  std::string Prefix;
+
+  /// \param InterpCI [in] The compiler instance that is used to trigger code
+  /// completion
+
+  /// \param Content [in] The string where code completion is triggered.
+
+  /// \param Line [in] The line number of the code completion point.
+
+  /// \param Col [in] The column number of the code completion point.
+
+  /// \param ParentCI [in] The running interpreter compiler instance that
+  /// provides ASTContexts.
+
+  /// \param CCResults [out] The completion results.
+  void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
+                    unsigned Line, unsigned Col,
+                    const CompilerInstance *ParentCI,
+                    std::vector<std::string> &CCResults);
+};
 } // namespace clang
 #endif
diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h
@@ -101,6 +101,7 @@ class Interpreter {
   const ASTContext &getASTContext() const;
   ASTContext &getASTContext();
   const CompilerInstance *getCompilerInstance() const;
+  CompilerInstance *getCompilerInstance();
   llvm::Expected<llvm::orc::LLJIT &> getExecutionEngine();
 
   llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);
diff --git a/clang/lib/Interpreter/CodeCompletion.cpp b/clang/lib/Interpreter/CodeCompletion.cpp
@@ -12,6 +12,7 @@
 
 #include "clang/Interpreter/CodeCompletion.h"
 #include "clang/AST/ASTImporter.h"
+#include "clang/AST/DeclLookups.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/Basic/IdentifierTable.h"
@@ -23,6 +24,8 @@
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/CodeCompleteOptions.h"
 #include "clang/Sema/Sema.h"
+#include "llvm/Support/Debug.h"
+#define DEBUG_TYPE "REPLCC"
 
 namespace clang {
 
@@ -39,11 +42,15 @@ clang::CodeCompleteOptions getClangCompleteOpts() {
 
 class ReplCompletionConsumer : public CodeCompleteConsumer {
 public:
-  ReplCompletionConsumer(std::vector<std::string> &Results)
+  ReplCompletionConsumer(std::vector<std::string> &Results,
+                         ReplCodeCompleter &CC)
       : CodeCompleteConsumer(getClangCompleteOpts()),
         CCAllocator(std::make_shared<GlobalCodeCompletionAllocator>()),
-        CCTUInfo(CCAllocator), Results(Results){};
+        CCTUInfo(CCAllocator), Results(Results), CC(CC) {}
 
+  // The entry of handling code completion. When the function is called, we
+  // create a `Context`-based handler (see classes defined below) to handle each
+  // completion result.
   void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context,
                                   CodeCompletionResult *InResults,
                                   unsigned NumResults) final;
@@ -56,26 +63,147 @@ class ReplCompletionConsumer : public CodeCompleteConsumer {
   std::shared_ptr<GlobalCodeCompletionAllocator> CCAllocator;
   CodeCompletionTUInfo CCTUInfo;
   std::vector<std::string> &Results;
+  ReplCodeCompleter &CC;
+};
+
+/// The class CompletionContextHandler contains four interfaces, each of
+/// which handles one type of completion result.
+/// Its derived classes are used to create concrete handlers based on
+/// \c CodeCompletionContext.
+class CompletionContextHandler {
+protected:
+  CodeCompletionContext CCC;
+  std::vector<std::string> &Results;
+
+private:
+  Sema &S;
+
+public:
+  CompletionContextHandler(Sema &S, CodeCompletionContext CCC,
+                           std::vector<std::string> &Results)
+      : CCC(CCC), Results(Results), S(S) {}
+
+  virtual ~CompletionContextHandler() = default;
+  /// Converts a Declaration completion result to a completion string, and then
+  /// stores it in Results.
+  virtual void handleDeclaration(const CodeCompletionResult &Result) {
+    auto PreferredType = CCC.getPreferredType();
+    if (PreferredType.isNull()) {
+      Results.push_back(Result.Declaration->getName().str());
+      return;
+    }
+
+    if (auto *VD = dyn_cast<VarDecl>(Result.Declaration)) {
+      auto ArgumentType = VD->getType();
+      if (PreferredType->isReferenceType()) {
+        QualType RT = PreferredType->castAs<ReferenceType>()->getPointeeType();
+        Sema::ReferenceConversions RefConv;
+        Sema::ReferenceCompareResult RefRelationship =
+            S.CompareReferenceRelationship(SourceLocation(), RT, ArgumentType,
+                                           &RefConv);
+        switch (RefRelationship) {
+        case Sema::Ref_Compatible:
+        case Sema::Ref_Related:
+          Results.push_back(VD->getName().str());
+          break;
+        case Sema::Ref_Incompatible:
+          break;
+        }
+      } else if (S.Context.hasSameType(ArgumentType, PreferredType)) {
+        Results.push_back(VD->getName().str());
+      }
+    }
+  }
+
+  /// Converts a Keyword completion result to a completion string, and then
+  /// stores it in Results.
+  virtual void handleKeyword(const CodeCompletionResult &Result) {
+    auto Prefix = S.getPreprocessor().getCodeCompletionFilter();
+    // Add keyword to the completion results only if we are in a type-aware
+    // situation.
+    if (!CCC.getBaseType().isNull() || !CCC.getPreferredType().isNull())
+      return;
+    if (StringRef(Result.Keyword).startswith(Prefix))
+      Results.push_back(Result.Keyword);
+  }
+
+  /// Converts a Pattern completion result to a completion string, and then
+  /// stores it in Results.
+  virtual void handlePattern(const CodeCompletionResult &Result) {}
+
+  /// Converts a Macro completion result to a completion string, and then stores
+  /// it in Results.
+  virtual void handleMacro(const CodeCompletionResult &Result) {}
+};
+
+class DotMemberAccessHandler : public CompletionContextHandler {
+public:
+  DotMemberAccessHandler(Sema &S, CodeCompletionContext CCC,
+                         std::vector<std::string> &Results)
+      : CompletionContextHandler(S, CCC, Results) {}
+  void handleDeclaration(const CodeCompletionResult &Result) override {
+    auto *ID = Result.Declaration->getIdentifier();
+    if (!ID)
+      return;
+    if (!isa<CXXMethodDecl>(Result.Declaration))
+      return;
+    const auto *Fun = cast<CXXMethodDecl>(Result.Declaration);
+    if (Fun->getParent()->getCanonicalDecl() ==
+        CCC.getBaseType()->getAsCXXRecordDecl()->getCanonicalDecl()) {
+      LLVM_DEBUG(llvm::dbgs() << "[In HandleCodeCompleteDOT] Name : "
+                              << ID->getName() << "\n");
+      Results.push_back(ID->getName().str());
+    }
+  }
+
+  void handleKeyword(const CodeCompletionResult &Result) override {}
 };
 
 void ReplCompletionConsumer::ProcessCodeCompleteResults(
     class Sema &S, CodeCompletionContext Context,
     CodeCompletionResult *InResults, unsigned NumResults) {
-  for (unsigned I = 0; I < NumResults; ++I) {
+
+  auto Prefix = S.getPreprocessor().getCodeCompletionFilter();
+  CC.Prefix = Prefix;
+
+  std::unique_ptr<CompletionContextHandler> CCH;
+
+  // initialize fine-grained code completion handler based on the code
+  // completion context.
+  switch (Context.getKind()) {
+  case CodeCompletionContext::CCC_DotMemberAccess:
+    CCH.reset(new DotMemberAccessHandler(S, Context, this->Results));
+    break;
+  default:
+    CCH.reset(new CompletionContextHandler(S, Context, this->Results));
+  };
+
+  for (unsigned I = 0; I < NumResults; I++) {
     auto &Result = InResults[I];
     switch (Result.Kind) {
     case CodeCompletionResult::RK_Declaration:
-      if (auto *ID = Result.Declaration->getIdentifier()) {
-        Results.push_back(ID->getName().str());
+      if (Result.Hidden) {
+        break;
+      }
+      if (!Result.Declaration->getDeclName().isIdentifier() ||
+          !Result.Declaration->getName().startswith(Prefix)) {
+        break;
       }
+      CCH->handleDeclaration(Result);
       break;
     case CodeCompletionResult::RK_Keyword:
-      Results.push_back(Result.Keyword);
+      CCH->handleKeyword(Result);
       break;
-    default:
+    case CodeCompletionResult::RK_Macro:
+      CCH->handleMacro(Result);
+      break;
+    case CodeCompletionResult::RK_Pattern:
+      CCH->handlePattern(Result);
       break;
     }
   }
+
+  std::sort(Results.begin(), Results.end());
 }
 
 class IncrementalSyntaxOnlyAction : public SyntaxOnlyAction {
@@ -118,6 +246,16 @@ void IncrementalSyntaxOnlyAction::ExecuteAction() {
   CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage(
       true);
 
+  // Load all external decls into current context. Under the hood, it calls
+  // ExternalSource::completeVisibleDeclsMap, which make all decls on the redecl
+  // chain visible.
+  //
+  // This is crucial to code completion on dot members, since a bound variable
+  // before "." would be otherwise treated out-of-scope.
+  //
+  // clang-repl> Foo f1;
+  // clang-repl> f1.<tab>
+  CI.getASTContext().getTranslationUnitDecl()->lookups();
   SyntaxOnlyAction::ExecuteAction();
 }
 
@@ -134,6 +272,7 @@ ExternalSource::ExternalSource(ASTContext &ChildASTCtxt, FileManager &ChildFM,
 
 bool ExternalSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
                                                     DeclarationName Name) {
+
   IdentifierTable &ParentIdTable = ParentASTCtxt.Idents;
 
   auto ParentDeclName =
@@ -159,29 +298,67 @@ void ExternalSource::completeVisibleDeclsMap(
   for (auto *DeclCtxt = ParentTUDeclCtxt; DeclCtxt != nullptr;
        DeclCtxt = DeclCtxt->getPreviousDecl()) {
     for (auto &IDeclContext : DeclCtxt->decls()) {
-      if (NamedDecl *Decl = llvm::dyn_cast<NamedDecl>(IDeclContext)) {
-        if (auto DeclOrErr = Importer->Import(Decl)) {
-          if (NamedDecl *importedNamedDecl =
-                  llvm::dyn_cast<NamedDecl>(*DeclOrErr)) {
-            SetExternalVisibleDeclsForName(ChildDeclContext,
-                                           importedNamedDecl->getDeclName(),
-                                           importedNamedDecl);
-          }
-
-        } else {
-          llvm::consumeError(DeclOrErr.takeError());
-        }
+      if (!llvm::isa<NamedDecl>(IDeclContext))
+        continue;
+
+      NamedDecl *Decl = llvm::cast<NamedDecl>(IDeclContext);
+
+      auto DeclOrErr = Importer->Import(Decl);
+      if (!DeclOrErr) {
+        // if an error happens, it usually means the decl has already been
+        // imported or the decl is a result of a failed import.  But in our
+        // case, every import is fresh each time code completion is
+        // triggered. So Import usually doesn't fail. If it does, it just means
+        // the related decl can't be used in code completion and we can safely
+        // drop it.
+        llvm::consumeError(DeclOrErr.takeError());
+        continue;
       }
+
+      if (!llvm::isa<NamedDecl>(*DeclOrErr))
+        continue;
+
+      NamedDecl *importedNamedDecl = llvm::cast<NamedDecl>(*DeclOrErr);
+
+      SetExternalVisibleDeclsForName(ChildDeclContext,
+                                     importedNamedDecl->getDeclName(),
+                                     importedNamedDecl);
+
+      if (!llvm::isa<CXXRecordDecl>(importedNamedDecl))
+        continue;
+
+      auto *Record = llvm::cast<CXXRecordDecl>(importedNamedDecl);
+
+      if (auto Err = Importer->ImportDefinition(Decl)) {
+        // the same as above
+        consumeError(std::move(Err));
+        continue;
+      }
+
+      Record->setHasLoadedFieldsFromExternalStorage(true);
+      LLVM_DEBUG(llvm::dbgs()
+                 << "\nCXXRecrod : " << Record->getName() << " size(methods): "
+                 << std::distance(Record->method_begin(), Record->method_end())
+                 << " has def?:  " << Record->hasDefinition()
+                 << " # (methods): "
+                 << std::distance(Record->getDefinition()->method_begin(),
+                                  Record->getDefinition()->method_end())
+                 << "\n");
+      for (auto *Meth : Record->methods())
+        SetExternalVisibleDeclsForName(ChildDeclContext, Meth->getDeclName(),
+                                       Meth);
     }
     ChildDeclContext->setHasExternalLexicalStorage(false);
   }
 }
 
-void codeComplete(CompilerInstance *InterpCI, llvm::StringRef Content,
-                  unsigned Line, unsigned Col, const CompilerInstance *ParentCI,
-                  std::vector<std::string> &CCResults) {
+void ReplCodeCompleter::codeComplete(CompilerInstance *InterpCI,
+                                     llvm::StringRef Content, unsigned Line,
+                                     unsigned Col,
+                                     const CompilerInstance *ParentCI,
+                                     std::vector<std::string> &CCResults) {
   auto DiagOpts = DiagnosticOptions();
-  auto consumer = ReplCompletionConsumer(CCResults);
+  auto consumer = ReplCompletionConsumer(CCResults, *this);
 
   auto diag = InterpCI->getDiagnosticsPtr();
   std::unique_ptr<ASTUnit> AU(ASTUnit::LoadFromCompilerInvocationAction(
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
@@ -319,6 +319,10 @@ const CompilerInstance *Interpreter::getCompilerInstance() const {
   return IncrParser->getCI();
 }
 
+CompilerInstance *Interpreter::getCompilerInstance() {
+  return IncrParser->getCI();
+}
+
 llvm::Expected<llvm::orc::LLJIT &> Interpreter::getExecutionEngine() {
   if (!IncrExecutor) {
     if (auto Err = CreateExecutor())
diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp
@@ -15,6 +15,8 @@
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Interpreter/CodeCompletion.h"
 #include "clang/Interpreter/Interpreter.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Sema/Sema.h"
 
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/LineEditor/LineEditor.h"
@@ -123,22 +125,14 @@ ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos,
 
     return {};
   }
-
-  codeComplete(
-      const_cast<clang::CompilerInstance *>((*Interp)->getCompilerInstance()),
-      Buffer, Lines, Pos + 1, MainInterp.getCompilerInstance(), Results);
-
-  size_t space_pos = Buffer.rfind(" ");
-  llvm::StringRef Prefix;
-  if (space_pos == llvm::StringRef::npos) {
-    Prefix = Buffer;
-  } else {
-    Prefix = Buffer.substr(space_pos + 1);
-  }
-
+  auto *MainCI = (*Interp)->getCompilerInstance();
+  auto CC = clang::ReplCodeCompleter();
+  CC.codeComplete(MainCI, Buffer, Lines, Pos + 1,
+                  MainInterp.getCompilerInstance(), Results);
   for (auto c : Results) {
-    if (c.find(Prefix) == 0)
-      Comps.push_back(llvm::LineEditor::Completion(c.substr(Prefix.size()), c));
+    if (c.find(CC.Prefix) == 0)
+      Comps.push_back(
+          llvm::LineEditor::Completion(c.substr(CC.Prefix.size()), c));
   }
   return Comps;
 }
diff --git a/clang/unittests/Interpreter/CodeCompletionTest.cpp b/clang/unittests/Interpreter/CodeCompletionTest.cpp
