[clangd][WIP] Add doxygen parsing using standalone doxygen parser

clang-tools-extra/clangd/CMakeLists.txt

@@ -108,6 +108,7 @@ add_clang_library(clangDaemon STATIC
   SemanticHighlighting.cpp
   SemanticSelection.cpp
   SourceCode.cpp
+  SymbolDocumentation.cpp
   SystemIncludeExtractor.cpp
   TidyProvider.cpp
   TUScheduler.cpp

clang-tools-extra/clangd/CodeCompletionStrings.cpp

@@ -9,6 +9,12 @@
 #include "CodeCompletionStrings.h"
 #include "clang-c/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Comment.h"
+#include "clang/AST/CommentCommandTraits.h"
+#include "clang/AST/CommentLexer.h"
+#include "clang/AST/CommentParser.h"
+#include "clang/AST/CommentSema.h"
+#include "clang/AST/Decl.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
@@ -100,14 +106,25 @@ std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) {
     // the comments for namespaces.
     return "";
   }
-  const RawComment *RC = getCompletionComment(Ctx, &Decl);
-  if (!RC)
-    return "";
-  // Sanity check that the comment does not come from the PCH. We choose to not
-  // write them into PCH, because they are racy and slow to load.
-  assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
-  std::string Doc =
-      RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+
+  std::string Doc;
+
+  if (isa<ParmVarDecl>(Decl)) {
+    // Parameters are documented in the function comment.
+    if (const auto *FD = dyn_cast<FunctionDecl>(Decl.getDeclContext()))
+      Doc = getParamDocString(Ctx.getCommentForDecl(FD, nullptr),
+                              Decl.getName(), Ctx.getCommentCommandTraits());
+  } else {
+
+    const RawComment *RC = getCompletionComment(Ctx, &Decl);
+    if (!RC)
+      return "";
+    // Sanity check that the comment does not come from the PCH. We choose to
+    // not write them into PCH, because they are racy and slow to load.
+    assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
+    Doc = RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+  }
+
   if (!looksLikeDocComment(Doc))
     return "";
   // Clang requires source to be UTF-8, but doesn't enforce this in comments.
@@ -316,5 +333,46 @@ std::string getReturnType(const CodeCompletionString &CCS) {
   return "";
 }
 
+void docCommentToMarkup(
+    markup::Document &Doc, llvm::StringRef Comment,
+    llvm::BumpPtrAllocator &Allocator, comments::CommandTraits &Traits,
+    std::optional<SymbolPrintedType> SymbolType,
+    std::optional<SymbolPrintedType> SymbolReturnType,
+    const std::optional<std::vector<SymbolParam>> &SymbolParameters) {
+
+  // The comment lexer expects doxygen markers, so add them back.
+  // We need to use the /// style doxygen markers because the comment could
+  // contain the closing the closing tag "*/" of a C Style "/** */" comment
+  // which would break the parsing if we would just enclose the comment text
+  // with "/** */".
+  std::string CommentWithMarkers = "///";
+  for (char C : Comment) {
+    if (C == '\n') {
+      CommentWithMarkers += "\n///";
+    } else {
+      CommentWithMarkers += C;
+    }
+  }
+  SourceManagerForFile SourceMgrForFile("mock_file.cpp", CommentWithMarkers);
+
+  SourceManager &SourceMgr = SourceMgrForFile.get();
+  // The doxygen Sema requires a Diagostics consumer, since it reports warnings
+  // e.g. when parameters are not documented correctly.
+  // These warnings are not relevant for us, so we can ignore them.
+  SourceMgr.getDiagnostics().setClient(new IgnoringDiagConsumer);
+
+  comments::Sema S(Allocator, SourceMgr, SourceMgr.getDiagnostics(), Traits,
+                   /*PP=*/nullptr);
+  comments::Lexer L(Allocator, SourceMgr.getDiagnostics(), Traits,
+                    SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()),
+                    CommentWithMarkers.data(),
+                    CommentWithMarkers.data() + CommentWithMarkers.size());
+  comments::Parser P(L, S, Allocator, SourceMgr, SourceMgr.getDiagnostics(),
+                     Traits);
+
+  fullCommentToMarkupDocument(Doc, P.parseFullComment(), Traits, SymbolType,
+                              SymbolReturnType, SymbolParameters);
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/CodeCompletionStrings.h

@@ -14,11 +14,17 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETIONSTRINGS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETIONSTRINGS_H
 
+#include "SymbolDocumentation.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 
 namespace clang {
 class ASTContext;
 
+namespace comments {
+class CommandTraits;
+class FullComment;
+} // namespace comments
+
 namespace clangd {
 
 /// Gets a minimally formatted documentation comment of \p Result, with comment
@@ -67,6 +73,23 @@ std::string formatDocumentation(const CodeCompletionString &CCS,
 /// is usually the return type of a function.
 std::string getReturnType(const CodeCompletionString &CCS);
 
+/// \brief Parse the \p Comment as doxygen comment and save the result in the
+/// given markup Document \p Doc.
+///
+/// It is assumed that comment markers have already been stripped (e.g. via
+/// getDocComment()).
+///
+/// This uses the Clang doxygen comment parser to parse the comment, and then
+/// converts the parsed comment to a markup::Document. The resulting document is
+/// a combination of the symbol information \p SymbolType, \p SymbolReturnType,
+/// and \p SymbolParameters and the parsed doxygen comment.
+void docCommentToMarkup(
+    markup::Document &Doc, llvm::StringRef Comment,
+    llvm::BumpPtrAllocator &Allocator, comments::CommandTraits &Traits,
+    std::optional<SymbolPrintedType> SymbolType,
+    std::optional<SymbolPrintedType> SymbolReturnType,
+    const std::optional<std::vector<SymbolParam>> &SymbolParameters);
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/Hover.cpp

@@ -17,6 +17,7 @@
 #include "ParsedAST.h"
 #include "Selection.h"
 #include "SourceCode.h"
+#include "SymbolDocumentation.h"
 #include "clang-include-cleaner/Analysis.h"
 #include "clang-include-cleaner/IncludeSpeller.h"
 #include "clang-include-cleaner/Types.h"
@@ -40,6 +41,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
@@ -160,14 +162,14 @@ const char *getMarkdownLanguage(const ASTContext &Ctx) {
   return LangOpts.ObjC ? "objective-c" : "cpp";
 }
 
-HoverInfo::PrintedType printType(QualType QT, ASTContext &ASTCtx,
-                                 const PrintingPolicy &PP) {
+SymbolPrintedType printType(QualType QT, ASTContext &ASTCtx,
+                            const PrintingPolicy &PP) {
   // TypePrinter doesn't resolve decltypes, so resolve them here.
   // FIXME: This doesn't handle composite types that contain a decltype in them.
   // We should rather have a printing policy for that.
   while (!QT.isNull() && QT->isDecltypeType())
     QT = QT->castAs<DecltypeType>()->getUnderlyingType();
-  HoverInfo::PrintedType Result;
+  SymbolPrintedType Result;
   llvm::raw_string_ostream OS(Result.Type);
   // Special case: if the outer type is a tag type without qualifiers, then
   // include the tag for extra clarity.
@@ -189,16 +191,16 @@ HoverInfo::PrintedType printType(QualType QT, ASTContext &ASTCtx,
   return Result;
 }
 
-HoverInfo::PrintedType printType(const TemplateTypeParmDecl *TTP) {
-  HoverInfo::PrintedType Result;
+SymbolPrintedType printType(const TemplateTypeParmDecl *TTP) {
+  SymbolPrintedType Result;
   Result.Type = TTP->wasDeclaredWithTypename() ? "typename" : "class";
   if (TTP->isParameterPack())
     Result.Type += "...";
   return Result;
 }
 
-HoverInfo::PrintedType printType(const NonTypeTemplateParmDecl *NTTP,
-                                 const PrintingPolicy &PP) {
+SymbolPrintedType printType(const NonTypeTemplateParmDecl *NTTP,
+                            const PrintingPolicy &PP) {
   auto PrintedType = printType(NTTP->getType(), NTTP->getASTContext(), PP);
   if (NTTP->isParameterPack()) {
     PrintedType.Type += "...";
@@ -208,9 +210,9 @@ HoverInfo::PrintedType printType(const NonTypeTemplateParmDecl *NTTP,
   return PrintedType;
 }
 
-HoverInfo::PrintedType printType(const TemplateTemplateParmDecl *TTP,
-                                 const PrintingPolicy &PP) {
-  HoverInfo::PrintedType Result;
+SymbolPrintedType printType(const TemplateTemplateParmDecl *TTP,
+                            const PrintingPolicy &PP) {
+  SymbolPrintedType Result;
   llvm::raw_string_ostream OS(Result.Type);
   OS << "template <";
   llvm::StringRef Sep = "";
@@ -230,14 +232,14 @@ HoverInfo::PrintedType printType(const TemplateTemplateParmDecl *TTP,
   return Result;
 }
 
-std::vector<HoverInfo::Param>
+std::vector<SymbolParam>
 fetchTemplateParameters(const TemplateParameterList *Params,
                         const PrintingPolicy &PP) {
   assert(Params);
-  std::vector<HoverInfo::Param> TempParameters;
+  std::vector<SymbolParam> TempParameters;
 
   for (const Decl *Param : *Params) {
-    HoverInfo::Param P;
+    SymbolParam P;
     if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(Param)) {
       P.Type = printType(TTP);
 
@@ -351,41 +353,13 @@ void enhanceFromIndex(HoverInfo &Hover, const NamedDecl &ND,
   });
 }
 
-// Default argument might exist but be unavailable, in the case of unparsed
-// arguments for example. This function returns the default argument if it is
-// available.
-const Expr *getDefaultArg(const ParmVarDecl *PVD) {
-  // Default argument can be unparsed or uninstantiated. For the former we
-  // can't do much, as token information is only stored in Sema and not
-  // attached to the AST node. For the latter though, it is safe to proceed as
-  // the expression is still valid.
-  if (!PVD->hasDefaultArg() || PVD->hasUnparsedDefaultArg())
-    return nullptr;
-  return PVD->hasUninstantiatedDefaultArg() ? PVD->getUninstantiatedDefaultArg()
-                                            : PVD->getDefaultArg();
-}
-
-HoverInfo::Param toHoverInfoParam(const ParmVarDecl *PVD,
-                                  const PrintingPolicy &PP) {
-  HoverInfo::Param Out;
-  Out.Type = printType(PVD->getType(), PVD->getASTContext(), PP);
-  if (!PVD->getName().empty())
-    Out.Name = PVD->getNameAsString();
-  if (const Expr *DefArg = getDefaultArg(PVD)) {
-    Out.Default.emplace();
-    llvm::raw_string_ostream OS(*Out.Default);
-    DefArg->printPretty(OS, nullptr, PP);
-  }
-  return Out;
-}
-
 // Populates Type, ReturnType, and Parameters for function-like decls.
 void fillFunctionTypeAndParams(HoverInfo &HI, const Decl *D,
                                const FunctionDecl *FD,
                                const PrintingPolicy &PP) {
   HI.Parameters.emplace();
   for (const ParmVarDecl *PVD : FD->parameters())
-    HI.Parameters->emplace_back(toHoverInfoParam(PVD, PP));
+    HI.Parameters->emplace_back(createSymbolParam(PVD, PP));
 
   // We don't want any type info, if name already contains it. This is true for
   // constructors/destructors and conversion operators.
@@ -626,6 +600,9 @@ HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP,
   HI.Name = printName(Ctx, *D);
   const auto *CommentD = getDeclForComment(D);
   HI.Documentation = getDeclComment(Ctx, *CommentD);
+  // safe the language options to be able to create the comment::CommandTraits
+  // to parse the documentation
+  HI.CommentOpts = D->getASTContext().getLangOpts().CommentOpts;
   enhanceFromIndex(HI, *CommentD, Index);
   if (HI.Documentation.empty())
     HI.Documentation = synthesizeDocumentation(D);
@@ -812,7 +789,7 @@ HoverInfo getHoverContents(const DefinedMacro &Macro, const syntax::Token &Tok,
   return HI;
 }
 
-std::string typeAsDefinition(const HoverInfo::PrintedType &PType) {
+std::string typeAsDefinition(const SymbolPrintedType &PType) {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
   OS << PType.Type;
@@ -1095,7 +1072,7 @@ void maybeAddCalleeArgInfo(const SelectionTree::Node *N, HoverInfo &HI,
 
     // Extract matching argument from function declaration.
     if (const ParmVarDecl *PVD = Parameters[I]) {
-      HI.CalleeArgInfo.emplace(toHoverInfoParam(PVD, PP));
+      HI.CalleeArgInfo.emplace(createSymbolParam(PVD, PP));
       if (N == &OuterNode)
         PassType.PassBy = getPassMode(PVD->getType());
     }
@@ -1455,30 +1432,38 @@ markup::Document HoverInfo::present() const {
 
   // Put a linebreak after header to increase readability.
   Output.addRuler();
-  // Print Types on their own lines to reduce chances of getting line-wrapped by
-  // editor, as they might be long.
-  if (ReturnType) {
-    // For functions we display signature in a list form, e.g.:
-    // → `x`
-    // Parameters:
-    // - `bool param1`
-    // - `int param2 = 5`
-    Output.addParagraph().appendText("→ ").appendCode(
-        llvm::to_string(*ReturnType));
-  }
 
-  if (Parameters && !Parameters->empty()) {
-    Output.addParagraph().appendText("Parameters: ");
-    markup::BulletList &L = Output.addBulletList();
-    for (const auto &Param : *Parameters)
-      L.addItem().addParagraph().appendCode(llvm::to_string(Param));
-  }
+  if (!Documentation.empty()) {
+    llvm::BumpPtrAllocator Allocator;
+    comments::CommandTraits Traits(Allocator, CommentOpts);
+    docCommentToMarkup(Output, Documentation, Allocator, Traits, Type,
+                       ReturnType, Parameters);
+  } else {
+    // Print Types on their own lines to reduce chances of getting line-wrapped
+    // by editor, as they might be long.
+    if (ReturnType) {
+      // For functions we display signature in a list form, e.g.:
+      // → `x`
+      // Parameters:
+      // - `bool param1`
+      // - `int param2 = 5`
+      Output.addParagraph().appendText("→ ").appendCode(
+          llvm::to_string(*ReturnType));
+    }
 
-  // Don't print Type after Parameters or ReturnType as this will just duplicate
-  // the information
-  if (Type && !ReturnType && !Parameters)
-    Output.addParagraph().appendText("Type: ").appendCode(
-        llvm::to_string(*Type));
+    if (Parameters && !Parameters->empty()) {
+      Output.addParagraph().appendText("Parameters: ");
+      markup::BulletList &L = Output.addBulletList();
+      for (const auto &Param : *Parameters)
+        L.addItem().addParagraph().appendCode(llvm::to_string(Param));
+    }
+
+    // Don't print Type after Parameters or ReturnType as this will just
+    // duplicate the information
+    if (Type && !ReturnType && !Parameters)
+      Output.addParagraph().appendText("Type: ").appendCode(
+          llvm::to_string(*Type));
+  }
 
   if (Value) {
     markup::Paragraph &P = Output.addParagraph();
@@ -1518,9 +1503,6 @@ markup::Document HoverInfo::present() const {
     Output.addParagraph().appendText(OS.str());
   }
 
-  if (!Documentation.empty())
-    parseDocumentation(Documentation, Output);
-
   if (!Definition.empty()) {
     Output.addRuler();
     std::string Buffer;
@@ -1646,26 +1628,5 @@ void parseDocumentation(llvm::StringRef Input, markup::Document &Output) {
   FlushParagraph();
 }
 
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                              const HoverInfo::PrintedType &T) {
-  OS << T.Type;
-  if (T.AKA)
-    OS << " (aka " << *T.AKA << ")";
-  return OS;
-}
-
-llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
-                              const HoverInfo::Param &P) {
-  if (P.Type)
-    OS << P.Type->Type;
-  if (P.Name)
-    OS << " " << *P.Name;
-  if (P.Default)
-    OS << " = " << *P.Default;
-  if (P.Type && P.Type->AKA)
-    OS << " (aka " << *P.Type->AKA << ")";
-  return OS;
-}
-
 } // namespace clangd
 } // namespace clang