Comment parsing: Complete list of Doxygen commands

These should be all the commands from [1] except those that are marked
obsolete, and "link" / "endlink", as that conflicts with the existing
HeaderDoc pair "link / "/link". For some commands we don't have the
ideal category, but it should work good enough for most cases.

There seems to be no existing test for most commands (except the ones
interpreted by -Wdocumentation), and to some extent such a test wouldn't
look very interesting. But I added a test for the correct parsing of
formulas, as they're a bit special. And I had to adapt
comment-lots-of-unknown-commands.c because typo correction was kicking
in and recognizing some of the commands.

This should fix a couple of reported bugs: PR17437, PR19581, PR24062
(partially, no diagnostic for matching cond/endcond), PR32909, PR37813,
PR44243 (partially, [email protected] must be addressed separately).

[1] https://www.doxygen.nl/manual/commands.html

Reviewed By: gribozavr2

Differential Revision: https://reviews.llvm.org/D111190

Author: aaronpuchert

clang/include/clang/AST/CommentCommands.td

@@ -87,8 +87,21 @@ def P      : InlineCommand<"p">;
 def A      : InlineCommand<"a">;
 def E      : InlineCommand<"e">;
 def Em     : InlineCommand<"em">;
-def Ref    : InlineCommand<"ref">;
-def Anchor : InlineCommand<"anchor">;
+def Emoji  : InlineCommand<"emoji">;
+
+def Anchor  : InlineCommand<"anchor">;
+def Ref     : InlineCommand<"ref">;
+def RefItem : InlineCommand<"refitem">;
+def Cite    : InlineCommand<"cite">;
+
+def CopyBrief   : InlineCommand<"copybrief">;
+def CopyDetails : InlineCommand<"copydetails">;
+def CopyDoc     : InlineCommand<"copydoc">;
+
+// Typically not used inline, but they take a single word.
+def Extends    : InlineCommand<"extends">;
+def Implements : InlineCommand<"implements">;
+def MemberOf   : InlineCommand<"memberof">;
 
 //===----------------------------------------------------------------------===//
 // BlockCommand
@@ -145,9 +158,11 @@ def Retval     : BlockCommand<"retval">;
 def Sa         : BlockCommand<"sa">;
 def See        : BlockCommand<"see">;
 def Since      : BlockCommand<"since">;
+def Test       : BlockCommand<"test">;
 def Todo       : BlockCommand<"todo">;
 def Version    : BlockCommand<"version">;
 def Warning    : BlockCommand<"warning">;
+def XRefItem   : BlockCommand<"xrefitem">;
 // HeaderDoc commands
 def Abstract      : BlockCommand<"abstract"> { let IsBriefCommand = 1; }
 def ClassDesign   : RecordLikeDetailCommand<"classdesign">;
@@ -170,6 +185,8 @@ def SuperClass    : RecordLikeDetailCommand<"superclass">;
 
 defm Code      : VerbatimBlockCommand<"code", "endcode">;
 defm Verbatim  : VerbatimBlockCommand<"verbatim", "endverbatim">;
+
+defm DocbookOnly : VerbatimBlockCommand<"docbookonly", "enddocbookonly">;
 defm Htmlonly  : VerbatimBlockCommand<"htmlonly", "endhtmlonly">;
 defm Latexonly : VerbatimBlockCommand<"latexonly", "endlatexonly">;
 defm Xmlonly   : VerbatimBlockCommand<"xmlonly", "endxmlonly">;
@@ -178,10 +195,19 @@ defm Rtfonly   : VerbatimBlockCommand<"rtfonly", "endrtfonly">;
 
 defm Dot : VerbatimBlockCommand<"dot", "enddot">;
 defm Msc : VerbatimBlockCommand<"msc", "endmsc">;
+defm Uml : VerbatimBlockCommand<"startuml", "enduml">;
+
+// Actually not verbatim blocks, we should also parse commands within them.
+defm Internal   : VerbatimBlockCommand<"internal", "endinternal">;
+// TODO: conflicts with HeaderDoc link, /link.
+//defm Link       : VerbatimBlockCommand<"link", "endlink">;
+defm ParBlock   : VerbatimBlockCommand<"parblock", "endparblock">;
+defm SecRefList : VerbatimBlockCommand<"secreflist", "endsecreflist">;
 
 // These three commands have special support in CommentLexer to recognize their
 // names.
 def  FDollar  : VerbatimBlockCommand<"f$">; // Inline LaTeX formula
+defm FParen   : VerbatimBlockCommand<"f(", "f)">; // Inline LaTeX text
 defm FBracket : VerbatimBlockCommand<"f[", "f]">; // Displayed LaTeX formula
 defm FBrace   : VerbatimBlockCommand<"f{", "f}">; // LaTeX environment
 
@@ -199,11 +225,18 @@ def Addtogroup : VerbatimLineCommand<"addtogroup">;
 def Weakgroup  : VerbatimLineCommand<"weakgroup">;
 def Name       : VerbatimLineCommand<"name">;
 
+// These actually take a single word, but it's optional.
+// And they're used on a separate line typically, not inline.
+def Dir  : VerbatimLineCommand<"dir">;
+def File : VerbatimLineCommand<"file">;
+
 def Section       : VerbatimLineCommand<"section">;
 def Subsection    : VerbatimLineCommand<"subsection">;
 def Subsubsection : VerbatimLineCommand<"subsubsection">;
 def Paragraph     : VerbatimLineCommand<"paragraph">;
+def TableOfContents : VerbatimLineCommand<"tableofcontents">;
 
+def Page     : VerbatimLineCommand<"page">;
 def Mainpage : VerbatimLineCommand<"mainpage">;
 def Subpage  : VerbatimLineCommand<"subpage">;
 
@@ -212,13 +245,79 @@ def Related     : VerbatimLineCommand<"related">;
 def RelatesAlso : VerbatimLineCommand<"relatesalso">;
 def RelatedAlso : VerbatimLineCommand<"relatedalso">;
 
+def AddIndex : VerbatimLineCommand<"addindex">;
+
+// These take a single argument mostly, but since they include a file they'll
+// typically be on their own line.
+def DocbookInclude : VerbatimLineCommand<"docbookinclude">;
+def DontInclude    : VerbatimLineCommand<"dontinclude">;
+def Example        : VerbatimLineCommand<"example">;
+def HtmlInclude    : VerbatimLineCommand<"htmlinclude">;
+def Include        : VerbatimLineCommand<"include">;
+def ManInclude     : VerbatimLineCommand<"maninclude">;
+def LatexInclude   : VerbatimLineCommand<"latexinclude">;
+def RtfInclude     : VerbatimLineCommand<"rtfinclude">;
+def Snippet        : VerbatimLineCommand<"snippet">;
+def VerbInclude    : VerbatimLineCommand<"verbinclude">;
+def XmlInclude     : VerbatimLineCommand<"xmlinclude">;
+
+def Image   : VerbatimLineCommand<"image">;
+def DotFile : VerbatimLineCommand<"dotfile">;
+def MscFile : VerbatimLineCommand<"mscfile">;
+def DiaFile : VerbatimLineCommand<"diafile">;
+
+def Line     : VerbatimLineCommand<"line">;
+def Skip     : VerbatimLineCommand<"skip">;
+def SkipLine : VerbatimLineCommand<"skipline">;
+def Until    : VerbatimLineCommand<"until">;
+
+def NoOp : VerbatimLineCommand<"noop">;
+
+// These have actually no arguments, but we can treat them as line commands.
+def CallGraph       : VerbatimLineCommand<"callgraph">;
+def HideCallGraph   : VerbatimLineCommand<"hidecallgraph">;
+def CallerGraph     : VerbatimLineCommand<"callergraph">;
+def HideCallerGraph : VerbatimLineCommand<"hidecallergraph">;
+def ShowInitializer : VerbatimLineCommand<"showinitializer">;
+def HideInitializer : VerbatimLineCommand<"hideinitializer">;
+def ShowRefBy       : VerbatimLineCommand<"showrefby">;
+def HideRefBy       : VerbatimLineCommand<"hiderefby">;
+def ShowRefs        : VerbatimLineCommand<"showrefs">;
+def HideRefs        : VerbatimLineCommand<"hiderefs">;
+
+// These also have no argument.
+def Private   : VerbatimLineCommand<"private">;
+def Protected : VerbatimLineCommand<"protected">;
+def Public    : VerbatimLineCommand<"public">;
+def Pure      : VerbatimLineCommand<"pure">;
+def Static    : VerbatimLineCommand<"static">;
+
+// These also have no argument.
+def NoSubgrouping    : VerbatimLineCommand<"nosubgrouping">;
+def PrivateSection   : VerbatimLineCommand<"privatesection">;
+def ProtectedSection : VerbatimLineCommand<"protectedsection">;
+def PublicSection    : VerbatimLineCommand<"publicsection">;
+
+// We might also build proper support for if/ifnot/else/elseif/endif.
+def If     : VerbatimLineCommand<"if">;
+def IfNot  : VerbatimLineCommand<"ifnot">;
+def Else   : VerbatimLineCommand<"else">;
+def ElseIf : VerbatimLineCommand<"elseif">;
+def Endif  : VerbatimLineCommand<"endif">;
+
+// Not treated as VerbatimBlockCommand because it spans multiple comments.
+def Cond    : VerbatimLineCommand<"cond">;
+def EndCond : VerbatimLineCommand<"endcond">;
+
 //===----------------------------------------------------------------------===//
 // DeclarationVerbatimLineCommand
 //===----------------------------------------------------------------------===//
 
 // Doxygen commands.
+def Concept   : DeclarationVerbatimLineCommand<"concept">;
 def Def       : DeclarationVerbatimLineCommand<"def">;
 def Fn        : DeclarationVerbatimLineCommand<"fn">;
+def IDLExcept : DeclarationVerbatimLineCommand<"idlexcept">;
 def Namespace : DeclarationVerbatimLineCommand<"namespace">;
 def Overload  : DeclarationVerbatimLineCommand<"overload">;
 def Property  : DeclarationVerbatimLineCommand<"property">;

clang/lib/AST/CommentLexer.cpp

@@ -392,10 +392,11 @@ void Lexer::lexCommentText(Token &T) {
       unsigned Length = TokenPtr - (BufferPtr + 1);
 
       // Hardcoded support for lexing LaTeX formula commands
-      // \f$ \f[ \f] \f{ \f} as a single command.
+      // \f$ \f( \f) \f[ \f] \f{ \f} as a single command.
       if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
         C = *TokenPtr;
-        if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
+        if (C == '$' || C == '(' || C == ')' || C == '[' || C == ']' ||
+            C == '{' || C == '}') {
           TokenPtr++;
           Length++;
         }

clang/test/AST/ast-dump-comment.cpp

@@ -76,10 +76,22 @@ int Test_HTMLTagComment;
 /// \verbatim
 /// Aaa
 /// \endverbatim
+/// \f$ a \f$
+/// \f( b \f)
+/// \f[ c \f]
+/// \f{env}{ c \f}
 int Test_VerbatimBlockComment;
 // CHECK:      VarDecl{{.*}}Test_VerbatimBlockComment
 // CHECK:        VerbatimBlockComment{{.*}} Name="verbatim" CloseName="endverbatim"
 // CHECK-NEXT:     VerbatimBlockLineComment{{.*}} Text=" Aaa"
+// CHECK:        VerbatimBlockComment{{.*}} Name="f$" CloseName="f$"
+// CHECK-NEXT:     VerbatimBlockLineComment{{.*}} Text=" a "
+// CHECK:        VerbatimBlockComment{{.*}} Name="f(" CloseName="f)"
+// CHECK-NEXT:     VerbatimBlockLineComment{{.*}} Text=" b "
+// CHECK:        VerbatimBlockComment{{.*}} Name="f[" CloseName="f]"
+// CHECK-NEXT:     VerbatimBlockLineComment{{.*}} Text=" c "
+// CHECK:        VerbatimBlockComment{{.*}} Name="f{" CloseName="f}"
+// CHECK-NEXT:     VerbatimBlockLineComment{{.*}} Text="env}{ c "
 
 /// \param ... More arguments
 template<typename T>

clang/test/Index/comment-lots-of-unknown-commands.c

@@ -39,7 +39,7 @@
 @ien
 @fr
 @en
-@tet
+@teq
 @le
 @L
 @os
@@ -77,7 +77,7 @@
 @rU
 @ar
 @eD
-@iE
+@iEx
 @se
 @st1
 @rr
@@ -106,15 +106,15 @@
 @hd
 @be
 @It
-@id
+@idz
 @cm
 @ua
 @fs
 @Al
 @axn
 @rt
 @to
-@is
+@isj
 @fo
 @i
 @an
@@ -185,7 +185,7 @@ void f();
 // CHECK:         (CXComment_InlineCommand CommandName=[ien] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[fr] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[en] RenderNormal HasTrailingNewline)
-// CHECK:         (CXComment_InlineCommand CommandName=[tet] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[teq] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[le] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[L] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[os] RenderNormal HasTrailingNewline)
@@ -223,7 +223,7 @@ void f();
 // CHECK:         (CXComment_InlineCommand CommandName=[rU] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[ar] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[eD] RenderNormal HasTrailingNewline)
-// CHECK:         (CXComment_InlineCommand CommandName=[iE] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[iEx] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[se] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[st1] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[rr] RenderNormal HasTrailingNewline)
@@ -252,15 +252,15 @@ void f();
 // CHECK:         (CXComment_InlineCommand CommandName=[hd] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[be] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[It] RenderNormal HasTrailingNewline)
-// CHECK:         (CXComment_InlineCommand CommandName=[id] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[idz] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[cm] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[ua] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[fs] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[Al] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[axn] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[rt] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[to] RenderNormal HasTrailingNewline)
-// CHECK:         (CXComment_InlineCommand CommandName=[is] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[isj] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[fo] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[i] RenderNormal HasTrailingNewline)
 // CHECK:         (CXComment_InlineCommand CommandName=[an] RenderNormal HasTrailingNewline)

clang/unittests/AST/CommentLexer.cpp

@@ -836,12 +836,12 @@ TEST_F(CommentLexerTest, VerbatimBlock8) {
 // LaTeX verbatim blocks.
 TEST_F(CommentLexerTest, VerbatimBlock9) {
   const char *Source =
-    "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f}";
+    "/// \\f$ Aaa \\f$ \\f[ Bbb \\f] \\f{ Ccc \\f} \\f( Ddd \\f)";
   std::vector<Token> Toks;
 
   lexString(Source, Toks);
 
-  ASSERT_EQ(13U, Toks.size());
+  ASSERT_EQ(17U, Toks.size());
 
   ASSERT_EQ(tok::text,                 Toks[0].getKind());
   ASSERT_EQ(StringRef(" "),            Toks[0].getText());
@@ -879,7 +879,19 @@ TEST_F(CommentLexerTest, VerbatimBlock9) {
   ASSERT_EQ(tok::verbatim_block_end,   Toks[11].getKind());
   ASSERT_EQ(StringRef("f}"),           getVerbatimBlockName(Toks[11]));
 
-  ASSERT_EQ(tok::newline,              Toks[12].getKind());
+  ASSERT_EQ(tok::text,                 Toks[12].getKind());
+  ASSERT_EQ(StringRef(" "),            Toks[12].getText());
+
+  ASSERT_EQ(tok::verbatim_block_begin, Toks[13].getKind());
+  ASSERT_EQ(StringRef("f("),           getVerbatimBlockName(Toks[13]));
+
+  ASSERT_EQ(tok::verbatim_block_line,  Toks[14].getKind());
+  ASSERT_EQ(StringRef(" Ddd "),        Toks[14].getVerbatimBlockText());
+
+  ASSERT_EQ(tok::verbatim_block_end,   Toks[15].getKind());
+  ASSERT_EQ(StringRef("f)"),           getVerbatimBlockName(Toks[15]));
+
+  ASSERT_EQ(tok::newline,              Toks[16].getKind());
 }
 
 // Empty verbatim line.

clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp

@@ -83,6 +83,12 @@ static std::string MangleName(StringRef Str) {
     default:
       Mangled += Str[i];
       break;
+    case '(':
+      Mangled += "lparen";
+      break;
+    case ')':
+      Mangled += "rparen";
+      break;
     case '[':
       Mangled += "lsquare";
       break;