Fix multiple cases where the parser produced token kinds that didn’t match the token kinds in the syntax node definitions

The main problem here was that we were accepting `self` and `Self` in `expectIdentifier` but not handling that in the syntax tree. I went through every call of `expectIdentifier` and checked what the C++ parser does to maintain the behavior. Where we need to accept `self` or `Self` for compatibility reasons, it is now remapped to an identifier.

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/AttributeNodes.swift

@@ -301,7 +301,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "DeclBaseName",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "PrefixOperatorToken"), .keyword(text: "init")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .keyword(text: "init"), .keyword(text: "self"), .keyword(text: "Self")]),
         nameForDiagnostics: "base name",
         description: "The base name of the protocol's requirement."
       ),
@@ -468,7 +468,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "DiffKind",
-        kind: .token(choices: [.keyword(text: "forward"), .keyword(text: "reverse"), .keyword(text: "linear")]),
+        kind: .token(choices: [.keyword(text: "_forward"), .keyword(text: "reverse"), .keyword(text: "_linear")]),
         isOptional: true
       ),
       Child(
@@ -644,7 +644,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "Label",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "available"), .keyword(text: "exported"), .keyword(text: "kind"), .keyword(text: "spi"), .keyword(text: "spiModule")]),
+        kind: .node(kind: "Token"),
         nameForDiagnostics: "label",
         description: "The label of the argument"
       ),
@@ -677,7 +677,7 @@ public let ATTRIBUTE_NODES: [Node] = [
     children: [
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+        kind: .node(kind: "Token"),
         nameForDiagnostics: "name",
         isOptional: true
       ),
@@ -779,7 +779,7 @@ public let ATTRIBUTE_NODES: [Node] = [
       ),
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "self"), .keyword(text: "Self"), .keyword(text: "init"), .token(tokenKind: "BinaryOperatorToken")]),
         nameForDiagnostics: "base name",
         description: "The base name of the referenced function."
       ),

CodeGeneration/Sources/SyntaxSupport/DeclNodes.swift

@@ -19,7 +19,7 @@ public let DECL_NODES: [Node] = [
     children: [
       Child(
         name: "Name",
-        kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+        kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "BinaryOperatorToken"), .token(tokenKind: "PrefixOperatorToken"), .token(tokenKind: "PostfixOperatorToken")]),
         nameForDiagnostics: "name"
       ),
       Child(

Sources/SwiftParser/Attributes.swift

@@ -575,11 +575,12 @@ extension Parser {
     let (unexpectedBetweenOfLabelAndColon, colon) = self.expect(.colon)
     let originalDeclName = self.parseQualifiedDeclarationName()
     let period = self.consume(if: .period)
+    let unexpectedBeforeAccessor: RawUnexpectedNodesSyntax?
     let accessor: RawTokenSyntax?
     if period != nil {
-      accessor = self.parseAnyIdentifier()
+      (unexpectedBeforeAccessor, accessor) = self.expect(.keyword(.get), .keyword(.set), default: .keyword(.get))
     } else {
-      accessor = nil
+      (unexpectedBeforeAccessor, accessor) = (nil, nil)
     }
     let comma = self.consume(if: .comma)
     let diffParams: RawDifferentiabilityParamsClauseSyntax?
@@ -595,6 +596,7 @@ extension Parser {
       colon: colon,
       originalDeclName: originalDeclName,
       period: period,
+      unexpectedBeforeAccessor,
       accessorKind: accessor,
       comma: comma,
       diffParams: diffParams,

Sources/SwiftParser/Declarations.swift

@@ -476,7 +476,7 @@ extension Parser {
         // Parse the 'each' keyword for a type parameter pack 'each T'.
         var each = self.consume(if: .keyword(.each))
 
-        let (unexpectedBetweenEachAndName, name) = self.expectIdentifier()
+        let (unexpectedBetweenEachAndName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
         if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty {
           break
         }
@@ -631,7 +631,7 @@ extension Parser {
               body: .sameTypeRequirement(
                 RawSameTypeRequirementSyntax(
                   leftTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
-                  equalityToken: missingToken(.equal),
+                  equalityToken: missingToken(.binaryOperator, text: "=="),
                   rightTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
                   arena: self.arena
                 )
@@ -890,7 +890,7 @@ extension Parser {
       var loopProgress = LoopProgressCondition()
       repeat {
         let unexpectedPeriod = self.consume(if: .period)
-        let (unexpectedBeforeName, name) = self.expectIdentifier(allowIdentifierLikeKeywords: false, keywordRecovery: true)
+        let (unexpectedBeforeName, name) = self.expectIdentifier(keywordRecovery: true)
 
         let associatedValue: RawParameterClauseSyntax?
         if self.at(TokenSpec(.leftParen, allowAtStartOfLine: false)) {
@@ -1916,7 +1916,7 @@ extension Parser {
     // checking.
     let precedenceAndTypes: RawOperatorPrecedenceAndTypesSyntax?
     if let colon = self.consume(if: .colon) {
-      let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(keywordRecovery: true)
+      let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
       var types = [RawDesignatedTypeElementSyntax]()
       while let comma = self.consume(if: .comma) {
         // FIXME: The compiler accepts... anything here. This is a bug.
@@ -2001,7 +2001,7 @@ extension Parser {
     _ handle: RecoveryConsumptionHandle
   ) -> RawPrecedenceGroupDeclSyntax {
     let (unexpectedBeforeGroup, group) = self.eat(handle)
-    let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(keywordRecovery: true)
+    let (unexpectedBeforeIdentifier, identifier) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
     let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)
 
     let groupAttributes = self.parsePrecedenceGroupAttributeListSyntax()

Sources/SwiftParser/Expressions.swift

@@ -1796,7 +1796,7 @@ extension Parser {
           let expression: RawExprSyntax
           if self.peek().rawTokenKind == .equal {
             // The name is a new declaration.
-            (unexpectedBeforeName, name) = self.expectIdentifier()
+            (unexpectedBeforeName, name) = self.expect(.identifier, TokenSpec(.self, remapping: .identifier), default: .identifier)
             (unexpectedBeforeAssignToken, assignToken) = self.expect(.equal)
             expression = self.parseExpression()
           } else {

Sources/SwiftParser/Lookahead.swift

@@ -187,7 +187,7 @@ extension Parser.Lookahead {
     while let _ = self.consume(if: .atSign) {
       // Consume qualified names that may or may not involve generic arguments.
       repeat {
-        self.expectIdentifierOrRethrowsWithoutRecovery()
+        self.consume(if: .identifier, .keyword(.rethrows))
         // We don't care whether this succeeds or fails to eat generic
         // parameters.
         _ = self.consumeGenericArguments()

Sources/SwiftParser/Names.swift

@@ -63,20 +63,20 @@ extension Parser {
 
   mutating func parseDeclNameRef(_ flags: DeclNameOptions = []) -> (RawTokenSyntax, RawDeclNameArgumentsSyntax?) {
     // Consume the base name.
-    let ident: RawTokenSyntax
-    if self.at(.identifier) || self.at(.keyword(.self), .keyword(.Self), .keyword(.`init`)) {
-      ident = self.expectIdentifierWithoutRecovery()
+    let base: RawTokenSyntax
+    if let identOrSelf = self.consume(if: .identifier, .keyword(.self), .keyword(.Self)) ?? self.consume(if: .keyword(.`init`)) {
+      base = identOrSelf
     } else if flags.contains(.operators), let (_, _) = self.at(anyIn: Operator.self) {
-      ident = self.consumeAnyToken(remapping: .binaryOperator)
+      base = self.consumeAnyToken(remapping: .binaryOperator)
     } else if flags.contains(.keywords) && self.currentToken.isLexerClassifiedKeyword {
-      ident = self.consumeAnyToken(remapping: .identifier)
+      base = self.consumeAnyToken(remapping: .identifier)
     } else {
-      ident = self.expectIdentifierWithoutRecovery()
+      base = missingToken(.identifier)
     }
 
     // Parse an argument list, if the flags allow it and it's present.
     let args = self.parseArgLabelList(flags)
-    return (ident, args)
+    return (base, args)
   }
 
   mutating func parseArgLabelList(_ flags: DeclNameOptions) -> RawDeclNameArgumentsSyntax? {
@@ -176,7 +176,7 @@ extension Parser {
     var keepGoing: RawTokenSyntax? = nil
     var loopProgress = LoopProgressCondition()
     repeat {
-      let (name, _) = self.parseDeclNameRef()
+      let (unexpectedBeforeName, name) = self.expect(.identifier, .keyword(.self), .keyword(.Self), default: .identifier)
       let generics: RawGenericArgumentClauseSyntax?
       if self.atContextualPunctuator("<") {
         generics = self.parseGenericArguments()
@@ -188,6 +188,7 @@ extension Parser {
           RawMemberTypeIdentifierSyntax(
             baseType: result!,
             period: keepGoing,
+            unexpectedBeforeName,
             name: name,
             genericArgumentClause: generics,
             arena: self.arena
@@ -196,6 +197,7 @@ extension Parser {
       } else {
         result = RawTypeSyntax(
           RawSimpleTypeIdentifierSyntax(
+            unexpectedBeforeName,
             name: name,
             genericArgumentClause: generics,
             arena: self.arena

Sources/SwiftParser/Nominals.swift

@@ -150,7 +150,7 @@ extension Parser {
     introucerHandle: RecoveryConsumptionHandle
   ) -> T where T: NominalTypeDeclarationTrait {
     let (unexpectedBeforeIntroducerKeyword, introducerKeyword) = self.eat(introucerHandle)
-    let (unexpectedBeforeName, name) = self.expectIdentifier(allowIdentifierLikeKeywords: false, keywordRecovery: true)
+    let (unexpectedBeforeName, name) = self.expectIdentifier(keywordRecovery: true)
     if unexpectedBeforeName == nil && name.isMissing && self.currentToken.isAtStartOfLine {
       return T.init(
         attributes: attrs.attributes,
@@ -258,7 +258,7 @@ extension Parser {
       var loopProgress = LoopProgressCondition()
       repeat {
         // Parse the name of the parameter.
-        let (unexpectedBeforeName, name) = self.expectIdentifier()
+        let (unexpectedBeforeName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true)
         keepGoing = self.consume(if: .comma)
         associatedTypes.append(
           RawPrimaryAssociatedTypeSyntax(

Sources/SwiftParser/Parser.swift

@@ -417,18 +417,20 @@ extension Parser {
   /// incorrectly used a keyword as an identifier.
   /// This should be set if keywords aren't strong recovery marker at this
   /// position, e.g. because the parser expects a punctuator next.
+  ///
+  /// If `allowSelfOrCapitalSelfAsIdentifier` is `true`, then `self` and `Self`
+  /// are also accepted and remapped to identifiers. This is exclusively used
+  /// to maintain compatibility with the C++ parser. No new uses of this should
+  /// be introduced.
   mutating func expectIdentifier(
-    allowIdentifierLikeKeywords: Bool = true,
-    keywordRecovery: Bool = false
+    keywordRecovery: Bool = false,
+    allowSelfOrCapitalSelfAsIdentifier: Bool = false
   ) -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
-    if allowIdentifierLikeKeywords {
-      if let (_, handle) = self.canRecoverTo(anyIn: IdentifierTokens.self) {
-        return self.eat(handle)
-      }
-    } else {
-      if let identifier = self.consume(if: .identifier) {
-        return (nil, identifier)
-      }
+    if let identifier = self.consume(if: .identifier) {
+      return (nil, identifier)
+    }
+    if allowSelfOrCapitalSelfAsIdentifier, let selfOrCapitalSelf = self.consume(if: TokenSpec(.self, remapping: .identifier), TokenSpec(.Self, remapping: .identifier)) {
+      return (nil, selfOrCapitalSelf)
     }
     if let unknown = self.consume(if: .unknown) {
       return (
@@ -457,22 +459,6 @@ extension Parser {
     )
   }
 
-  mutating func expectIdentifierOrRethrows() -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
-    if let (_, handle) = self.canRecoverTo(anyIn: IdentifierOrRethrowsTokens.self) {
-      return self.eat(handle)
-    }
-    if let unknown = self.consume(if: .unknown) {
-      return (
-        RawUnexpectedNodesSyntax(elements: [RawSyntax(unknown)], arena: self.arena),
-        self.missingToken(.identifier)
-      )
-    }
-    return (
-      nil,
-      self.missingToken(.identifier)
-    )
-  }
-
   /// Expect a right brace but with a little smart recovery logic:
   /// If `leftBrace` is missing, we only recover to a `}` whose indentation is greater or equal to that of `introducer`.
   /// That way, if the developer forgot to to type `{`, we won't eat `}` that were most likely intended to close an outer scope.

Sources/SwiftParser/Statements.swift

@@ -932,7 +932,7 @@ extension Parser {
       return nil
     }
 
-    return self.expectIdentifierWithoutRecovery()
+    return self.expectWithoutRecovery(.identifier)
   }
 }
 

Sources/SwiftParser/TokenConsumer.swift

@@ -219,22 +219,6 @@ extension TokenConsumer {
 // MARK: Convenience functions
 
 extension TokenConsumer {
-  @inline(__always)
-  mutating func expectIdentifierWithoutRecovery() -> Token {
-    if let (_, handle) = self.at(anyIn: IdentifierTokens.self) {
-      return self.eat(handle)
-    }
-    return missingToken(.identifier)
-  }
-
-  @inline(__always)
-  mutating func expectIdentifierOrRethrowsWithoutRecovery() -> Token {
-    if let (_, handle) = self.at(anyIn: IdentifierOrRethrowsTokens.self) {
-      return self.eat(handle)
-    }
-    return missingToken(.identifier)
-  }
-
   var canHaveParameterSpecifier: Bool {
     // The parameter specifiers like `isolated`, `consuming`, `borrowing` are
     // also valid identifiers and could be the name of a type. Check whether

Sources/SwiftParser/TokenSpecSet.swift

@@ -298,64 +298,6 @@ enum DeclarationStart: TokenSpecSet {
   }
 }
 
-enum IdentifierTokens: TokenSpecSet {
-  case anyKeyword
-  case capitalSelfKeyword
-  case identifier
-  case initKeyword
-  case selfKeyword
-
-  init?(lexeme: Lexer.Lexeme) {
-    switch PrepareForKeywordMatch(lexeme) {
-    case TokenSpec(.Any): self = .anyKeyword
-    case TokenSpec(.Self): self = .capitalSelfKeyword
-    case TokenSpec(.identifier): self = .identifier
-    case TokenSpec(.`init`): self = .initKeyword
-    case TokenSpec(.self): self = .selfKeyword
-    default: return nil
-    }
-  }
-
-  var spec: TokenSpec {
-    switch self {
-    case .anyKeyword: return .keyword(.Any)
-    case .capitalSelfKeyword: return .keyword(.Self)
-    case .identifier: return .identifier
-    case .initKeyword: return .keyword(.`init`)
-    case .selfKeyword: return .keyword(.self)
-    }
-  }
-}
-
-enum IdentifierOrRethrowsTokens: TokenSpecSet {
-  case anyKeyword
-  case capitalSelfKeyword
-  case identifier
-  case selfKeyword
-  case rethrowsKeyword
-
-  init?(lexeme: Lexer.Lexeme) {
-    switch PrepareForKeywordMatch(lexeme) {
-    case TokenSpec(.Any): self = .anyKeyword
-    case TokenSpec(.Self): self = .capitalSelfKeyword
-    case TokenSpec(.identifier): self = .identifier
-    case TokenSpec(.self): self = .selfKeyword
-    case TokenSpec(.rethrows): self = .rethrowsKeyword
-    default: return nil
-    }
-  }
-
-  var spec: TokenSpec {
-    switch self {
-    case .anyKeyword: return .keyword(.Any)
-    case .capitalSelfKeyword: return .keyword(.Self)
-    case .identifier: return .identifier
-    case .selfKeyword: return .keyword(.self)
-    case .rethrowsKeyword: return TokenSpec(.rethrows, remapping: .identifier)
-    }
-  }
-}
-
 enum Operator: TokenSpecSet {
   case binaryOperator
   case postfixOperator

Sources/SwiftParser/Types.swift

@@ -271,7 +271,7 @@ extension Parser {
             )
           )
         } else {
-          let (name, generics) = self.parseTypeNameWithGenerics(include: .keywords)
+          let (name, generics) = self.parseTypeNameWithGenerics(allowKeywordAsName: true)
           base = RawTypeSyntax(
             RawMemberTypeIdentifierSyntax(
               baseType: base,
@@ -337,8 +337,15 @@ extension Parser {
     )
   }
 
-  mutating func parseTypeNameWithGenerics(include flags: DeclNameOptions = []) -> (RawTokenSyntax, RawGenericArgumentClauseSyntax?) {
-    let (name, _) = self.parseDeclNameRef(flags)
+  mutating func parseTypeNameWithGenerics(allowKeywordAsName: Bool) -> (RawTokenSyntax, RawGenericArgumentClauseSyntax?) {
+    let name: RawTokenSyntax
+    if let identOrSelf = self.consume(if: .identifier, .keyword(.self), .keyword(.Self)) {
+      name = identOrSelf
+    } else if allowKeywordAsName && self.currentToken.isLexerClassifiedKeyword {
+      name = self.consumeAnyToken(remapping: .identifier)
+    } else {
+      name = missingToken(.identifier)
+    }
     if self.atContextualPunctuator("<") {
       return (name, self.parseGenericArguments())
     }
@@ -356,7 +363,7 @@ extension Parser {
       return RawTypeSyntax(self.parseAnyType())
     }
 
-    let (name, generics) = parseTypeNameWithGenerics()
+    let (name, generics) = parseTypeNameWithGenerics(allowKeywordAsName: false)
     return RawTypeSyntax(
       RawSimpleTypeIdentifierSyntax(
         name: name,