PHP 8.3 | Tokenizer/PHP: add support for readonly anonymous classes

PHP 8.3 introduced readonly anonymous classes, fixing an oversight in the PHP 8.2 introduction of readonly classes.

As things were, for PHP 8.1+, the tokenizer would change the token code for the `readonly` keyword from `T_READONLY` to `T_STRING` in the "context sensitive keyword" layer, thinking it to be a class name.

And for PHP < 8.1, the readonly polyfill would ignore the token as it being preceded by the `new` keyword would be seen as conflicting with the "context sensitive keyword" layer, which meant it would not be re-tokenized from `T_STRING` to `T_READONLY`.

This commit fixes both.

Includes adding tests in a number of pre-existing test classes to cover this change.

Author: jrfnl

src/Tokenizers/PHP.php

@@ -621,6 +621,21 @@ protected function tokenize($string)
                         $preserveKeyword = true;
                     }
 
+                    // `new readonly class` should be preserved.
+                    if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW) {
+                        for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                            if (is_array($tokens[$i]) === false
+                                || isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false
+                            ) {
+                                break;
+                            }
+                        }
+
+                        if (is_array($tokens[$i]) === true && $tokens[$i][0] === T_CLASS) {
+                            $preserveKeyword = true;
+                        }
+                    }
+
                     // `new class extends` `new class implements` should be preserved
                     if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS)
                         && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS
@@ -1315,7 +1330,8 @@ protected function tokenize($string)
 
             if ($tokenIsArray === true
                 && strtolower($token[1]) === 'readonly'
-                && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false
+                && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === false
+                || $finalTokens[$lastNotEmptyToken]['code'] === T_NEW)
             ) {
                 // Get the next non-whitespace token.
                 for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {

tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc

@@ -5,6 +5,11 @@ $anonClass = new class {
     function __construct() {}
 };
 
+/* testReadonlyNoParentheses */
+$anonClass = new readonly class {
+    function __construct() {}
+};
+
 /* testNoParenthesesAndEmptyTokens */
 $anonClass = new class // phpcs:ignore Standard.Cat
 {
@@ -14,6 +19,11 @@ $anonClass = new class // phpcs:ignore Standard.Cat
 /* testWithParentheses */
 $anonClass = new class() {};
 
+/* testReadonlyWithParentheses */
+$anonClass = new readonly class() {
+    function __construct() {}
+};
+
 /* testWithParenthesesAndEmptyTokens */
 $anonClass = new class /*comment */
 () {};

tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php

@@ -76,6 +76,9 @@ public static function dataAnonClassNoParentheses()
             'plain'                                              => [
                 'testMarker' => '/* testNoParentheses */',
             ],
+            'readonly'                                           => [
+                'testMarker' => '/* testReadonlyNoParentheses */',
+            ],
             'declaration contains comments and extra whitespace' => [
                 'testMarker' => '/* testNoParenthesesAndEmptyTokens */',
             ],
@@ -139,6 +142,9 @@ public static function dataAnonClassWithParentheses()
             'plain'                                              => [
                 'testMarker' => '/* testWithParentheses */',
             ],
+            'readonly'                                           => [
+                'testMarker' => '/* testReadonlyWithParentheses */',
+            ],
             'declaration contains comments and extra whitespace' => [
                 'testMarker' => '/* testWithParenthesesAndEmptyTokens */',
             ],

tests/Core/Tokenizer/BackfillReadonlyTest.inc

@@ -138,6 +138,19 @@ class ReadonlyWithDisjunctiveNormalForm
     public function readonly (A&B $param): void {}
 }
 
+/* testReadonlyAnonClassWithParens */
+$anon = new readonly class() {};
+
+/* testReadonlyAnonClassWithoutParens */
+$anon = new Readonly class {};
+
+/* testReadonlyAnonClassWithCommentsAndWhitespace */
+$anon = new
+// comment
+ReadOnly
+// phpcs:ignore Stnd.Cat.Sniff
+class {};
+
 /* testParseErrorLiveCoding */
 // This must be the last test in the file.
 readonly

tests/Core/Tokenizer/BackfillReadonlyTest.php

@@ -151,6 +151,17 @@ public static function dataReadonly()
             'property declaration, constructor property promotion, DNF type and reference'    => [
                 'testMarker' => '/* testReadonlyConstructorPropertyPromotionWithDNFAndReference */',
             ],
+            'anon class declaration, with parentheses'                                        => [
+                'testMarker' => '/* testReadonlyAnonClassWithParens */',
+            ],
+            'anon class declaration, without parentheses'                                     => [
+                'testMarker'  => '/* testReadonlyAnonClassWithoutParens */',
+                'testContent' => 'Readonly',
+            ],
+            'anon class declaration, with comments and whitespace'                            => [
+                'testMarker'  => '/* testReadonlyAnonClassWithCommentsAndWhitespace */',
+                'testContent' => 'ReadOnly',
+            ],
             'live coding / parse error'                                                       => [
                 'testMarker' => '/* testParseErrorLiveCoding */',
             ],

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

@@ -99,12 +99,16 @@ namespace /* testNamespaceNameIsString1 */ my\ /* testNamespaceNameIsString2 */
     /* testVarIsKeyword */ var $var;
     /* testStaticIsKeyword */ static $static;
 
-    /* testReadonlyIsKeyword */ readonly $readonly;
+    /* testReadonlyIsKeywordForProperty */ readonly $readonly;
 
     /* testFinalIsKeyword */ final /* testFunctionIsKeyword */ function someFunction(
         /* testCallableIsKeyword */
         callable $callable,
     ) {
+        $anon = new /* testReadonlyIsKeywordForAnonClass */ readonly class() {
+            public function foo() {}
+        };
+
         /* testReturnIsKeyword */
         return $this;
     }

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

@@ -232,7 +232,7 @@ public static function dataKeywords()
                 'expectedTokenType' => 'T_STATIC',
             ],
             'readonly: property declaration'         => [
-                'testMarker'        => '/* testReadonlyIsKeyword */',
+                'testMarker'        => '/* testReadonlyIsKeywordForProperty */',
                 'expectedTokenType' => 'T_READONLY',
             ],
             'final: function declaration'            => [
@@ -247,6 +247,10 @@ public static function dataKeywords()
                 'testMarker'        => '/* testCallableIsKeyword */',
                 'expectedTokenType' => 'T_CALLABLE',
             ],
+            ],
+            'readonly: anon class declaration'        => [
+                'testMarker'        => '/* testReadonlyIsKeywordForAnonClass */',
+                'expectedTokenType' => 'T_READONLY',
             'return: statement'                      => [
                 'testMarker'        => '/* testReturnIsKeyword */',
                 'expectedTokenType' => 'T_RETURN',