Skip to content

Commit 53dd20c

Browse files
jrfnljrfnl
committed
PHP 8.3 | Tokenizer/PHP: add support for typed OO constants
PHP 8.3 introduced typed OO constants, where the type is between the `const` keyword and the constant name. All type variations are supported, including nullable types, union types, intersection types, with the exception of `callable`, `void` and `never`. `self` and `static` types are only allowed in Enum constants. This PR adds support for typed OO constants in the Tokenizer layer of PHPCS. The following issues had to be fixed to support typed constants: 1. Consistently tokenizing the constant _name_ as `T_STRING`, even if the name mirrors a reserved keyword, like `foreach` or a special keyword, like `self` or `true`. 2. Tokenizing a `?` at the start of a constant type declaration as `T_NULLABLE`. 3. Tokenizing a `|` and `&` operators within a constant type declaration as `T_TYPE_UNION` and `T_TYPE_INTERSECTION` respectively. Each and every part of the above has been covered by extensive tests. Includes additional tests safeguarding that the `array` keyword when used in a type declaration for a constant is tokenized as `T_STRING`. Ref: https://wiki.php.net/rfc/typed_class_constants
1 parent 10249a2 commit 53dd20c

13 files changed

+963
-8
lines changed

‎src/Tokenizers/PHP.php

Lines changed: 98 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -526,8 +526,9 @@ protected function tokenize($string)
526526
$numTokens = count($tokens);
527527
$lastNotEmptyToken = 0;
528528

529-
$insideInlineIf = [];
530-
$insideUseGroup = false;
529+
$insideInlineIf = [];
530+
$insideUseGroup = false;
531+
$insideConstDeclaration = false;
531532

532533
$commentTokenizer = new Comment();
533534

@@ -608,7 +609,8 @@ protected function tokenize($string)
608609
if ($tokenIsArray === true
609610
&& isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
610611
&& (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
611-
|| $finalTokens[$lastNotEmptyToken]['content'] === '&')
612+
|| $finalTokens[$lastNotEmptyToken]['content'] === '&'
613+
|| $insideConstDeclaration === true)
612614
) {
613615
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
614616
$preserveKeyword = false;
@@ -665,6 +667,30 @@ protected function tokenize($string)
665667
}
666668
}//end if
667669

670+
// Types in typed constants should not be touched, but the constant name should be.
671+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
672+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
673+
|| $insideConstDeclaration === true
674+
) {
675+
$preserveKeyword = true;
676+
677+
// Find the next non-empty token.
678+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
679+
if (is_array($tokens[$i]) === true
680+
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
681+
) {
682+
continue;
683+
}
684+
685+
break;
686+
}
687+
688+
if ($tokens[$i] === '=' || $tokens[$i] === ';') {
689+
$preserveKeyword = false;
690+
$insideConstDeclaration = false;
691+
}
692+
}//end if
693+
668694
if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
669695
$preserveKeyword = true;
670696

@@ -698,6 +724,26 @@ protected function tokenize($string)
698724
}
699725
}//end if
700726

727+
/*
728+
Mark the start of a constant declaration to allow for handling keyword to T_STRING
729+
convertion for constant names using reserved keywords.
730+
*/
731+
732+
if ($tokenIsArray === true && $token[0] === T_CONST) {
733+
$insideConstDeclaration = true;
734+
}
735+
736+
/*
737+
Close an open "inside constant declaration" marker when no keyword convertion was needed.
738+
*/
739+
740+
if ($insideConstDeclaration === true
741+
&& $tokenIsArray === false
742+
&& ($token[0] === '=' || $token[0] === ';')
743+
) {
744+
$insideConstDeclaration = false;
745+
}
746+
701747
/*
702748
Special case for `static` used as a function name, i.e. `static()`.
703749
*/
@@ -1869,6 +1915,20 @@ protected function tokenize($string)
18691915
$newToken = [];
18701916
$newToken['content'] = '?';
18711917

1918+
// For typed constants, we only need to check the token before the ? to be sure.
1919+
if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
1920+
$newToken['code'] = T_NULLABLE;
1921+
$newToken['type'] = 'T_NULLABLE';
1922+
1923+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1924+
echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
1925+
}
1926+
1927+
$finalTokens[$newStackPtr] = $newToken;
1928+
$newStackPtr++;
1929+
continue;
1930+
}
1931+
18721932
/*
18731933
* Check if the next non-empty token is one of the tokens which can be used
18741934
* in type declarations. If not, it's definitely a ternary.
@@ -2236,7 +2296,30 @@ function return types. We want to keep the parenthesis map clean,
22362296
if ($tokenIsArray === true && $token[0] === T_STRING) {
22372297
$preserveTstring = false;
22382298

2239-
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
2299+
// True/false/parent/self/static in typed constants should be fixed to their own token,
2300+
// but the constant name should not be.
2301+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2302+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
2303+
|| $insideConstDeclaration === true
2304+
) {
2305+
// Find the next non-empty token.
2306+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
2307+
if (is_array($tokens[$i]) === true
2308+
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
2309+
) {
2310+
continue;
2311+
}
2312+
2313+
break;
2314+
}
2315+
2316+
if ($tokens[$i] === '=') {
2317+
$preserveTstring = true;
2318+
$insideConstDeclaration = false;
2319+
}
2320+
} else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2321+
&& $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
2322+
) {
22402323
$preserveTstring = true;
22412324

22422325
// Special case for syntax like: return new self/new parent
@@ -3008,6 +3091,12 @@ protected function processAdditional()
30083091
$suspectedType = 'return';
30093092
}
30103093

3094+
if ($this->tokens[$x]['code'] === T_EQUAL) {
3095+
// Possible constant declaration, the `T_STRING` name will have been skipped over already.
3096+
$suspectedType = 'constant';
3097+
break;
3098+
}
3099+
30113100
break;
30123101
}//end for
30133102

@@ -3049,6 +3138,11 @@ protected function processAdditional()
30493138
break;
30503139
}
30513140

3141+
if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
3142+
$confirmed = true;
3143+
break;
3144+
}
3145+
30523146
if ($suspectedType === 'property or parameter'
30533147
&& (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
30543148
|| $this->tokens[$x]['code'] === T_VAR

‎tests/Core/Tokenizer/ArrayKeywordTest.inc

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,21 @@ $var = array(
2121
);
2222

2323
/* testFunctionDeclarationParamType */
24-
function foo(array $a) {}
24+
function typedParam(array $a) {}
2525

2626
/* testFunctionDeclarationReturnType */
27-
function foo($a) : int|array|null {}
27+
function returnType($a) : int|array|null {}
2828

2929
class Bar {
3030
/* testClassConst */
3131
const ARRAY = [];
3232

3333
/* testClassMethod */
3434
public function array() {}
35+
36+
/* testOOConstType */
37+
const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();
38+
39+
/* testOOPropertyType */
40+
protected array $property;
3541
}

‎tests/Core/Tokenizer/ArrayKeywordTest.php

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ public static function dataArrayKeyword()
6868
'nested: inner array' => [
6969
'testMarker' => '/* testNestedArray */',
7070
],
71+
'OO constant default value' => [
72+
'testMarker' => '/* testOOConstDefault */',
73+
],
7174
];
7275

7376
}//end dataArrayKeyword()
@@ -122,6 +125,12 @@ public static function dataArrayType()
122125
'function union return type' => [
123126
'testMarker' => '/* testFunctionDeclarationReturnType */',
124127
],
128+
'OO constant type' => [
129+
'testMarker' => '/* testOOConstType */',
130+
],
131+
'OO property type' => [
132+
'testMarker' => '/* testOOPropertyType */',
133+
],
125134
];
126135

127136
}//end dataArrayType()
@@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
167176
public static function dataNotArrayKeyword()
168177
{
169178
return [
170-
'class-constant-name' => [
179+
'class-constant-name' => [
171180
'testMarker' => '/* testClassConst */',
172181
'testContent' => 'ARRAY',
173182
],
174-
'class-method-name' => [
183+
'class-method-name' => [
175184
'testMarker' => '/* testClassMethod */',
176185
],
186+
'class-constant-name-after-type' => [
187+
'testMarker' => '/* testTypedOOConstName */',
188+
'testContent' => 'ARRAY',
189+
],
177190
];
178191

179192
}//end dataNotArrayKeyword()

‎tests/Core/Tokenizer/BitwiseOrTest.inc

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;
99

1010
class TypeUnion
1111
{
12+
/* testTypeUnionOOConstSimple */
13+
public const Foo|Bar SIMPLE = new Foo;
14+
15+
/* testTypeUnionOOConstReverseModifierOrder */
16+
protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;
17+
18+
const
19+
/* testTypeUnionOOConstMulti1 */
20+
array |
21+
/* testTypeUnionOOConstMulti2 */
22+
Traversable | // phpcs:ignore Stnd.Cat.Sniff
23+
false
24+
/* testTypeUnionOOConstMulti3 */
25+
| null MULTI_UNION = false;
26+
27+
/* testTypeUnionOOConstNamespaceRelative */
28+
final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
29+
30+
/* testTypeUnionOOConstPartiallyQualified */
31+
const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
32+
33+
/* testTypeUnionOOConstFullyQualified */
34+
const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();
35+
1236
/* testTypeUnionPropertySimple */
1337
public static Foo|Bar $obj;
1438

‎tests/Core/Tokenizer/BitwiseOrTest.php

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public static function dataBitwiseOr()
4747
return [
4848
'in simple assignment 1' => ['/* testBitwiseOr1 */'],
4949
'in simple assignment 2' => ['/* testBitwiseOr2 */'],
50+
'in OO constant default value' => ['/* testBitwiseOrOOConstDefaultValue */'],
5051
'in property default value' => ['/* testBitwiseOrPropertyDefaultValue */'],
5152
'in method parameter default value' => ['/* testBitwiseOrParamDefaultValue */'],
5253
'in return statement' => ['/* testBitwiseOr3 */'],
@@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
9798
public static function dataTypeUnion()
9899
{
99100
return [
101+
'type for OO constant' => ['/* testTypeUnionOOConstSimple */'],
102+
'type for OO constant, reversed modifier order' => ['/* testTypeUnionOOConstReverseModifierOrder */'],
103+
'type for OO constant, first of multi-union' => ['/* testTypeUnionOOConstMulti1 */'],
104+
'type for OO constant, middle of multi-union + comments' => ['/* testTypeUnionOOConstMulti2 */'],
105+
'type for OO constant, last of multi-union' => ['/* testTypeUnionOOConstMulti3 */'],
106+
'type for OO constant, using namespace relative names' => ['/* testTypeUnionOOConstNamespaceRelative */'],
107+
'type for OO constant, using partially qualified names' => ['/* testTypeUnionOOConstPartiallyQualified */'],
108+
'type for OO constant, using fully qualified names' => ['/* testTypeUnionOOConstFullyQualified */'],
100109
'type for static property' => ['/* testTypeUnionPropertySimple */'],
101110
'type for static property, reversed modifier order' => ['/* testTypeUnionPropertyReverseModifierOrder */'],
102111
'type for property, first of multi-union' => ['/* testTypeUnionPropertyMulti1 */'],

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ class ContextSensitiveKeywords
7676
const /* testAnd */ AND = 'LOGICAL_AND';
7777
const /* testOr */ OR = 'LOGICAL_OR';
7878
const /* testXor */ XOR = 'LOGICAL_XOR';
79+
80+
const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
81+
const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;
82+
83+
const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
84+
const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
7985
}
8086

8187
namespace /* testKeywordAfterNamespaceShouldBeString */ class;

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ public static function dataStrings()
118118
'constant declaration: or' => ['/* testOr */'],
119119
'constant declaration: xor' => ['/* testXor */'],
120120

121+
'constant declaration: array in type' => ['/* testArrayIsTstringInConstType */'],
122+
'constant declaration: array, name after type' => ['/* testArrayNameForTypedConstant */'],
123+
'constant declaration: static, name after type' => ['/* testStaticIsNameForTypedConstant */'],
124+
'constant declaration: private, name after type' => ['/* testPrivateNameForUnionTypedConstant */'],
125+
'constant declaration: final, name after type' => ['/* testFinalNameForIntersectionTypedConstant */'],
126+
121127
'namespace declaration: class' => ['/* testKeywordAfterNamespaceShouldBeString */'],
122128
'namespace declaration (partial): my' => ['/* testNamespaceNameIsString1 */'],
123129
'namespace declaration (partial): class' => ['/* testNamespaceNameIsString2 */'],
@@ -179,6 +185,19 @@ public static function dataKeywords()
179185
'testMarker' => '/* testNamespaceIsKeyword */',
180186
'expectedTokenType' => 'T_NAMESPACE',
181187
],
188+
'array: default value in const decl' => [
189+
'testMarker' => '/* testArrayIsKeywordInConstDefault */',
190+
'expectedTokenType' => 'T_ARRAY',
191+
],
192+
'static: type in constant declaration' => [
193+
'testMarker' => '/* testStaticIsKeywordAsConstType */',
194+
'expectedTokenType' => 'T_STATIC',
195+
],
196+
'static: value in constant declaration' => [
197+
'testMarker' => '/* testStaticIsKeywordAsConstDefault */',
198+
'expectedTokenType' => 'T_STATIC',
199+
],
200+
182201
'abstract: class declaration' => [
183202
'testMarker' => '/* testAbstractIsKeyword */',
184203
'expectedTokenType' => 'T_ABSTRACT',

‎tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
5151
|| $a === /* testNullIsKeywordInComparison */ null
5252
) {}
5353
}
54+
55+
class TypedConstProp {
56+
const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
57+
const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
58+
const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
59+
const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
60+
const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;
61+
62+
public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
63+
protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
64+
static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
65+
var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
66+
protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
67+
}

0 commit comments

Comments
 (0)