Skip to content

Commit c2b6235

Browse files
jrfnljrfnl
authored
Merge pull request #647 from PHPCSStandards/feature/529-tokenizer-yield-from-tokenization-review
PHP 8.3 | Tokenizer/PHP: support "yield from" with comments
2 parents db1ebe1 + fc6d98a commit c2b6235

29 files changed

+682
-84
lines changed

‎src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public function process(File $phpcsFile, $stackPtr)
7878
T_COMMENT => true,
7979
T_END_HEREDOC => true,
8080
T_END_NOWDOC => true,
81+
T_YIELD_FROM => true,
8182
];
8283

8384
for ($i = 0; $i < $phpcsFile->numTokens; $i++) {

‎src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php

Lines changed: 43 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -80,42 +80,57 @@ public function process(File $phpcsFile, $stackPtr)
8080
if ($tokens[$stackPtr]['code'] === T_YIELD_FROM
8181
&& strtolower($content) !== 'yield from'
8282
) {
83-
if ($tokens[($stackPtr - 1)]['code'] === T_YIELD_FROM) {
84-
// A multi-line statement that has already been processed.
85-
return;
86-
}
83+
$found = $content;
84+
$hasComment = false;
85+
$yieldFromEnd = $stackPtr;
86+
87+
// Handle potentially multi-line/multi-token "yield from" expressions.
88+
if (preg_match('`yield\s+from`i', $content) !== 1) {
89+
for ($i = ($stackPtr + 1); $i < $phpcsFile->numTokens; $i++) {
90+
if (isset(Tokens::$emptyTokens[$tokens[$i]['code']]) === false
91+
&& $tokens[$i]['code'] !== T_YIELD_FROM
92+
) {
93+
break;
94+
}
95+
96+
if (isset(Tokens::$commentTokens[$tokens[$i]['code']]) === true) {
97+
$hasComment = true;
98+
}
8799

88-
$found = $content;
89-
if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
90-
// This yield from statement is split over multiple lines.
91-
$i = ($stackPtr + 1);
92-
do {
93100
$found .= $tokens[$i]['content'];
94-
$i++;
95-
} while ($tokens[$i]['code'] === T_YIELD_FROM);
96-
}
101+
102+
if ($tokens[$i]['code'] === T_YIELD_FROM
103+
&& strtolower(trim($tokens[$i]['content'])) === 'from'
104+
) {
105+
break;
106+
}
107+
}
108+
109+
$yieldFromEnd = $i;
110+
}//end if
97111

98112
$error = 'Language constructs must be followed by a single space; expected 1 space between YIELD FROM found "%s"';
99113
$data = [Common::prepareForOutput($found)];
100-
$fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
101-
if ($fix === true) {
102-
preg_match('/yield/i', $found, $yield);
103-
preg_match('/from/i', $found, $from);
104-
$phpcsFile->fixer->beginChangeset();
105-
$phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
106-
107-
if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
108-
$i = ($stackPtr + 1);
109-
do {
114+
115+
if ($hasComment === true) {
116+
$phpcsFile->addError($error, $stackPtr, 'IncorrectYieldFromWithComment', $data);
117+
} else {
118+
$fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
119+
if ($fix === true) {
120+
preg_match('/yield/i', $found, $yield);
121+
preg_match('/from/i', $found, $from);
122+
$phpcsFile->fixer->beginChangeset();
123+
$phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
124+
125+
for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) {
110126
$phpcsFile->fixer->replaceToken($i, '');
111-
$i++;
112-
} while ($tokens[$i]['code'] === T_YIELD_FROM);
113-
}
127+
}
114128

115-
$phpcsFile->fixer->endChangeset();
116-
}
129+
$phpcsFile->fixer->endChangeset();
130+
}
131+
}//end if
117132

118-
return;
133+
return ($yieldFromEnd + 1);
119134
}//end if
120135

121136
if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {

‎src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,12 @@ if (ISSET($a) && !Empty($a)) { UnSeT($a); }
5353
eval('foo');
5454
eVaL('foo');
5555

56+
$c = function() {
57+
Yield /*comment*/ From fun();
58+
YIELD
59+
/*comment*/
60+
FROM fun();
61+
}
62+
5663
__HALT_COMPILER(); // An exception due to phar support.
5764
function

‎src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,12 @@ if (isset($a) && !empty($a)) { unset($a); }
5353
eval('foo');
5454
eval('foo');
5555

56+
$c = function() {
57+
yield /*comment*/ from fun();
58+
yield
59+
/*comment*/
60+
from fun();
61+
}
62+
5663
__HALT_COMPILER(); // An exception due to phar support.
5764
function

‎src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ public function getErrorList()
5252
48 => 1,
5353
52 => 3,
5454
54 => 1,
55+
57 => 2,
56+
58 => 1,
57+
60 => 1,
5558
];
5659

5760
}//end getErrorList()

‎src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,10 @@ $x = 1;
116116
117117
Another line.
118118
*/
119+
120+
// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
121+
function myGenerator() {
122+
yield
123+
from
124+
gen2();
125+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,10 @@ $x = 1;
116116

117117
Another line.
118118
*/
119+
120+
// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
121+
function myGenerator() {
122+
yield
123+
from
124+
gen2();
125+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,10 @@ $x = 1;
116116
117117
Another line.
118118
*/
119+
120+
// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
121+
function myGenerator() {
122+
yield
123+
from
124+
gen2();
125+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,10 @@ $x = 1;
116116

117117
Another line.
118118
*/
119+
120+
// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
121+
function myGenerator() {
122+
yield
123+
from
124+
gen2();
125+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public function getErrorList($testFile='')
8989
115 => 1,
9090
117 => 1,
9191
118 => 1,
92+
123 => 1,
9293
];
9394

9495
case 'DisallowSpaceIndentUnitTest.3.inc':

‎src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,12 @@ $var = "$hello $there";
9191
9292
Another line.
9393
*/
94+
95+
// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords.
96+
function myGenerator() {
97+
yield from gen1();
98+
99+
yield
100+
from
101+
gen2();
102+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,12 @@ $var = "$hello $there";
9191

9292
Another line.
9393
*/
94+
95+
// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords.
96+
function myGenerator() {
97+
yield from gen1();
98+
99+
yield
100+
from
101+
gen2();
102+
}

‎src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php

Lines changed: 39 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -50,43 +50,45 @@ public function getErrorList($testFile='')
5050
switch ($testFile) {
5151
case 'DisallowTabIndentUnitTest.1.inc':
5252
return [
53-
5 => 2,
54-
9 => 1,
55-
15 => 1,
56-
20 => 2,
57-
21 => 1,
58-
22 => 2,
59-
23 => 1,
60-
24 => 2,
61-
31 => 1,
62-
32 => 2,
63-
33 => 2,
64-
41 => 1,
65-
42 => 1,
66-
43 => 1,
67-
44 => 1,
68-
45 => 1,
69-
46 => 1,
70-
47 => 1,
71-
48 => 1,
72-
54 => 1,
73-
55 => 1,
74-
56 => 1,
75-
57 => 1,
76-
58 => 1,
77-
59 => 1,
78-
79 => 1,
79-
80 => 1,
80-
81 => 1,
81-
82 => 1,
82-
83 => 1,
83-
85 => 1,
84-
86 => 1,
85-
87 => 1,
86-
89 => 1,
87-
90 => 1,
88-
92 => 1,
89-
93 => 1,
53+
5 => 2,
54+
9 => 1,
55+
15 => 1,
56+
20 => 2,
57+
21 => 1,
58+
22 => 2,
59+
23 => 1,
60+
24 => 2,
61+
31 => 1,
62+
32 => 2,
63+
33 => 2,
64+
41 => 1,
65+
42 => 1,
66+
43 => 1,
67+
44 => 1,
68+
45 => 1,
69+
46 => 1,
70+
47 => 1,
71+
48 => 1,
72+
54 => 1,
73+
55 => 1,
74+
56 => 1,
75+
57 => 1,
76+
58 => 1,
77+
59 => 1,
78+
79 => 1,
79+
80 => 1,
80+
81 => 1,
81+
82 => 1,
82+
83 => 1,
83+
85 => 1,
84+
86 => 1,
85+
87 => 1,
86+
89 => 1,
87+
90 => 1,
88+
92 => 1,
89+
93 => 1,
90+
97 => 1,
91+
100 => 1,
9092
];
9193

9294
case 'DisallowTabIndentUnitTest.2.inc':

‎src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,12 @@ $newLine;
8989
// The following line must have a single space at the end (after return)
9090
return
9191
$spaceAndNewLine;
92+
93+
// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment.
94+
yield /*comment*/ from $test();
95+
yield
96+
# comment
97+
from $test();
98+
yield
99+
// phpcs:ignore Stnd.Category.SniffName
100+
from $test();

‎src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,12 @@ return $newLine;
8383

8484
// The following line must have a single space at the end (after return)
8585
return $spaceAndNewLine;
86+
87+
// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment.
88+
yield /*comment*/ from $test();
89+
yield
90+
# comment
91+
from $test();
92+
yield
93+
// phpcs:ignore Stnd.Category.SniffName
94+
from $test();

‎src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ public function getErrorList($testFile='')
7171
85 => 1,
7272
86 => 1,
7373
90 => 1,
74+
94 => 1,
75+
95 => 1,
76+
98 => 1,
7477
];
7578

7679
default:

‎src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,6 +1614,12 @@ match (true) {
16141614
]
16151615
};
16161616

1617+
// Issue squizlabs/PHP_CodeSniffer#3808
1618+
function test() {
1619+
yield
1620+
from [ 3, 4 ];
1621+
}
1622+
16171623
/* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
16181624
?>
16191625

‎src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,6 +1614,12 @@ match (true) {
16141614
]
16151615
};
16161616

1617+
// Issue squizlabs/PHP_CodeSniffer#3808
1618+
function test() {
1619+
yield
1620+
from [ 3, 4 ];
1621+
}
1622+
16171623
/* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
16181624
?>
16191625

‎src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,6 +1614,12 @@ match (true) {
16141614
]
16151615
};
16161616

1617+
// Issue squizlabs/PHP_CodeSniffer#3808
1618+
function test() {
1619+
yield
1620+
from [ 3, 4 ];
1621+
}
1622+
16171623
/* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
16181624
?>
16191625

‎src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1614,6 +1614,12 @@ match (true) {
16141614
]
16151615
};
16161616

1617+
// Issue squizlabs/PHP_CodeSniffer#3808
1618+
function test() {
1619+
yield
1620+
from [ 3, 4 ];
1621+
}
1622+
16171623
/* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
16181624
?>
16191625

‎src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
2-
phpcs:set Generic.WhiteSpace.ScopeIndent exact true
31
<?php
2+
// phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
3+
// phpcs:set Generic.WhiteSpace.ScopeIndent exact true
44
function test()
55
{
66
echo 'test';
@@ -26,3 +26,9 @@ if ($foo) {
2626
$this->foo()
2727
->bar()
2828
->baz();
29+
30+
// Issue squizlabs/PHP_CodeSniffer#3808
31+
function test() {
32+
yield
33+
from [ 3, 4 ];
34+
}

0 commit comments

Comments
 (0)