Fixed bug #78604

<?php followed by EOF is valid since PHP 7.4.

Author: nikic

NEWS

@@ -2,6 +2,10 @@ PHP                                                                        NEWS
 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
 ?? ??? ????, PHP 7.4.0RC3
 
+- Core:
+  . Fixed bug #78604 (token_get_all() does not properly tokenize FOO<?php with
+    short_open_tag=0). (Nikita)
+
 - FFI:
   . Fixed bug #78543 (is_callable() on FFI\CData throws Exception). (cmb)
 

Zend/zend_language_scanner.l

@@ -2057,7 +2057,8 @@ inline_char_handler:
 			if (CG(short_tags) /* <? */
 				|| (*(YYCURSOR + 1) == '=') /* <?= */
 				|| (!strncasecmp((char*)YYCURSOR + 1, "php", 3) && /* <?php[ \t\r\n] */
-					(YYCURSOR[4] == ' ' || YYCURSOR[4] == '\t' ||
+					(YYCURSOR + 4 == YYLIMIT ||
+					YYCURSOR[4] == ' ' || YYCURSOR[4] == '\t' ||
 					YYCURSOR[4] == '\n' || YYCURSOR[4] == '\r'))
 			) {
 				YYCURSOR--;

ext/tokenizer/tests/php_tag_only.phpt

@@ -2,6 +2,8 @@
 Tokenization of only the <?php tag
 --SKIPIF--
 <?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--INI--
+short_open_tag=1
 --FILE--
 <?php
 

ext/tokenizer/tests/php_tag_only_2.phpt

@@ -0,0 +1,23 @@
+--TEST--
+Tokenization of only the <?php tag
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--INI--
+short_open_tag=0
+--FILE--
+<?php
+
+foreach (token_get_all("<?php") as $token) {
+    echo token_name($token[0]), "\n";
+}
+echo "\n";
+foreach (token_get_all("Foobar<?php") as $token) {
+    echo token_name($token[0]), "\n";
+}
+
+?>
+--EXPECT--
+T_OPEN_TAG
+
+T_INLINE_HTML
+T_OPEN_TAG