Use ENT_QUOTES|ENT_SUBSTITUTE default for HTML encoding and decoding functions

htmlspecialchars() etc now use ENT_QUOTES | ENT_SUBSTITUTE rather
than ENT_COMPAT by default.

Closes GH-6583.

Author: craigfrancis

UPGRADING

@@ -69,6 +69,12 @@ PHP 8.1 UPGRADE NOTES
 
 - Standard:
   . version_compare() no longer accepts undocumented operator abbreviations.
+  . htmlspecialchars(), htmlentities(), htmlspecialchars_decode(),
+    html_entitity_decode() and get_html_translation_table() now use
+    ENT_QUOTES | ENT_SUBSTITUTE rather than ENT_COMPAT by default. This means
+    that ' is escaped to ' while previously it was left alone.
+    Additionally, malformed UTF-8 will be replaced by a Unicode substitution
+    character, instead of resulting in an empty string.
 
 ========================================
 2. New Features

ext/standard/basic_functions.stub.php

@@ -512,15 +512,15 @@ function headers_list(): array {}
 
 /* {{{ html.c */
 
-function htmlspecialchars(string $string, int $flags = ENT_COMPAT, ?string $encoding = null, bool $double_encode = true): string {}
+function htmlspecialchars(string $string, int $flags = ENT_QUOTES | ENT_SUBSTITUTE, ?string $encoding = null, bool $double_encode = true): string {}
 
-function htmlspecialchars_decode(string $string, int $flags = ENT_COMPAT): string {}
+function htmlspecialchars_decode(string $string, int $flags = ENT_QUOTES | ENT_SUBSTITUTE): string {}
 
-function html_entity_decode(string $string, int $flags = ENT_COMPAT, ?string $encoding = null): string {}
+function html_entity_decode(string $string, int $flags = ENT_QUOTES | ENT_SUBSTITUTE, ?string $encoding = null): string {}
 
-function htmlentities(string $string, int $flags = ENT_COMPAT, ?string $encoding = null, bool $double_encode = true): string {}
+function htmlentities(string $string, int $flags = ENT_QUOTES | ENT_SUBSTITUTE, ?string $encoding = null, bool $double_encode = true): string {}
 
-function get_html_translation_table(int $table = HTML_SPECIALCHARS, int $flags = ENT_COMPAT, string $encoding = "UTF-8"): array {}
+function get_html_translation_table(int $table = HTML_SPECIALCHARS, int $flags = ENT_QUOTES | ENT_SUBSTITUTE, string $encoding = "UTF-8"): array {}
 
 /* }}} */
 

ext/standard/basic_functions_arginfo.h

@@ -765,27 +765,27 @@ ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_htmlspecialchars, 0, 1, IS_STRING, 0)
 	ZEND_ARG_TYPE_INFO(0, string, IS_STRING, 0)
-	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_COMPAT")
+	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_QUOTES | ENT_SUBSTITUTE")
 	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, encoding, IS_STRING, 1, "null")
 	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, double_encode, _IS_BOOL, 0, "true")
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_htmlspecialchars_decode, 0, 1, IS_STRING, 0)
 	ZEND_ARG_TYPE_INFO(0, string, IS_STRING, 0)
-	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_COMPAT")
+	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_QUOTES | ENT_SUBSTITUTE")
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_html_entity_decode, 0, 1, IS_STRING, 0)
 	ZEND_ARG_TYPE_INFO(0, string, IS_STRING, 0)
-	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_COMPAT")
+	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_QUOTES | ENT_SUBSTITUTE")
 	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, encoding, IS_STRING, 1, "null")
 ZEND_END_ARG_INFO()
 
 #define arginfo_htmlentities arginfo_htmlspecialchars
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_get_html_translation_table, 0, 0, IS_ARRAY, 0)
 	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, table, IS_LONG, 0, "HTML_SPECIALCHARS")
-	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_COMPAT")
+	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "ENT_QUOTES | ENT_SUBSTITUTE")
 	ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, encoding, IS_STRING, 0, "\"UTF-8\"")
 ZEND_END_ARG_INFO()
 

ext/standard/html.c

@@ -1316,7 +1316,7 @@ PHPAPI zend_string *php_escape_html_entities_ex(const unsigned char *old, size_t
 static void php_html_entities(INTERNAL_FUNCTION_PARAMETERS, int all)
 {
 	zend_string *str, *hint_charset = NULL;
-	zend_long flags = ENT_COMPAT;
+	zend_long flags = ENT_QUOTES|ENT_SUBSTITUTE;
 	zend_string *replaced;
 	bool double_encode = 1;
 
@@ -1367,7 +1367,7 @@ PHP_FUNCTION(htmlspecialchars)
 PHP_FUNCTION(htmlspecialchars_decode)
 {
 	zend_string *str;
-	zend_long quote_style = ENT_COMPAT;
+	zend_long quote_style = ENT_QUOTES|ENT_SUBSTITUTE;
 	zend_string *replaced;
 
 	ZEND_PARSE_PARAMETERS_START(1, 2)
@@ -1385,7 +1385,7 @@ PHP_FUNCTION(htmlspecialchars_decode)
 PHP_FUNCTION(html_entity_decode)
 {
 	zend_string *str, *hint_charset = NULL;
-	zend_long quote_style = ENT_COMPAT;
+	zend_long quote_style = ENT_QUOTES|ENT_SUBSTITUTE;
 	zend_string *replaced;
 
 	ZEND_PARSE_PARAMETERS_START(1, 3)
@@ -1468,7 +1468,7 @@ static inline void write_s3row_data(
 PHP_FUNCTION(get_html_translation_table)
 {
 	zend_long all = HTML_SPECIALCHARS,
-		 flags = ENT_COMPAT;
+		 flags = ENT_QUOTES|ENT_SUBSTITUTE;
 	int doctype;
 	entity_table_opt entity_table;
 	const enc_to_uni *to_uni_table = NULL;

ext/standard/tests/strings/bug53021.phpt

@@ -38,4 +38,4 @@ single quotes variations:
 '
 '
 '
-'
+'

ext/standard/tests/strings/bug61116.phpt

@@ -10,7 +10,7 @@ Function [ <internal:standard> function htmlspecialchars ] {
 
   - Parameters [4] {
     Parameter #0 [ <required> string $string ]
-    Parameter #1 [ <optional> int $flags = ENT_COMPAT ]
+    Parameter #1 [ <optional> int $flags = ENT_QUOTES | ENT_SUBSTITUTE ]
     Parameter #2 [ <optional> ?string $encoding = null ]
     Parameter #3 [ <optional> bool $double_encode = true ]
   }
@@ -21,7 +21,7 @@ Function [ <internal:standard> function get_html_translation_table ] {
 
   - Parameters [3] {
     Parameter #0 [ <optional> int $table = HTML_SPECIALCHARS ]
-    Parameter #1 [ <optional> int $flags = ENT_COMPAT ]
+    Parameter #1 [ <optional> int $flags = ENT_QUOTES | ENT_SUBSTITUTE ]
     Parameter #2 [ <optional> string $encoding = "UTF-8" ]
   }
   - Return [ array ]

ext/standard/tests/strings/html_entity_decode3.phpt

@@ -218,7 +218,7 @@ echo "\nDone.\n";
 	NOT DECODED
 	NOT DECODED
  	DECODED
-'	NOT DECODED
+'	DECODED
 	NOT DECODED
 €	NOT DECODED
 Ÿ	NOT DECODED

ext/standard/tests/strings/htmlentities24.phpt

@@ -310,7 +310,7 @@ string(198) "‚†™Ÿ€‚†„€&perm
 string(42) "<html> This is a test! </html>"
 
 *** Testing htmlentites() on a quote ***
-string(36) "A 'quote' is <b>bold</b>"
+string(46) "A 'quote' is <b>bold</b>"
 string(46) "A 'quote' is <b>bold</b>"
 string(36) "A 'quote' is <b>bold</b>"
 string(36) "A 'quote' is <b>bold</b>"

ext/standard/tests/strings/htmlspecialchars.phpt

@@ -306,7 +306,7 @@ string(187) "<br>Testing<p>New file.</p><p><br>Fil
 string(46) "<br>Testing<p>New file.</p> "
 
 *** Testing htmlspecialchars() on a quote...
-string(36) "A 'quote' is <b>bold</b>"
+string(46) "A 'quote' is <b>bold</b>"
 string(46) "A 'quote' is <b>bold</b>"
 string(36) "A 'quote' is <b>bold</b>"
 string(36) "A 'quote' is <b>bold</b>"

ext/standard/tests/strings/htmlspecialchars_basic.phpt

@@ -56,7 +56,7 @@ Basic tests
 Test 1: abc<>"&
 Test 2: &&abc<>"&
 Test 3: a>,\<bc<>"&
-Test 4: a\'\'&bc<>"&
+Test 4: a\'\'&bc<>"&
 Test 5: &<
 Test 6: abc<>"&
 Test 7: &&abc<>"&

ext/standard/tests/strings/htmlspecialchars_decode_basic.phpt

@@ -26,8 +26,8 @@ echo "Done";
 ?>
 --EXPECT--
 *** Testing htmlspecialchars_decode() : basic functionality ***
-string(92) "Roy&#039;s height > Sam&#039;s height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
-string(92) "Roy&#039;s height > Sam&#039;s height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
+string(82) "Roy's height > Sam's height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
+string(82) "Roy's height > Sam's height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
 string(92) "Roy&#039;s height > Sam&#039;s height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
 string(92) "Roy&#039;s height > Sam&#039;s height. 13 < 25. 1111 & 0000 = 0000. " double quoted string ""
 string(102) "Roy&#039;s height > Sam&#039;s height. 13 < 25. 1111 & 0000 = 0000. &quot; double quoted string &quot;"

ext/standard/tests/strings/htmlspecialchars_decode_variation3.phpt

@@ -73,12 +73,12 @@ string(0) ""
 -- Iteration 2 --
 string(0) ""
 -- Iteration 3 --
-string(103) "<html>Roy&#039;s height > Sam&#039;s height
+string(93) "<html>Roy's height > Sam's height
 13 < 25
 1111 & 0000 = 0000
 "This is a double quoted string""
 -- Iteration 4 --
-string(130) "<html>Roy&#039;s height > Sam	&#039;s height
+string(120) "<html>Roy's height > Sam	's height
 1111		 & 0000 = 0000
 " heredoc
 double quoted string. withdifferentwhitespaces""
@@ -87,8 +87,8 @@ string(62) "<html>11 < 12. 123 string 4567
 "string" 1111	 & 0000	 = 0000
 ;"
 -- Iteration 6 --
-string(153) "<html>< This's a string with quotes:
+string(143) "<html>< This's a string with quotes:
 "strings in double quote" &
 'strings in single quote' "
-this\line is &#039;single quoted&#039; /with\slashes </html>"
+this\line is 'single quoted' /with\slashes </html>"
 Done

ext/standard/tests/strings/htmlspecialchars_decode_variation4.phpt

@@ -33,22 +33,22 @@ echo "Done";
 --EXPECT--
 *** Testing htmlspecialchars_decode() : usage variations ***
 -- Iteration 1 --
-string(90) "Roy&#039s height > Sam's \$height... 1111 ≈ 0000 = 0000... " double quote string ""
+string(85) "Roy&#039s height > Sam's \$height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(90) "Roy&#039s height > Sam's \$height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(100) "Roy&#039s height > Sam's \$height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(85) "Roy&#039s height > Sam's \$height... 1111 ≈ 0000 = 0000... " double quote string ""
 -- Iteration 2 --
-string(88) "Roy&#039;s height > Sam&#039;s height... \t\t 13 < 15...\n\r " double quote\f\v string ""
+string(78) "Roy's height > Sam's height... \t\t 13 < 15...\n\r " double quote\f\v string ""
 string(88) "Roy&#039;s height > Sam&#039;s height... \t\t 13 < 15...\n\r " double quote\f\v string ""
 string(98) "Roy&#039;s height > Sam&#039;s height... \t\t 13 < 15...\n\r &quot; double quote\f\v string &quot;"
 string(78) "Roy's height > Sam's height... \t\t 13 < 15...\n\r " double quote\f\v string ""
 -- Iteration 3 --
-string(48) "\nRoy&#039;s height &gt\t; Sam&#039;s\v height\f"
+string(38) "\nRoy's height &gt\t; Sam's\v height\f"
 string(48) "\nRoy&#039;s height &gt\t; Sam&#039;s\v height\f"
 string(48) "\nRoy&#039;s height &gt\t; Sam&#039;s\v height\f"
 string(38) "\nRoy's height &gt\t; Sam's\v height\f"
 -- Iteration 4 --
-string(48) "\r\tRoy&#039;s height &gt\r; Sam\t&#039;s height"
+string(38) "\r\tRoy's height &gt\r; Sam\t's height"
 string(48) "\r\tRoy&#039;s height &gt\r; Sam\t&#039;s height"
 string(48) "\r\tRoy&#039;s height &gt\r; Sam\t&#039;s height"
 string(38) "\r\tRoy's height &gt\r; Sam\t's height"

ext/standard/tests/strings/htmlspecialchars_decode_variation5.phpt

@@ -32,12 +32,12 @@ echo "Done";
 --EXPECT--
 *** Testing htmlspecialchars_decode() : usage variations ***
 -- Iteration 1 --
-string(89) "Roy&#039s height > Sam's $height... 1111 ≈ 0000 = 0000... " double quote string ""
+string(84) "Roy&#039s height > Sam's $height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(89) "Roy&#039s height > Sam's $height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(99) "Roy&#039s height > Sam's $height... 1111 ≈ 0000 = 0000... " double quote string ""
 string(84) "Roy&#039s height > Sam's $height... 1111 ≈ 0000 = 0000... " double quote string ""
 -- Iteration 2 --
-string(82) "Roy&#039;s height > Sam&#039;s height... 		 13 < 15...
+string(72) "Roy's height > Sam's height... 		 13 < 15...
  " double quote string ""
 string(82) "Roy&#039;s height > Sam&#039;s height... 		 13 < 15...
  " double quote string ""
@@ -46,16 +46,16 @@ string(92) "Roy&#039;s height > Sam&#039;s height... 		 13 < 15...
 string(72) "Roy's height > Sam's height... 		 13 < 15...
  " double quote string ""
 -- Iteration 3 --
-string(44) "
-Roy&#039;s height &gt	; Sam&#039;s height"
+string(34) "
+Roy's height &gt	; Sam's height"
 string(44) "
 Roy&#039;s height &gt	; Sam&#039;s height"
 string(44) "
 Roy&#039;s height &gt	; Sam&#039;s height"
 string(34) "
 Roy's height &gt	; Sam's height"
 -- Iteration 4 --
-string(44) "	Roy&#039;s height &gt; Sam	&#039;s height"
+string(34) "	Roy's height &gt; Sam	's height"
 string(44) "	Roy&#039;s height &gt; Sam	&#039;s height"
 string(44) "	Roy&#039;s height &gt; Sam	&#039;s height"
 string(34) "	Roy's height &gt; Sam	's height"