Skip to content

Commit 4fb718d

Browse files
author
Gusted
authored
Don't treat BOM escape sequence as hidden character. (#18909) (#18910)
* Don't treat BOM escape sequence as hidden character. (#18909) Backport #18909
1 parent df35049 commit 4fb718d

File tree

2 files changed

+27
-5
lines changed

2 files changed

+27
-5
lines changed

modules/charset/escape.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ func EscapeControlBytes(text []byte) (EscapeStatus, []byte) {
6363
func EscapeControlReader(text io.Reader, output io.Writer) (escaped EscapeStatus, err error) {
6464
buf := make([]byte, 4096)
6565
readStart := 0
66+
runeCount := 0
6667
var n int
6768
var writePos int
6869

@@ -79,6 +80,8 @@ readingloop:
7980

8081
for i < len(bs) {
8182
r, size := utf8.DecodeRune(bs[i:])
83+
runeCount++
84+
8285
// Now handle the codepoints
8386
switch {
8487
case r == utf8.RuneError:
@@ -113,6 +116,8 @@ readingloop:
113116
lineHasRTLScript = false
114117
lineHasLTRScript = false
115118

119+
case runeCount == 1 && r == 0xFEFF: // UTF BOM
120+
// the first BOM is safe
116121
case r == '\r' || r == '\t' || r == ' ':
117122
// These are acceptable control characters and space characters
118123
case unicode.IsSpace(r):
@@ -144,7 +149,8 @@ readingloop:
144149
return
145150
}
146151
writePos = i + size
147-
case unicode.Is(unicode.C, r):
152+
// 65279 == BOM rune.
153+
case unicode.Is(unicode.C, r) && r != rune(65279):
148154
escaped.Escaped = true
149155
escaped.HasControls = true
150156
if writePos < i {

modules/charset/escape_test.go

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,14 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
129129
"\n" + `if access_level != "user<span class="escaped-code-point" data-escaped="[U+202E]"><span class="char">` + "\u202e" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>// Check if admin<span class="escaped-code-point" data-escaped="[U+2069]"><span class="char">` + "\u2069" + `</span></span> <span class="escaped-code-point" data-escaped="[U+2066]"><span class="char">` + "\u2066" + `</span></span>" {` + "\n",
130130
status: EscapeStatus{Escaped: true, HasBIDI: true, BadBIDI: true, HasLTRScript: true, HasRTLScript: true},
131131
},
132+
{
133+
// UTF-8/16/32 all use the same codepoint for BOM
134+
// Gitea could read UTF-16/32 content and convert into UTF-8 internally then render it, so we only process UTF-8 internally
135+
name: "UTF BOM",
136+
text: "\xef\xbb\xbftest",
137+
result: "\xef\xbb\xbftest",
138+
status: EscapeStatus{HasLTRScript: true},
139+
},
132140
}
133141

134142
func TestEscapeControlString(t *testing.T) {
@@ -163,19 +171,27 @@ func TestEscapeControlReader(t *testing.T) {
163171
// lets add some control characters to the tests
164172
tests := make([]escapeControlTest, 0, len(escapeControlTests)*3)
165173
copy(tests, escapeControlTests)
174+
175+
// if there is a BOM, we should keep the BOM
176+
addPrefix := func(prefix, s string) string {
177+
if strings.HasPrefix(s, "\xef\xbb\xbf") {
178+
return s[:3] + prefix + s[3:]
179+
}
180+
return prefix + s
181+
}
166182
for _, test := range escapeControlTests {
167183
test.name += " (+Control)"
168-
test.text = "\u001E" + test.text
169-
test.result = `<span class="escaped-code-point" data-escaped="[U+001E]"><span class="char">` + "\u001e" + `</span></span>` + test.result
184+
test.text = addPrefix("\u001E", test.text)
185+
test.result = addPrefix(`<span class="escaped-code-point" data-escaped="[U+001E]"><span class="char">`+"\u001e"+`</span></span>`, test.result)
170186
test.status.Escaped = true
171187
test.status.HasControls = true
172188
tests = append(tests, test)
173189
}
174190

175191
for _, test := range escapeControlTests {
176192
test.name += " (+Mark)"
177-
test.text = "\u0300" + test.text
178-
test.result = `<span class="escaped-code-point" data-escaped="[U+0300]"><span class="char">` + "\u0300" + `</span></span>` + test.result
193+
test.text = addPrefix("\u0300", test.text)
194+
test.result = addPrefix(`<span class="escaped-code-point" data-escaped="[U+0300]"><span class="char">`+"\u0300"+`</span></span>`, test.result)
179195
test.status.Escaped = true
180196
test.status.HasMarks = true
181197
tests = append(tests, test)

0 commit comments

Comments
 (0)