Skip to content

Commit e79a807

Browse files
authored
Refactor markup/csv: don't read all to memory (#29760)
1 parent bbef5fc commit e79a807

File tree

2 files changed

+55
-12
lines changed

2 files changed

+55
-12
lines changed

modules/markup/csv/csv.go

Lines changed: 45 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -77,29 +77,62 @@ func writeField(w io.Writer, element, class, field string) error {
7777
}
7878

7979
// Render implements markup.Renderer
80-
func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
80+
func (r Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error {
8181
tmpBlock := bufio.NewWriter(output)
82+
maxSize := setting.UI.CSV.MaxFileSize
8283

83-
// FIXME: don't read all to memory
84-
rawBytes, err := io.ReadAll(input)
84+
if maxSize == 0 {
85+
return r.tableRender(ctx, input, tmpBlock)
86+
}
87+
88+
rawBytes, err := io.ReadAll(io.LimitReader(input, maxSize+1))
8589
if err != nil {
8690
return err
8791
}
8892

89-
if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
90-
if _, err := tmpBlock.WriteString("<pre>"); err != nil {
91-
return err
92-
}
93-
if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
94-
return err
93+
if int64(len(rawBytes)) <= maxSize {
94+
return r.tableRender(ctx, bytes.NewReader(rawBytes), tmpBlock)
95+
}
96+
return r.fallbackRender(io.MultiReader(bytes.NewReader(rawBytes), input), tmpBlock)
97+
}
98+
99+
func (Renderer) fallbackRender(input io.Reader, tmpBlock *bufio.Writer) error {
100+
_, err := tmpBlock.WriteString("<pre>")
101+
if err != nil {
102+
return err
103+
}
104+
105+
scan := bufio.NewScanner(input)
106+
scan.Split(bufio.ScanRunes)
107+
for scan.Scan() {
108+
switch scan.Text() {
109+
case `&`:
110+
_, err = tmpBlock.WriteString("&amp;")
111+
case `'`:
112+
_, err = tmpBlock.WriteString("&#39;") // "&#39;" is shorter than "&apos;" and apos was not in HTML until HTML5.
113+
case `<`:
114+
_, err = tmpBlock.WriteString("&lt;")
115+
case `>`:
116+
_, err = tmpBlock.WriteString("&gt;")
117+
case `"`:
118+
_, err = tmpBlock.WriteString("&#34;") // "&#34;" is shorter than "&quot;".
119+
default:
120+
_, err = tmpBlock.Write(scan.Bytes())
95121
}
96-
if _, err := tmpBlock.WriteString("</pre>"); err != nil {
122+
if err != nil {
97123
return err
98124
}
99-
return tmpBlock.Flush()
100125
}
101126

102-
rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, bytes.NewReader(rawBytes))
127+
_, err = tmpBlock.WriteString("</pre>")
128+
if err != nil {
129+
return err
130+
}
131+
return tmpBlock.Flush()
132+
}
133+
134+
func (Renderer) tableRender(ctx *markup.RenderContext, input io.Reader, tmpBlock *bufio.Writer) error {
135+
rd, err := csv.CreateReaderAndDetermineDelimiter(ctx, input)
103136
if err != nil {
104137
return err
105138
}

modules/markup/csv/csv_test.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package markup
55

66
import (
7+
"bufio"
8+
"bytes"
79
"strings"
810
"testing"
911

@@ -29,4 +31,12 @@ func TestRenderCSV(t *testing.T) {
2931
assert.NoError(t, err)
3032
assert.EqualValues(t, v, buf.String())
3133
}
34+
35+
t.Run("fallbackRender", func(t *testing.T) {
36+
var buf bytes.Buffer
37+
err := render.fallbackRender(strings.NewReader("1,<a>\n2,<b>"), bufio.NewWriter(&buf))
38+
assert.NoError(t, err)
39+
want := "<pre>1,&lt;a&gt;\n2,&lt;b&gt;</pre>"
40+
assert.Equal(t, want, buf.String())
41+
})
3242
}

0 commit comments

Comments
 (0)