Skip to content

Commit 1fbdf96

Browse files
KN4CK3Rzeripath
andauthored
Fix CSV render error (#17406) (#17431)
Backport #17406. Closes #17378 Both errors from #17378 were caused by #15175. Problem 1 (error with added file): `ToUTF8WithFallbackReader` creates a `MultiReader` from a `byte[2048]` and the remaining reader. `CreateReaderAndGuessDelimiter` tries to read 10000 bytes from this reader but only gets 2048 because that's the first reader in the `MultiReader`. Then the `if size < 1e4` thinks the input is at EOF and just returns that. Problem 2 (error with changed file): The blob reader gets defer closed. That was fine because the old version reads the whole file into memory. Now with the streaming version the close needs to defer after the method. Co-authored-by: zeripath <[email protected]>
1 parent 5159055 commit 1fbdf96

File tree

2 files changed

+21
-22
lines changed

2 files changed

+21
-22
lines changed

modules/csv/csv.go

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,32 +28,24 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
2828
}
2929

3030
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
31+
// Reads at most 10k bytes.
3132
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
3233
var data = make([]byte, 1e4)
3334
size, err := util.ReadAtMost(rd, data)
3435
if err != nil {
3536
return nil, err
3637
}
3738

38-
delimiter := guessDelimiter(data[:size])
39-
40-
var newInput io.Reader
41-
if size < 1e4 {
42-
newInput = bytes.NewReader(data[:size])
43-
} else {
44-
newInput = io.MultiReader(bytes.NewReader(data), rd)
45-
}
46-
47-
return CreateReader(newInput, delimiter), nil
39+
return CreateReader(
40+
io.MultiReader(bytes.NewReader(data[:size]), rd),
41+
guessDelimiter(data[:size]),
42+
), nil
4843
}
4944

5045
// guessDelimiter scores the input CSV data against delimiters, and returns the best match.
51-
// Reads at most 10k bytes & 10 lines.
5246
func guessDelimiter(data []byte) rune {
5347
maxLines := 10
54-
maxBytes := util.Min(len(data), 1e4)
55-
text := string(data[:maxBytes])
56-
text = quoteRegexp.ReplaceAllLiteralString(text, "")
48+
text := quoteRegexp.ReplaceAllLiteralString(string(data), "")
5749
lines := strings.SplitN(text, "\n", maxLines+1)
5850
lines = lines[:util.Min(maxLines, len(lines))]
5951

routers/web/repo/compare.go

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"errors"
1111
"fmt"
1212
"html"
13+
"io"
1314
"net/http"
1415
"path"
1516
"path/filepath"
@@ -104,30 +105,36 @@ func setCsvCompareContext(ctx *context.Context) {
104105

105106
errTooLarge := errors.New(ctx.Locale.Tr("repo.error.csv.too_large"))
106107

107-
csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, error) {
108+
csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, io.Closer, error) {
108109
blob, err := c.GetBlobByPath(diffFile.Name)
109110
if err != nil {
110-
return nil, err
111+
return nil, nil, err
111112
}
112113

113114
if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < blob.Size() {
114-
return nil, errTooLarge
115+
return nil, nil, errTooLarge
115116
}
116117

117118
reader, err := blob.DataAsync()
118119
if err != nil {
119-
return nil, err
120+
return nil, nil, err
120121
}
121-
defer reader.Close()
122122

123-
return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader))
123+
csvReader, err := csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader))
124+
return csvReader, reader, err
124125
}
125126

126-
baseReader, err := csvReaderFromCommit(baseCommit)
127+
baseReader, baseBlobCloser, err := csvReaderFromCommit(baseCommit)
128+
if baseBlobCloser != nil {
129+
defer baseBlobCloser.Close()
130+
}
127131
if err == errTooLarge {
128132
return CsvDiffResult{nil, err.Error()}
129133
}
130-
headReader, err := csvReaderFromCommit(headCommit)
134+
headReader, headBlobCloser, err := csvReaderFromCommit(headCommit)
135+
if headBlobCloser != nil {
136+
defer headBlobCloser.Close()
137+
}
131138
if err == errTooLarge {
132139
return CsvDiffResult{nil, err.Error()}
133140
}

0 commit comments

Comments
 (0)