@@ -11,6 +11,9 @@ import (
11
11
"math"
12
12
"strconv"
13
13
"strings"
14
+
15
+ "github.com/djherbis/buffer"
16
+ "github.com/djherbis/nio/v3"
14
17
)
15
18
16
19
// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function
@@ -42,7 +45,7 @@ func CatFileBatchCheck(repoPath string) (WriteCloserError, *bufio.Reader, func()
42
45
}
43
46
}()
44
47
45
- // For simplicities sake we'll us a buffered reader to read from the cat-file --batch
48
+ // For simplicities sake we'll use a buffered reader to read from the cat-file --batch-check
46
49
batchReader := bufio .NewReader (batchStdoutReader )
47
50
48
51
return batchStdinWriter , batchReader , cancel
@@ -53,7 +56,7 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) {
53
56
// We often want to feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
54
57
// so let's create a batch stdin and stdout
55
58
batchStdinReader , batchStdinWriter := io .Pipe ()
56
- batchStdoutReader , batchStdoutWriter := io .Pipe ()
59
+ batchStdoutReader , batchStdoutWriter := nio .Pipe (buffer . New ( 32 * 1024 ) )
57
60
cancel := func () {
58
61
_ = batchStdinReader .Close ()
59
62
_ = batchStdinWriter .Close ()
@@ -74,7 +77,7 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) {
74
77
}()
75
78
76
79
// For simplicities sake we'll us a buffered reader to read from the cat-file --batch
77
- batchReader := bufio .NewReader (batchStdoutReader )
80
+ batchReader := bufio .NewReaderSize (batchStdoutReader , 32 * 1024 )
78
81
79
82
return batchStdinWriter , batchReader , cancel
80
83
}
@@ -84,22 +87,31 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) {
84
87
// <sha> SP <type> SP <size> LF
85
88
// sha is a 40byte not 20byte here
86
89
func ReadBatchLine (rd * bufio.Reader ) (sha []byte , typ string , size int64 , err error ) {
87
- sha , err = rd .ReadBytes ( ' ' )
90
+ typ , err = rd .ReadString ( '\n ' )
88
91
if err != nil {
89
92
return
90
93
}
91
- sha = sha [:len (sha )- 1 ]
92
-
93
- typ , err = rd .ReadString ('\n' )
94
- if err != nil {
94
+ if len (typ ) == 1 {
95
+ typ , err = rd .ReadString ('\n' )
96
+ if err != nil {
97
+ return
98
+ }
99
+ }
100
+ idx := strings .IndexByte (typ , ' ' )
101
+ if idx < 0 {
102
+ log ("missing space typ: %s" , typ )
103
+ err = ErrNotExist {ID : string (sha )}
95
104
return
96
105
}
106
+ sha = []byte (typ [:idx ])
107
+ typ = typ [idx + 1 :]
97
108
98
- idx : = strings .Index (typ , " " )
109
+ idx = strings .IndexByte (typ , ' ' )
99
110
if idx < 0 {
100
111
err = ErrNotExist {ID : string (sha )}
101
112
return
102
113
}
114
+
103
115
sizeStr := typ [idx + 1 : len (typ )- 1 ]
104
116
typ = typ [:idx ]
105
117
@@ -130,7 +142,7 @@ headerLoop:
130
142
}
131
143
132
144
// Discard the rest of the tag
133
- discard := size - n
145
+ discard := size - n + 1
134
146
for discard > math .MaxInt32 {
135
147
_ , err := rd .Discard (math .MaxInt32 )
136
148
if err != nil {
@@ -211,14 +223,20 @@ func To40ByteSHA(sha, out []byte) []byte {
211
223
func ParseTreeLineSkipMode (rd * bufio.Reader , fnameBuf , shaBuf []byte ) (fname , sha []byte , n int , err error ) {
212
224
var readBytes []byte
213
225
// Skip the Mode
214
- readBytes , err = rd .ReadSlice (' ' ) // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
226
+ readBytes , err = rd .ReadSlice ('\x00 ' ) // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
215
227
if err != nil {
216
228
return
217
229
}
218
- n += len (readBytes )
230
+ idx := bytes .IndexByte (readBytes , ' ' )
231
+ if idx < 0 {
232
+ log ("missing space in readBytes: %s" , readBytes )
233
+ err = & ErrNotExist {}
234
+ return
235
+ }
236
+ n += idx + 1
237
+ readBytes = readBytes [idx + 1 :]
219
238
220
239
// Deal with the fname
221
- readBytes , err = rd .ReadSlice ('\x00' )
222
240
copy (fnameBuf , readBytes )
223
241
if len (fnameBuf ) > len (readBytes ) {
224
242
fnameBuf = fnameBuf [:len (readBytes )] // cut the buf the correct size
@@ -237,7 +255,7 @@ func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sh
237
255
fname = fnameBuf // set the returnable fname to the slice
238
256
239
257
// Now deal with the 20-byte SHA
240
- idx : = 0
258
+ idx = 0
241
259
for idx < 20 {
242
260
read := 0
243
261
read , err = rd .Read (shaBuf [idx :20 ])
@@ -262,23 +280,102 @@ func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sh
262
280
func ParseTreeLine (rd * bufio.Reader , modeBuf , fnameBuf , shaBuf []byte ) (mode , fname , sha []byte , n int , err error ) {
263
281
var readBytes []byte
264
282
265
- // Read the Mode
266
- readBytes , err = rd .ReadSlice (' ' )
283
+ // Read the Mode & fname
284
+ readBytes , err = rd .ReadSlice ('\x00 ' )
267
285
if err != nil {
268
286
return
269
287
}
270
- n += len (readBytes )
271
- copy (modeBuf , readBytes )
272
- if len (modeBuf ) > len (readBytes ) {
273
- modeBuf = modeBuf [:len (readBytes )]
274
- } else {
275
- modeBuf = append (modeBuf , readBytes [len (modeBuf ):]... )
288
+ idx := bytes .IndexByte (readBytes , ' ' )
289
+ if idx < 0 {
290
+ log ("missing space in readBytes ParseTreeLine: %s" , readBytes )
291
+
292
+ err = & ErrNotExist {}
293
+ return
294
+ }
276
295
296
+ n += idx + 1
297
+ copy (modeBuf , readBytes [:idx ])
298
+ if len (modeBuf ) >= idx {
299
+ modeBuf = modeBuf [:idx ]
300
+ } else {
301
+ modeBuf = append (modeBuf , readBytes [len (modeBuf ):idx ]... )
277
302
}
278
- mode = modeBuf [:len (modeBuf )- 1 ] // Drop the SP
303
+ mode = modeBuf
304
+
305
+ readBytes = readBytes [idx + 1 :]
279
306
280
307
// Deal with the fname
308
+ copy (fnameBuf , readBytes )
309
+ if len (fnameBuf ) > len (readBytes ) {
310
+ fnameBuf = fnameBuf [:len (readBytes )]
311
+ } else {
312
+ fnameBuf = append (fnameBuf , readBytes [len (fnameBuf ):]... )
313
+ }
314
+ for err == bufio .ErrBufferFull {
315
+ readBytes , err = rd .ReadSlice ('\x00' )
316
+ fnameBuf = append (fnameBuf , readBytes ... )
317
+ }
318
+ n += len (fnameBuf )
319
+ if err != nil {
320
+ return
321
+ }
322
+ fnameBuf = fnameBuf [:len (fnameBuf )- 1 ]
323
+ fname = fnameBuf
324
+
325
+ // Deal with the 20-byte SHA
326
+ idx = 0
327
+ for idx < 20 {
328
+ read := 0
329
+ read , err = rd .Read (shaBuf [idx :20 ])
330
+ n += read
331
+ if err != nil {
332
+ return
333
+ }
334
+ idx += read
335
+ }
336
+ sha = shaBuf
337
+ return
338
+ }
339
+
340
+ // ParseTreeLineTree reads a tree entry from a tree in a cat-file --batch stream
341
+ //
342
+ // This carefully avoids allocations - except where fnameBuf is too small.
343
+ // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
344
+ //
345
+ // Each line is composed of:
346
+ // <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
347
+ //
348
+ // We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
349
+ func ParseTreeLineTree (rd * bufio.Reader , modeBuf , fnameBuf , shaBuf []byte ) (isTree bool , fname , sha []byte , n int , err error ) {
350
+ var readBytes []byte
351
+
352
+ // Read the Mode & fname
281
353
readBytes , err = rd .ReadSlice ('\x00' )
354
+ if err != nil {
355
+ return
356
+ }
357
+ if len (readBytes ) < 6 {
358
+ log ("missing space in readBytes ParseTreeLineTree: %v" , readBytes )
359
+ err = & ErrNotExist {}
360
+ return
361
+ }
362
+ if ! bytes .Equal (readBytes [:6 ], []byte ("40000 " )) {
363
+ n += len (readBytes )
364
+ for err == bufio .ErrBufferFull {
365
+ readBytes , err = rd .ReadSlice ('\x00' )
366
+ n += len (readBytes )
367
+ }
368
+ d := 0
369
+ d , err = rd .Discard (20 )
370
+ n += d
371
+ return
372
+ }
373
+ isTree = true
374
+
375
+ n += 6
376
+ readBytes = readBytes [6 :]
377
+
378
+ // Deal with the fname
282
379
copy (fnameBuf , readBytes )
283
380
if len (fnameBuf ) > len (readBytes ) {
284
381
fnameBuf = fnameBuf [:len (readBytes )]
0 commit comments