@@ -228,36 +228,48 @@ impl SerializedDepGraph {
228
228
let mut edge_list_data =
229
229
Vec :: with_capacity ( graph_bytes - node_count * size_of :: < SerializedNodeHeader < D > > ( ) ) ;
230
230
231
- for _ in 0 ..node_count {
232
- // Decode the header for this edge; the header packs together as many of the fixed-size
233
- // fields as possible to limit the number of times we update decoder state.
234
- let node_header =
235
- SerializedNodeHeader :: < D > { bytes : d. read_array ( ) , _marker : PhantomData } ;
231
+ let mut decoded_nodes = 0 ;
232
+ while decoded_nodes < node_count {
233
+ let mut current_index = d. read_u32 ( ) ;
234
+
235
+ loop {
236
+ // Decode the header for this edge; the header packs together as many of the fixed-size
237
+ // fields as possible to limit the number of times we update decoder state.
238
+ let node_header =
239
+ SerializedNodeHeader :: < D > { bytes : d. read_array ( ) , _marker : PhantomData } ;
240
+
241
+ if node_header. node ( ) . kind == D :: DEP_KIND_NULL {
242
+ break ;
243
+ }
244
+
245
+ decoded_nodes += 1 ;
236
246
237
- let index = node_header. index ( ) ;
247
+ let index = SerializedDepNodeIndex :: from_u32 ( current_index) ;
248
+ current_index += 1 ;
238
249
239
- let node = & mut nodes[ index] ;
240
- // Make sure there's no duplicate indices in the dep graph.
241
- assert ! ( node_header . node ( ) . kind != D :: DEP_KIND_NULL && node. kind == D :: DEP_KIND_NULL ) ;
242
- * node = node_header. node ( ) ;
250
+ let node = & mut nodes[ index] ;
251
+ // Make sure there's no duplicate indices in the dep graph.
252
+ assert ! ( node. kind == D :: DEP_KIND_NULL ) ;
253
+ * node = node_header. node ( ) ;
243
254
244
- fingerprints[ index] = node_header. fingerprint ( ) ;
255
+ fingerprints[ index] = node_header. fingerprint ( ) ;
245
256
246
- // If the length of this node's edge list is small, the length is stored in the header.
247
- // If it is not, we fall back to another decoder call.
248
- let num_edges = node_header. len ( ) . unwrap_or_else ( || d. read_u32 ( ) ) ;
257
+ // If the length of this node's edge list is small, the length is stored in the header.
258
+ // If it is not, we fall back to another decoder call.
259
+ let num_edges = node_header. len ( ) . unwrap_or_else ( || d. read_u32 ( ) ) ;
249
260
250
- // The edges index list uses the same varint strategy as rmeta tables; we select the
251
- // number of byte elements per-array not per-element. This lets us read the whole edge
252
- // list for a node with one decoder call and also use the on-disk format in memory.
253
- let edges_len_bytes = node_header. bytes_per_index ( ) * ( num_edges as usize ) ;
254
- // The in-memory structure for the edges list stores the byte width of the edges on
255
- // this node with the offset into the global edge data array.
256
- let edges_header = node_header. edges_header ( & edge_list_data, num_edges) ;
261
+ // The edges index list uses the same varint strategy as rmeta tables; we select the
262
+ // number of byte elements per-array not per-element. This lets us read the whole edge
263
+ // list for a node with one decoder call and also use the on-disk format in memory.
264
+ let edges_len_bytes = node_header. bytes_per_index ( ) * ( num_edges as usize ) ;
265
+ // The in-memory structure for the edges list stores the byte width of the edges on
266
+ // this node with the offset into the global edge data array.
267
+ let edges_header = node_header. edges_header ( & edge_list_data, num_edges) ;
257
268
258
- edge_list_data. extend ( d. read_raw_bytes ( edges_len_bytes) ) ;
269
+ edge_list_data. extend ( d. read_raw_bytes ( edges_len_bytes) ) ;
259
270
260
- edge_list_indices[ index] = edges_header;
271
+ edge_list_indices[ index] = edges_header;
272
+ }
261
273
}
262
274
263
275
// When we access the edge list data, we do a fixed-size read from the edge list data then
@@ -308,10 +320,9 @@ impl SerializedDepGraph {
308
320
/// * In whatever bits remain, the length of the edge list for this node, if it fits
309
321
struct SerializedNodeHeader < D > {
310
322
// 2 bytes for the DepNode
311
- // 4 bytes for the index
312
323
// 16 for Fingerprint in DepNode
313
324
// 16 for Fingerprint in NodeInfo
314
- bytes : [ u8 ; 38 ] ,
325
+ bytes : [ u8 ; 34 ] ,
315
326
_marker : PhantomData < D > ,
316
327
}
317
328
@@ -321,7 +332,6 @@ struct Unpacked {
321
332
len : Option < u32 > ,
322
333
bytes_per_index : usize ,
323
334
kind : DepKind ,
324
- index : SerializedDepNodeIndex ,
325
335
hash : PackedFingerprint ,
326
336
fingerprint : Fingerprint ,
327
337
}
@@ -343,7 +353,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
343
353
#[ inline]
344
354
fn new (
345
355
node : DepNode ,
346
- index : DepNodeIndex ,
347
356
fingerprint : Fingerprint ,
348
357
edge_max_index : u32 ,
349
358
edge_count : usize ,
@@ -365,11 +374,10 @@ impl<D: Deps> SerializedNodeHeader<D> {
365
374
let hash: Fingerprint = node. hash . into ( ) ;
366
375
367
376
// Using half-open ranges ensures an unconditional panic if we get the magic numbers wrong.
368
- let mut bytes = [ 0u8 ; 38 ] ;
377
+ let mut bytes = [ 0u8 ; 34 ] ;
369
378
bytes[ ..2 ] . copy_from_slice ( & head. to_le_bytes ( ) ) ;
370
- bytes[ 2 ..6 ] . copy_from_slice ( & index. as_u32 ( ) . to_le_bytes ( ) ) ;
371
- bytes[ 6 ..22 ] . copy_from_slice ( & hash. to_le_bytes ( ) ) ;
372
- bytes[ 22 ..] . copy_from_slice ( & fingerprint. to_le_bytes ( ) ) ;
379
+ bytes[ 2 ..18 ] . copy_from_slice ( & hash. to_le_bytes ( ) ) ;
380
+ bytes[ 18 ..] . copy_from_slice ( & fingerprint. to_le_bytes ( ) ) ;
373
381
374
382
#[ cfg( debug_assertions) ]
375
383
{
@@ -386,9 +394,8 @@ impl<D: Deps> SerializedNodeHeader<D> {
386
394
#[ inline]
387
395
fn unpack ( & self ) -> Unpacked {
388
396
let head = u16:: from_le_bytes ( self . bytes [ ..2 ] . try_into ( ) . unwrap ( ) ) ;
389
- let index = u32:: from_le_bytes ( self . bytes [ 2 ..6 ] . try_into ( ) . unwrap ( ) ) ;
390
- let hash = self . bytes [ 6 ..22 ] . try_into ( ) . unwrap ( ) ;
391
- let fingerprint = self . bytes [ 22 ..] . try_into ( ) . unwrap ( ) ;
397
+ let hash = self . bytes [ 2 ..18 ] . try_into ( ) . unwrap ( ) ;
398
+ let fingerprint = self . bytes [ 18 ..] . try_into ( ) . unwrap ( ) ;
392
399
393
400
let kind = head & mask ( Self :: KIND_BITS ) as u16 ;
394
401
let bytes_per_index = ( head >> Self :: KIND_BITS ) & mask ( Self :: WIDTH_BITS ) as u16 ;
@@ -398,7 +405,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
398
405
len : len. checked_sub ( 1 ) ,
399
406
bytes_per_index : bytes_per_index as usize + 1 ,
400
407
kind : DepKind :: new ( kind) ,
401
- index : SerializedDepNodeIndex :: from_u32 ( index) ,
402
408
hash : Fingerprint :: from_le_bytes ( hash) . into ( ) ,
403
409
fingerprint : Fingerprint :: from_le_bytes ( fingerprint) ,
404
410
}
@@ -414,11 +420,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
414
420
self . unpack ( ) . bytes_per_index
415
421
}
416
422
417
- #[ inline]
418
- fn index ( & self ) -> SerializedDepNodeIndex {
419
- self . unpack ( ) . index
420
- }
421
-
422
423
#[ inline]
423
424
fn fingerprint ( & self ) -> Fingerprint {
424
425
self . unpack ( ) . fingerprint
@@ -447,15 +448,10 @@ struct NodeInfo {
447
448
}
448
449
449
450
impl NodeInfo {
450
- fn encode < D : Deps > ( & self , e : & mut MemEncoder , index : DepNodeIndex ) {
451
+ fn encode < D : Deps > ( & self , e : & mut MemEncoder ) {
451
452
let NodeInfo { node, fingerprint, ref edges } = * self ;
452
- let header = SerializedNodeHeader :: < D > :: new (
453
- node,
454
- index,
455
- fingerprint,
456
- edges. max_index ( ) ,
457
- edges. len ( ) ,
458
- ) ;
453
+ let header =
454
+ SerializedNodeHeader :: < D > :: new ( node, fingerprint, edges. max_index ( ) , edges. len ( ) ) ;
459
455
e. write_array ( header. bytes ) ;
460
456
461
457
if header. len ( ) . is_none ( ) {
@@ -479,7 +475,6 @@ impl NodeInfo {
479
475
fn encode_promoted < D : Deps > (
480
476
e : & mut MemEncoder ,
481
477
node : DepNode ,
482
- index : DepNodeIndex ,
483
478
fingerprint : Fingerprint ,
484
479
prev_index : SerializedDepNodeIndex ,
485
480
colors : & DepNodeColorMap ,
@@ -492,7 +487,7 @@ impl NodeInfo {
492
487
let edge_max =
493
488
edges. clone ( ) . map ( |i| colors. current ( i) . unwrap ( ) . as_u32 ( ) ) . max ( ) . unwrap_or ( 0 ) ;
494
489
495
- let header = SerializedNodeHeader :: < D > :: new ( node, index , fingerprint, edge_max, edge_count) ;
490
+ let header = SerializedNodeHeader :: < D > :: new ( node, fingerprint, edge_max, edge_count) ;
496
491
e. write_array ( header. bytes ) ;
497
492
498
493
if header. len ( ) . is_none ( ) {
@@ -526,6 +521,8 @@ struct LocalEncoderState {
526
521
node_count : usize ,
527
522
edge_count : usize ,
528
523
524
+ in_chunk : bool ,
525
+
529
526
/// Stores the number of times we've encoded each dep kind.
530
527
kind_stats : Vec < u32 > ,
531
528
}
@@ -561,6 +558,7 @@ impl<D: Deps> EncoderState<D> {
561
558
remaining_node_index : 0 ,
562
559
edge_count : 0 ,
563
560
node_count : 0 ,
561
+ in_chunk : false ,
564
562
encoder : MemEncoder :: new ( ) ,
565
563
kind_stats : iter:: repeat ( 0 ) . take ( D :: DEP_KIND_MAX as usize + 1 ) . collect ( ) ,
566
564
} )
@@ -569,6 +567,30 @@ impl<D: Deps> EncoderState<D> {
569
567
}
570
568
}
571
569
570
+ #[ inline]
571
+ fn end_chunk ( & self , local : & mut LocalEncoderState ) {
572
+ if !local. in_chunk {
573
+ return ;
574
+ }
575
+ local. in_chunk = false ;
576
+
577
+ NodeInfo {
578
+ node : DepNode { kind : D :: DEP_KIND_NULL , hash : Fingerprint :: ZERO . into ( ) } ,
579
+ fingerprint : Fingerprint :: ZERO ,
580
+ edges : EdgesVec :: new ( ) ,
581
+ }
582
+ . encode :: < D > ( & mut local. encoder ) ;
583
+ }
584
+
585
+ #[ inline]
586
+ fn start_chunk ( & self , local : & mut LocalEncoderState , first_index : u32 ) {
587
+ if local. in_chunk {
588
+ self . end_chunk ( local) ;
589
+ }
590
+ local. in_chunk = true ;
591
+ local. encoder . emit_u32 ( first_index) ;
592
+ }
593
+
572
594
#[ inline]
573
595
fn next_index ( & self , local : & mut LocalEncoderState ) -> DepNodeIndex {
574
596
if local. remaining_node_index == 0 {
@@ -579,6 +601,8 @@ impl<D: Deps> EncoderState<D> {
579
601
local. next_node_index =
580
602
self . next_node_index . fetch_add ( count, Ordering :: Relaxed ) . try_into ( ) . unwrap ( ) ;
581
603
604
+ self . start_chunk ( local, local. next_node_index ) ;
605
+
582
606
local. remaining_node_index = count as u32 ;
583
607
}
584
608
@@ -635,10 +659,13 @@ impl<D: Deps> EncoderState<D> {
635
659
636
660
#[ inline]
637
661
fn flush_mem_encoder ( & self , local : & mut LocalEncoderState ) {
638
- let data = & mut local. encoder . data ;
639
- if data. len ( ) > 64 * 1024 {
640
- self . file . lock ( ) . as_mut ( ) . unwrap ( ) . emit_raw_bytes ( & data[ ..] ) ;
641
- data. clear ( ) ;
662
+ if local. encoder . data . len ( ) > 64 * 1024 {
663
+ self . end_chunk ( local) ;
664
+ self . file . lock ( ) . as_mut ( ) . unwrap ( ) . emit_raw_bytes ( & local. encoder . data [ ..] ) ;
665
+ local. encoder . data . clear ( ) ;
666
+ if local. remaining_node_index > 0 {
667
+ self . start_chunk ( local, local. next_node_index ) ;
668
+ }
642
669
}
643
670
}
644
671
@@ -650,7 +677,7 @@ impl<D: Deps> EncoderState<D> {
650
677
record_graph : & Option < Lock < DepGraphQuery > > ,
651
678
local : & mut LocalEncoderState ,
652
679
) {
653
- node. encode :: < D > ( & mut local. encoder , index ) ;
680
+ node. encode :: < D > ( & mut local. encoder ) ;
654
681
self . flush_mem_encoder ( & mut * local) ;
655
682
self . record (
656
683
node. node ,
@@ -682,7 +709,6 @@ impl<D: Deps> EncoderState<D> {
682
709
let edge_count = NodeInfo :: encode_promoted :: < D > (
683
710
& mut local. encoder ,
684
711
node,
685
- index,
686
712
fingerprint,
687
713
prev_index,
688
714
colors,
@@ -711,6 +737,8 @@ impl<D: Deps> EncoderState<D> {
711
737
let results = broadcast ( |_| {
712
738
let mut local = self . local . borrow_mut ( ) ;
713
739
740
+ self . end_chunk ( & mut * local) ;
741
+
714
742
// Prevent more indices from being allocated on this thread.
715
743
local. remaining_node_index = 0 ;
716
744
0 commit comments