Skip to content

Commit d17b845

Browse files
committed
Group dep nodes by chunks with sequential indices and only store the first index at the start of the chunk
1 parent 9bb8430 commit d17b845

File tree

1 file changed

+84
-56
lines changed

1 file changed

+84
-56
lines changed

compiler/rustc_query_system/src/dep_graph/serialized.rs

+84-56
Original file line numberDiff line numberDiff line change
@@ -228,36 +228,48 @@ impl SerializedDepGraph {
228228
let mut edge_list_data =
229229
Vec::with_capacity(graph_bytes - node_count * size_of::<SerializedNodeHeader<D>>());
230230

231-
for _ in 0..node_count {
232-
// Decode the header for this edge; the header packs together as many of the fixed-size
233-
// fields as possible to limit the number of times we update decoder state.
234-
let node_header =
235-
SerializedNodeHeader::<D> { bytes: d.read_array(), _marker: PhantomData };
231+
let mut decoded_nodes = 0;
232+
while decoded_nodes < node_count {
233+
let mut current_index = d.read_u32();
234+
235+
loop {
236+
// Decode the header for this edge; the header packs together as many of the fixed-size
237+
// fields as possible to limit the number of times we update decoder state.
238+
let node_header =
239+
SerializedNodeHeader::<D> { bytes: d.read_array(), _marker: PhantomData };
240+
241+
if node_header.node().kind == D::DEP_KIND_NULL {
242+
break;
243+
}
244+
245+
decoded_nodes += 1;
236246

237-
let index = node_header.index();
247+
let index = SerializedDepNodeIndex::from_u32(current_index);
248+
current_index += 1;
238249

239-
let node = &mut nodes[index];
240-
// Make sure there's no duplicate indices in the dep graph.
241-
assert!(node_header.node().kind != D::DEP_KIND_NULL && node.kind == D::DEP_KIND_NULL);
242-
*node = node_header.node();
250+
let node = &mut nodes[index];
251+
// Make sure there's no duplicate indices in the dep graph.
252+
assert!(node.kind == D::DEP_KIND_NULL);
253+
*node = node_header.node();
243254

244-
fingerprints[index] = node_header.fingerprint();
255+
fingerprints[index] = node_header.fingerprint();
245256

246-
// If the length of this node's edge list is small, the length is stored in the header.
247-
// If it is not, we fall back to another decoder call.
248-
let num_edges = node_header.len().unwrap_or_else(|| d.read_u32());
257+
// If the length of this node's edge list is small, the length is stored in the header.
258+
// If it is not, we fall back to another decoder call.
259+
let num_edges = node_header.len().unwrap_or_else(|| d.read_u32());
249260

250-
// The edges index list uses the same varint strategy as rmeta tables; we select the
251-
// number of byte elements per-array not per-element. This lets us read the whole edge
252-
// list for a node with one decoder call and also use the on-disk format in memory.
253-
let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize);
254-
// The in-memory structure for the edges list stores the byte width of the edges on
255-
// this node with the offset into the global edge data array.
256-
let edges_header = node_header.edges_header(&edge_list_data, num_edges);
261+
// The edges index list uses the same varint strategy as rmeta tables; we select the
262+
// number of byte elements per-array not per-element. This lets us read the whole edge
263+
// list for a node with one decoder call and also use the on-disk format in memory.
264+
let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize);
265+
// The in-memory structure for the edges list stores the byte width of the edges on
266+
// this node with the offset into the global edge data array.
267+
let edges_header = node_header.edges_header(&edge_list_data, num_edges);
257268

258-
edge_list_data.extend(d.read_raw_bytes(edges_len_bytes));
269+
edge_list_data.extend(d.read_raw_bytes(edges_len_bytes));
259270

260-
edge_list_indices[index] = edges_header;
271+
edge_list_indices[index] = edges_header;
272+
}
261273
}
262274

263275
// When we access the edge list data, we do a fixed-size read from the edge list data then
@@ -308,10 +320,9 @@ impl SerializedDepGraph {
308320
/// * In whatever bits remain, the length of the edge list for this node, if it fits
309321
struct SerializedNodeHeader<D> {
310322
// 2 bytes for the DepNode
311-
// 4 bytes for the index
312323
// 16 for Fingerprint in DepNode
313324
// 16 for Fingerprint in NodeInfo
314-
bytes: [u8; 38],
325+
bytes: [u8; 34],
315326
_marker: PhantomData<D>,
316327
}
317328

@@ -321,7 +332,6 @@ struct Unpacked {
321332
len: Option<u32>,
322333
bytes_per_index: usize,
323334
kind: DepKind,
324-
index: SerializedDepNodeIndex,
325335
hash: PackedFingerprint,
326336
fingerprint: Fingerprint,
327337
}
@@ -343,7 +353,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
343353
#[inline]
344354
fn new(
345355
node: DepNode,
346-
index: DepNodeIndex,
347356
fingerprint: Fingerprint,
348357
edge_max_index: u32,
349358
edge_count: usize,
@@ -365,11 +374,10 @@ impl<D: Deps> SerializedNodeHeader<D> {
365374
let hash: Fingerprint = node.hash.into();
366375

367376
// Using half-open ranges ensures an unconditional panic if we get the magic numbers wrong.
368-
let mut bytes = [0u8; 38];
377+
let mut bytes = [0u8; 34];
369378
bytes[..2].copy_from_slice(&head.to_le_bytes());
370-
bytes[2..6].copy_from_slice(&index.as_u32().to_le_bytes());
371-
bytes[6..22].copy_from_slice(&hash.to_le_bytes());
372-
bytes[22..].copy_from_slice(&fingerprint.to_le_bytes());
379+
bytes[2..18].copy_from_slice(&hash.to_le_bytes());
380+
bytes[18..].copy_from_slice(&fingerprint.to_le_bytes());
373381

374382
#[cfg(debug_assertions)]
375383
{
@@ -386,9 +394,8 @@ impl<D: Deps> SerializedNodeHeader<D> {
386394
#[inline]
387395
fn unpack(&self) -> Unpacked {
388396
let head = u16::from_le_bytes(self.bytes[..2].try_into().unwrap());
389-
let index = u32::from_le_bytes(self.bytes[2..6].try_into().unwrap());
390-
let hash = self.bytes[6..22].try_into().unwrap();
391-
let fingerprint = self.bytes[22..].try_into().unwrap();
397+
let hash = self.bytes[2..18].try_into().unwrap();
398+
let fingerprint = self.bytes[18..].try_into().unwrap();
392399

393400
let kind = head & mask(Self::KIND_BITS) as u16;
394401
let bytes_per_index = (head >> Self::KIND_BITS) & mask(Self::WIDTH_BITS) as u16;
@@ -398,7 +405,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
398405
len: len.checked_sub(1),
399406
bytes_per_index: bytes_per_index as usize + 1,
400407
kind: DepKind::new(kind),
401-
index: SerializedDepNodeIndex::from_u32(index),
402408
hash: Fingerprint::from_le_bytes(hash).into(),
403409
fingerprint: Fingerprint::from_le_bytes(fingerprint),
404410
}
@@ -414,11 +420,6 @@ impl<D: Deps> SerializedNodeHeader<D> {
414420
self.unpack().bytes_per_index
415421
}
416422

417-
#[inline]
418-
fn index(&self) -> SerializedDepNodeIndex {
419-
self.unpack().index
420-
}
421-
422423
#[inline]
423424
fn fingerprint(&self) -> Fingerprint {
424425
self.unpack().fingerprint
@@ -447,15 +448,10 @@ struct NodeInfo {
447448
}
448449

449450
impl NodeInfo {
450-
fn encode<D: Deps>(&self, e: &mut MemEncoder, index: DepNodeIndex) {
451+
fn encode<D: Deps>(&self, e: &mut MemEncoder) {
451452
let NodeInfo { node, fingerprint, ref edges } = *self;
452-
let header = SerializedNodeHeader::<D>::new(
453-
node,
454-
index,
455-
fingerprint,
456-
edges.max_index(),
457-
edges.len(),
458-
);
453+
let header =
454+
SerializedNodeHeader::<D>::new(node, fingerprint, edges.max_index(), edges.len());
459455
e.write_array(header.bytes);
460456

461457
if header.len().is_none() {
@@ -479,7 +475,6 @@ impl NodeInfo {
479475
fn encode_promoted<D: Deps>(
480476
e: &mut MemEncoder,
481477
node: DepNode,
482-
index: DepNodeIndex,
483478
fingerprint: Fingerprint,
484479
prev_index: SerializedDepNodeIndex,
485480
colors: &DepNodeColorMap,
@@ -492,7 +487,7 @@ impl NodeInfo {
492487
let edge_max =
493488
edges.clone().map(|i| colors.current(i).unwrap().as_u32()).max().unwrap_or(0);
494489

495-
let header = SerializedNodeHeader::<D>::new(node, index, fingerprint, edge_max, edge_count);
490+
let header = SerializedNodeHeader::<D>::new(node, fingerprint, edge_max, edge_count);
496491
e.write_array(header.bytes);
497492

498493
if header.len().is_none() {
@@ -526,6 +521,8 @@ struct LocalEncoderState {
526521
node_count: usize,
527522
edge_count: usize,
528523

524+
in_chunk: bool,
525+
529526
/// Stores the number of times we've encoded each dep kind.
530527
kind_stats: Vec<u32>,
531528
}
@@ -561,6 +558,7 @@ impl<D: Deps> EncoderState<D> {
561558
remaining_node_index: 0,
562559
edge_count: 0,
563560
node_count: 0,
561+
in_chunk: false,
564562
encoder: MemEncoder::new(),
565563
kind_stats: iter::repeat(0).take(D::DEP_KIND_MAX as usize + 1).collect(),
566564
})
@@ -569,6 +567,30 @@ impl<D: Deps> EncoderState<D> {
569567
}
570568
}
571569

570+
#[inline]
571+
fn end_chunk(&self, local: &mut LocalEncoderState) {
572+
if !local.in_chunk {
573+
return;
574+
}
575+
local.in_chunk = false;
576+
577+
NodeInfo {
578+
node: DepNode { kind: D::DEP_KIND_NULL, hash: Fingerprint::ZERO.into() },
579+
fingerprint: Fingerprint::ZERO,
580+
edges: EdgesVec::new(),
581+
}
582+
.encode::<D>(&mut local.encoder);
583+
}
584+
585+
#[inline]
586+
fn start_chunk(&self, local: &mut LocalEncoderState, first_index: u32) {
587+
if local.in_chunk {
588+
self.end_chunk(local);
589+
}
590+
local.in_chunk = true;
591+
local.encoder.emit_u32(first_index);
592+
}
593+
572594
#[inline]
573595
fn next_index(&self, local: &mut LocalEncoderState) -> DepNodeIndex {
574596
if local.remaining_node_index == 0 {
@@ -579,6 +601,8 @@ impl<D: Deps> EncoderState<D> {
579601
local.next_node_index =
580602
self.next_node_index.fetch_add(count, Ordering::Relaxed).try_into().unwrap();
581603

604+
self.start_chunk(local, local.next_node_index);
605+
582606
local.remaining_node_index = count as u32;
583607
}
584608

@@ -635,10 +659,13 @@ impl<D: Deps> EncoderState<D> {
635659

636660
#[inline]
637661
fn flush_mem_encoder(&self, local: &mut LocalEncoderState) {
638-
let data = &mut local.encoder.data;
639-
if data.len() > 64 * 1024 {
640-
self.file.lock().as_mut().unwrap().emit_raw_bytes(&data[..]);
641-
data.clear();
662+
if local.encoder.data.len() > 64 * 1024 {
663+
self.end_chunk(local);
664+
self.file.lock().as_mut().unwrap().emit_raw_bytes(&local.encoder.data[..]);
665+
local.encoder.data.clear();
666+
if local.remaining_node_index > 0 {
667+
self.start_chunk(local, local.next_node_index);
668+
}
642669
}
643670
}
644671

@@ -650,7 +677,7 @@ impl<D: Deps> EncoderState<D> {
650677
record_graph: &Option<Lock<DepGraphQuery>>,
651678
local: &mut LocalEncoderState,
652679
) {
653-
node.encode::<D>(&mut local.encoder, index);
680+
node.encode::<D>(&mut local.encoder);
654681
self.flush_mem_encoder(&mut *local);
655682
self.record(
656683
node.node,
@@ -682,7 +709,6 @@ impl<D: Deps> EncoderState<D> {
682709
let edge_count = NodeInfo::encode_promoted::<D>(
683710
&mut local.encoder,
684711
node,
685-
index,
686712
fingerprint,
687713
prev_index,
688714
colors,
@@ -711,6 +737,8 @@ impl<D: Deps> EncoderState<D> {
711737
let results = broadcast(|_| {
712738
let mut local = self.local.borrow_mut();
713739

740+
self.end_chunk(&mut *local);
741+
714742
// Prevent more indices from being allocated on this thread.
715743
local.remaining_node_index = 0;
716744

0 commit comments

Comments
 (0)