Skip to content

Commit 8467e8d

Browse files
authored
Auto merge of #36758 - michaelwoerister:incr-comp-file-headers, r=eddyb
incr.comp.: Let the compiler ignore incompatible incr. comp. cache artifacts Implements #35720. cc @nikomatsakis
2 parents ec7679b + 263ba92 commit 8467e8d

File tree

7 files changed

+202
-40
lines changed

7 files changed

+202
-40
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
//! This module defines a generic file format that allows to check if a given
12+
//! file generated by incremental compilation was generated by a compatible
13+
//! compiler version. This file format is used for the on-disk version of the
14+
//! dependency graph and the exported metadata hashes.
15+
//!
16+
//! In practice "compatible compiler version" means "exactly the same compiler
17+
//! version", since the header encodes the git commit hash of the compiler.
18+
//! Since we can always just ignore the incremental compilation cache and
19+
//! compiler versions don't change frequently for the typical user, being
20+
//! conservative here practically has no downside.
21+
22+
use std::io::{self, Read};
23+
use std::path::Path;
24+
use std::fs::File;
25+
use std::env;
26+
27+
use rustc::session::config::nightly_options;
28+
29+
/// The first few bytes of files generated by incremental compilation
30+
const FILE_MAGIC: &'static [u8] = b"RSIC";
31+
32+
/// Change this if the header format changes
33+
const HEADER_FORMAT_VERSION: u16 = 0;
34+
35+
/// A version string that hopefully is always different for compiler versions
36+
/// with different encodings of incremental compilation artifacts. Contains
37+
/// the git commit hash.
38+
const RUSTC_VERSION: Option<&'static str> = option_env!("CFG_VERSION");
39+
40+
pub fn write_file_header<W: io::Write>(stream: &mut W) -> io::Result<()> {
41+
stream.write_all(FILE_MAGIC)?;
42+
stream.write_all(&[(HEADER_FORMAT_VERSION >> 0) as u8,
43+
(HEADER_FORMAT_VERSION >> 8) as u8])?;
44+
45+
let rustc_version = rustc_version();
46+
assert_eq!(rustc_version.len(), (rustc_version.len() as u8) as usize);
47+
stream.write_all(&[rustc_version.len() as u8])?;
48+
stream.write_all(rustc_version.as_bytes())?;
49+
50+
Ok(())
51+
}
52+
53+
/// Reads the contents of a file with a file header as defined in this module.
54+
///
55+
/// - Returns `Ok(Some(data))` if the file existed and was generated by a
56+
/// compatible compiler version. `data` is the entire contents of the file
57+
/// *after* the header.
58+
/// - Returns `Ok(None)` if the file did not exist or was generated by an
59+
/// incompatible version of the compiler.
60+
/// - Returns `Err(..)` if some kind of IO error occurred while reading the
61+
/// file.
62+
pub fn read_file(path: &Path) -> io::Result<Option<Vec<u8>>> {
63+
if !path.exists() {
64+
return Ok(None);
65+
}
66+
67+
let mut file = File::open(path)?;
68+
69+
// Check FILE_MAGIC
70+
{
71+
debug_assert!(FILE_MAGIC.len() == 4);
72+
let mut file_magic = [0u8; 4];
73+
file.read_exact(&mut file_magic)?;
74+
if file_magic != FILE_MAGIC {
75+
return Ok(None)
76+
}
77+
}
78+
79+
// Check HEADER_FORMAT_VERSION
80+
{
81+
debug_assert!(::std::mem::size_of_val(&HEADER_FORMAT_VERSION) == 2);
82+
let mut header_format_version = [0u8; 2];
83+
file.read_exact(&mut header_format_version)?;
84+
let header_format_version = (header_format_version[0] as u16) |
85+
((header_format_version[1] as u16) << 8);
86+
87+
if header_format_version != HEADER_FORMAT_VERSION {
88+
return Ok(None)
89+
}
90+
}
91+
92+
// Check RUSTC_VERSION
93+
{
94+
let mut rustc_version_str_len = [0u8; 1];
95+
file.read_exact(&mut rustc_version_str_len)?;
96+
let rustc_version_str_len = rustc_version_str_len[0] as usize;
97+
let mut buffer = Vec::with_capacity(rustc_version_str_len);
98+
buffer.resize(rustc_version_str_len, 0);
99+
file.read_exact(&mut buffer[..])?;
100+
101+
if &buffer[..] != rustc_version().as_bytes() {
102+
return Ok(None);
103+
}
104+
}
105+
106+
let mut data = vec![];
107+
file.read_to_end(&mut data)?;
108+
109+
Ok(Some(data))
110+
}
111+
112+
fn rustc_version() -> String {
113+
if nightly_options::is_nightly_build() {
114+
if let Some(val) = env::var_os("RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER") {
115+
return val.to_string_lossy().into_owned()
116+
}
117+
}
118+
119+
RUSTC_VERSION.expect("Cannot use rustc without explicit version for \
120+
incremental compilation")
121+
.to_string()
122+
}

src/librustc_incremental/persist/fs.rs

+9
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,15 @@ pub fn finalize_session_directory(sess: &Session, svh: Svh) {
345345
let _ = garbage_collect_session_directories(sess);
346346
}
347347

348+
pub fn delete_all_session_dir_contents(sess: &Session) -> io::Result<()> {
349+
let sess_dir_iterator = sess.incr_comp_session_dir().read_dir()?;
350+
for entry in sess_dir_iterator {
351+
let entry = entry?;
352+
safe_remove_file(&entry.path())?
353+
}
354+
Ok(())
355+
}
356+
348357
fn copy_files(target_dir: &Path,
349358
source_dir: &Path,
350359
print_stats_on_success: bool)

src/librustc_incremental/persist/hash.rs

+9-18
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@ use rustc_data_structures::fnv::FnvHashMap;
1616
use rustc_data_structures::flock;
1717
use rustc_serialize::Decodable;
1818
use rustc_serialize::opaque::Decoder;
19-
use std::io::{ErrorKind, Read};
20-
use std::fs::File;
2119

2220
use IncrementalHashesMap;
2321
use super::data::*;
2422
use super::fs::*;
23+
use super::file_format;
2524

2625
pub struct HashContext<'a, 'tcx: 'a> {
2726
pub tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -153,12 +152,9 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
153152

154153
let hashes_file_path = metadata_hash_import_path(&session_dir);
155154

156-
let mut data = vec![];
157-
match
158-
File::open(&hashes_file_path)
159-
.and_then(|mut file| file.read_to_end(&mut data))
155+
match file_format::read_file(&hashes_file_path)
160156
{
161-
Ok(_) => {
157+
Ok(Some(data)) => {
162158
match self.load_from_data(cnum, &data, svh) {
163159
Ok(()) => { }
164160
Err(err) => {
@@ -167,18 +163,13 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
167163
}
168164
}
169165
}
166+
Ok(None) => {
167+
// If the file is not found, that's ok.
168+
}
170169
Err(err) => {
171-
match err.kind() {
172-
ErrorKind::NotFound => {
173-
// If the file is not found, that's ok.
174-
}
175-
_ => {
176-
self.tcx.sess.err(
177-
&format!("could not load dep information from `{}`: {}",
178-
hashes_file_path.display(), err));
179-
return;
180-
}
181-
}
170+
self.tcx.sess.err(
171+
&format!("could not load dep information from `{}`: {}",
172+
hashes_file_path.display(), err));
182173
}
183174
}
184175
}

src/librustc_incremental/persist/load.rs

+30-22
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ use rustc::ty::TyCtxt;
1818
use rustc_data_structures::fnv::{FnvHashSet, FnvHashMap};
1919
use rustc_serialize::Decodable as RustcDecodable;
2020
use rustc_serialize::opaque::Decoder;
21-
use std::io::Read;
22-
use std::fs::{self, File};
21+
use std::fs;
2322
use std::path::{Path};
2423

2524
use IncrementalHashesMap;
@@ -28,6 +27,7 @@ use super::directory::*;
2827
use super::dirty_clean;
2928
use super::hash::*;
3029
use super::fs::*;
30+
use super::file_format;
3131

3232
pub type DirtyNodes = FnvHashSet<DepNode<DefPathIndex>>;
3333

@@ -94,25 +94,26 @@ fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
9494
}
9595

9696
fn load_data(sess: &Session, path: &Path) -> Option<Vec<u8>> {
97-
if !path.exists() {
98-
return None;
99-
}
100-
101-
let mut data = vec![];
102-
match
103-
File::open(path)
104-
.and_then(|mut file| file.read_to_end(&mut data))
105-
{
106-
Ok(_) => {
107-
Some(data)
97+
match file_format::read_file(path) {
98+
Ok(Some(data)) => return Some(data),
99+
Ok(None) => {
100+
// The file either didn't exist or was produced by an incompatible
101+
// compiler version. Neither is an error.
108102
}
109103
Err(err) => {
110104
sess.err(
111105
&format!("could not load dep-graph from `{}`: {}",
112106
path.display(), err));
113-
None
114107
}
115108
}
109+
110+
if let Err(err) = delete_all_session_dir_contents(sess) {
111+
sess.err(&format!("could not clear incompatible incremental \
112+
compilation session directory `{}`: {}",
113+
path.display(), err));
114+
}
115+
116+
None
116117
}
117118

118119
/// Decode the dep graph and load the edges/nodes that are still clean
@@ -331,16 +332,22 @@ fn load_prev_metadata_hashes(tcx: TyCtxt,
331332

332333
debug!("load_prev_metadata_hashes() - File: {}", file_path.display());
333334

334-
let mut data = vec![];
335-
if !File::open(&file_path)
336-
.and_then(|mut file| file.read_to_end(&mut data)).is_ok() {
337-
debug!("load_prev_metadata_hashes() - Couldn't read file containing \
338-
hashes at `{}`", file_path.display());
339-
return
340-
}
335+
let data = match file_format::read_file(&file_path) {
336+
Ok(Some(data)) => data,
337+
Ok(None) => {
338+
debug!("load_prev_metadata_hashes() - File produced by incompatible \
339+
compiler version: {}", file_path.display());
340+
return
341+
}
342+
Err(err) => {
343+
debug!("load_prev_metadata_hashes() - Error reading file `{}`: {}",
344+
file_path.display(), err);
345+
return
346+
}
347+
};
341348

342349
debug!("load_prev_metadata_hashes() - Decoding hashes");
343-
let mut decoder = Decoder::new(&mut data, 0);
350+
let mut decoder = Decoder::new(&data, 0);
344351
let _ = Svh::decode(&mut decoder).unwrap();
345352
let serialized_hashes = SerializedMetadataHashes::decode(&mut decoder).unwrap();
346353

@@ -358,3 +365,4 @@ fn load_prev_metadata_hashes(tcx: TyCtxt,
358365
debug!("load_prev_metadata_hashes() - successfully loaded {} hashes",
359366
serialized_hashes.index_map.len());
360367
}
368+

src/librustc_incremental/persist/mod.rs

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ mod load;
2121
mod preds;
2222
mod save;
2323
mod work_product;
24+
mod file_format;
2425

2526
pub use self::fs::finalize_session_directory;
2627
pub use self::fs::in_incr_comp_dir;

src/librustc_incremental/persist/save.rs

+2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use super::hash::*;
2828
use super::preds::*;
2929
use super::fs::*;
3030
use super::dirty_clean;
31+
use super::file_format;
3132

3233
pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
3334
incremental_hashes_map: &IncrementalHashesMap,
@@ -102,6 +103,7 @@ fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
102103

103104
// generate the data in a memory buffer
104105
let mut wr = Cursor::new(Vec::new());
106+
file_format::write_file_header(&mut wr).unwrap();
105107
match encode(&mut Encoder::new(&mut wr)) {
106108
Ok(()) => {}
107109
Err(err) => {
+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// This test case makes sure that the compiler does not try to re-use anything
12+
// from the incremental compilation cache if the cache was produced by a
13+
// different compiler version. This is tested by artificially forcing the
14+
// emission of a different compiler version in the header of rpass1 artifacts,
15+
// and then making sure that the only object file of the test program gets
16+
// re-translated although the program stays unchanged.
17+
18+
// The `l33t haxx0r` Rust compiler is known to produce incr. comp. artifacts
19+
// that are outrageously incompatible with just about anything, even itself:
20+
//[rpass1] rustc-env:RUSTC_FORCE_INCR_COMP_ARTIFACT_HEADER="l33t haxx0r rustc 2.1 LTS"
21+
22+
// revisions:rpass1 rpass2
23+
24+
#![feature(rustc_attrs)]
25+
#![rustc_partition_translated(module="cache_file_headers", cfg="rpass2")]
26+
27+
fn main() {
28+
// empty
29+
}

0 commit comments

Comments
 (0)