Skip to content

Commit 7ebf2cd

Browse files
authored
Rollup merge of #108772 - jyn514:faster-tidy, r=the8472
Speed up tidy quite a lot I highly recommend reviewing this commit-by-commit. Based on #106440 for convenience. ## Timings These were collected by running `x test tidy -v` to copy paste the command, then using [`samply record`](https://github.com/mstange/samply). before (8 threads) ![image](https://user-images.githubusercontent.com/23638587/222965319-352ad2c8-367c-4d74-960a-e4bb161a6aff.png) after (8 threads) ![image](https://user-images.githubusercontent.com/23638587/222965323-fa846f4e-727a-4bf8-8e3b-1b7b40505cc3.png) before (64 threads) ![image](https://user-images.githubusercontent.com/23638587/222965302-dc88020c-19e9-49d9-a87d-cad054d717f3.png) after (64 threads) ![image](https://user-images.githubusercontent.com/23638587/222965335-e73d7622-59de-41d2-9cc4-1bd67042a349.png) The last commit makes tidy use more threads, so comparing "before (8 threads)" to "after (64 threads)" is IMO the most realistic comparison. Locally, that brings the time for me to run tidy down from 4 to .9 seconds, i.e. the majority of the time for `x test tidy` is now spend running `fmt --check`. r? `@the8472`
2 parents 9599f3c + 675c4aa commit 7ebf2cd

File tree

13 files changed

+225
-187
lines changed

13 files changed

+225
-187
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ no_llvm_build
4242
/llvm/
4343
/mingw-build/
4444
build/
45+
!/compiler/rustc_mir_build/src/build/
4546
/build-rust-analyzer/
4647
/dist/
4748
/unicode-downloads

compiler/rustc_mir_build/src/build/expr/as_rvalue.rs

+35-12
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,34 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
7373
}
7474
ExprKind::Binary { op, lhs, rhs } => {
7575
let lhs = unpack!(
76-
block =
77-
this.as_operand(block, scope, &this.thir[lhs], LocalInfo::Boring, NeedsTemporary::Maybe)
76+
block = this.as_operand(
77+
block,
78+
scope,
79+
&this.thir[lhs],
80+
LocalInfo::Boring,
81+
NeedsTemporary::Maybe
82+
)
7883
);
7984
let rhs = unpack!(
80-
block =
81-
this.as_operand(block, scope, &this.thir[rhs], LocalInfo::Boring, NeedsTemporary::No)
85+
block = this.as_operand(
86+
block,
87+
scope,
88+
&this.thir[rhs],
89+
LocalInfo::Boring,
90+
NeedsTemporary::No
91+
)
8292
);
8393
this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs)
8494
}
8595
ExprKind::Unary { op, arg } => {
8696
let arg = unpack!(
87-
block =
88-
this.as_operand(block, scope, &this.thir[arg], LocalInfo::Boring, NeedsTemporary::No)
97+
block = this.as_operand(
98+
block,
99+
scope,
100+
&this.thir[arg],
101+
LocalInfo::Boring,
102+
NeedsTemporary::No
103+
)
89104
);
90105
// Check for -MIN on signed integers
91106
if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() {
@@ -272,8 +287,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
272287
}
273288
ExprKind::Pointer { cast, source } => {
274289
let source = unpack!(
275-
block =
276-
this.as_operand(block, scope, &this.thir[source], LocalInfo::Boring, NeedsTemporary::No)
290+
block = this.as_operand(
291+
block,
292+
scope,
293+
&this.thir[source],
294+
LocalInfo::Boring,
295+
NeedsTemporary::No
296+
)
277297
);
278298
block.and(Rvalue::Cast(CastKind::Pointer(cast), source, expr.ty))
279299
}
@@ -502,8 +522,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
502522
Category::of(&expr.kind),
503523
Some(Category::Rvalue(RvalueFunc::AsRvalue) | Category::Constant)
504524
));
505-
let operand =
506-
unpack!(block = this.as_operand(block, scope, expr, LocalInfo::Boring, NeedsTemporary::No));
525+
let operand = unpack!(
526+
block =
527+
this.as_operand(block, scope, expr, LocalInfo::Boring, NeedsTemporary::No)
528+
);
507529
block.and(Rvalue::Use(operand))
508530
}
509531
}
@@ -662,8 +684,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
662684
// Repeating a const does nothing
663685
} else {
664686
// For a non-const, we may need to generate an appropriate `Drop`
665-
let value_operand =
666-
unpack!(block = this.as_operand(block, scope, value, LocalInfo::Boring, NeedsTemporary::No));
687+
let value_operand = unpack!(
688+
block = this.as_operand(block, scope, value, LocalInfo::Boring, NeedsTemporary::No)
689+
);
667690
if let Operand::Move(to_drop) = value_operand {
668691
let success = this.cfg.start_new_block();
669692
this.cfg.terminate(

compiler/rustc_mir_build/src/build/matches/mod.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -2252,7 +2252,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
22522252
user_ty: None,
22532253
source_info,
22542254
internal: false,
2255-
local_info: ClearCrossCrate::Set(Box::new(LocalInfo::User(BindingForm::RefForGuard))),
2255+
local_info: ClearCrossCrate::Set(Box::new(LocalInfo::User(
2256+
BindingForm::RefForGuard,
2257+
))),
22562258
});
22572259
self.var_debug_info.push(VarDebugInfo {
22582260
name,

compiler/rustc_mir_build/src/build/mod.rs

+8-11
Original file line numberDiff line numberDiff line change
@@ -876,21 +876,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
876876
} => {
877877
self.local_decls[local].mutability = mutability;
878878
self.local_decls[local].source_info.scope = self.source_scope;
879-
**self.local_decls[local].local_info.as_mut().assert_crate_local() = if let Some(kind) = param.self_kind {
880-
LocalInfo::User(
881-
BindingForm::ImplicitSelf(kind),
882-
)
883-
} else {
884-
let binding_mode = ty::BindingMode::BindByValue(mutability);
885-
LocalInfo::User(BindingForm::Var(
886-
VarBindingForm {
879+
**self.local_decls[local].local_info.as_mut().assert_crate_local() =
880+
if let Some(kind) = param.self_kind {
881+
LocalInfo::User(BindingForm::ImplicitSelf(kind))
882+
} else {
883+
let binding_mode = ty::BindingMode::BindByValue(mutability);
884+
LocalInfo::User(BindingForm::Var(VarBindingForm {
887885
binding_mode,
888886
opt_ty_info: param.ty_span,
889887
opt_match_place: Some((None, span)),
890888
pat_span: span,
891-
},
892-
))
893-
};
889+
}))
890+
};
894891
self.var_indices.insert(var, LocalsForNode::One(local));
895892
}
896893
_ => {

src/bootstrap/test.rs

+5-1
Original file line numberDiff line numberDiff line change
@@ -1118,7 +1118,11 @@ impl Step for Tidy {
11181118
cmd.arg(&builder.src);
11191119
cmd.arg(&builder.initial_cargo);
11201120
cmd.arg(&builder.out);
1121-
cmd.arg(builder.jobs().to_string());
1121+
// Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured.
1122+
let jobs = builder.config.jobs.unwrap_or_else(|| {
1123+
8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32
1124+
});
1125+
cmd.arg(jobs.to_string());
11221126
if builder.is_verbose() {
11231127
cmd.arg("--verbose");
11241128
}

src/tools/tidy/src/bins.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ mod os_impl {
103103

104104
// FIXME: we don't need to look at all binaries, only files that have been modified in this branch
105105
// (e.g. using `git ls-files`).
106-
walk_no_read(path, |path| filter_dirs(path) || path.ends_with("src/etc"), &mut |entry| {
106+
walk_no_read(&[path], |path| filter_dirs(path) || path.ends_with("src/etc"), &mut |entry| {
107107
let file = entry.path();
108108
let extension = file.extension();
109109
let scripts = ["py", "sh", "ps1"];

src/tools/tidy/src/debug_artifacts.rs

+3-9
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,15 @@
11
//! Tidy check to prevent creation of unnecessary debug artifacts while running tests.
22
3-
use crate::walk::{filter_dirs, walk};
3+
use crate::walk::{filter_dirs, filter_not_rust, walk};
44
use std::path::Path;
55

66
const GRAPHVIZ_POSTFLOW_MSG: &str = "`borrowck_graphviz_postflow` attribute in test";
77

88
pub fn check(test_dir: &Path, bad: &mut bool) {
9-
walk(test_dir, filter_dirs, &mut |entry, contents| {
10-
let filename = entry.path();
11-
let is_rust = filename.extension().map_or(false, |ext| ext == "rs");
12-
if !is_rust {
13-
return;
14-
}
15-
9+
walk(test_dir, |path| filter_dirs(path) || filter_not_rust(path), &mut |entry, contents| {
1610
for (i, line) in contents.lines().enumerate() {
1711
if line.contains("borrowck_graphviz_postflow") {
18-
tidy_error!(bad, "{}:{}: {}", filename.display(), i + 1, GRAPHVIZ_POSTFLOW_MSG);
12+
tidy_error!(bad, "{}:{}: {}", entry.path().display(), i + 1, GRAPHVIZ_POSTFLOW_MSG);
1913
}
2014
}
2115
});

src/tools/tidy/src/features.rs

+8-9
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@
99
//! * All unstable lang features have tests to ensure they are actually unstable.
1010
//! * Language features in a group are sorted by feature name.
1111
12-
use crate::walk::{filter_dirs, walk, walk_many};
12+
use crate::walk::{filter_dirs, filter_not_rust, walk, walk_many};
1313
use std::collections::hash_map::{Entry, HashMap};
14+
use std::ffi::OsStr;
1415
use std::fmt;
1516
use std::fs;
1617
use std::num::NonZeroU32;
@@ -101,17 +102,15 @@ pub fn check(
101102
&tests_path.join("rustdoc-ui"),
102103
&tests_path.join("rustdoc"),
103104
],
104-
filter_dirs,
105+
|path| {
106+
filter_dirs(path)
107+
|| filter_not_rust(path)
108+
|| path.file_name() == Some(OsStr::new("features.rs"))
109+
|| path.file_name() == Some(OsStr::new("diagnostic_list.rs"))
110+
},
105111
&mut |entry, contents| {
106112
let file = entry.path();
107113
let filename = file.file_name().unwrap().to_string_lossy();
108-
if !filename.ends_with(".rs")
109-
|| filename == "features.rs"
110-
|| filename == "diagnostic_list.rs"
111-
{
112-
return;
113-
}
114-
115114
let filen_underscore = filename.replace('-', "_").replace(".rs", "");
116115
let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);
117116

src/tools/tidy/src/main.rs

+16-5
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use std::path::PathBuf;
1313
use std::process;
1414
use std::str::FromStr;
1515
use std::sync::atomic::{AtomicBool, Ordering};
16-
use std::thread::{scope, ScopedJoinHandle};
16+
use std::thread::{self, scope, ScopedJoinHandle};
1717

1818
fn main() {
1919
let root_path: PathBuf = env::args_os().nth(1).expect("need path to root of repo").into();
@@ -55,16 +55,28 @@ fn main() {
5555
VecDeque::with_capacity(concurrency.get());
5656

5757
macro_rules! check {
58-
($p:ident $(, $args:expr)* ) => {
58+
($p:ident) => {
59+
check!(@ $p, name=format!("{}", stringify!($p)));
60+
};
61+
($p:ident, $path:expr $(, $args:expr)* ) => {
62+
let shortened = $path.strip_prefix(&root_path).unwrap();
63+
let name = if shortened == std::path::Path::new("") {
64+
format!("{} (.)", stringify!($p))
65+
} else {
66+
format!("{} ({})", stringify!($p), shortened.display())
67+
};
68+
check!(@ $p, name=name, $path $(,$args)*);
69+
};
70+
(@ $p:ident, name=$name:expr $(, $args:expr)* ) => {
5971
drain_handles(&mut handles);
6072

61-
let handle = s.spawn(|| {
73+
let handle = thread::Builder::new().name($name).spawn_scoped(s, || {
6274
let mut flag = false;
6375
$p::check($($args, )* &mut flag);
6476
if (flag) {
6577
bad.store(true, Ordering::Relaxed);
6678
}
67-
});
79+
}).unwrap();
6880
handles.push_back(handle);
6981
}
7082
}
@@ -108,7 +120,6 @@ fn main() {
108120
check!(edition, &library_path);
109121

110122
check!(alphabetical, &src_path);
111-
check!(alphabetical, &tests_path);
112123
check!(alphabetical, &compiler_path);
113124
check!(alphabetical, &library_path);
114125

src/tools/tidy/src/style.rs

+19-12
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
use crate::walk::{filter_dirs, walk};
2121
use regex::{Regex, RegexSet};
22-
use std::path::Path;
22+
use std::{ffi::OsStr, path::Path};
2323

2424
/// Error code markdown is restricted to 80 columns because they can be
2525
/// displayed on the console with --example.
@@ -228,21 +228,33 @@ fn is_unexplained_ignore(extension: &str, line: &str) -> bool {
228228

229229
pub fn check(path: &Path, bad: &mut bool) {
230230
fn skip(path: &Path) -> bool {
231-
filter_dirs(path) || skip_markdown_path(path)
231+
if path.file_name().map_or(false, |name| name.to_string_lossy().starts_with(".#")) {
232+
// vim or emacs temporary file
233+
return true;
234+
}
235+
236+
if filter_dirs(path) || skip_markdown_path(path) {
237+
return true;
238+
}
239+
240+
let extensions = ["rs", "py", "js", "sh", "c", "cpp", "h", "md", "css", "ftl", "goml"];
241+
if extensions.iter().all(|e| path.extension() != Some(OsStr::new(e))) {
242+
return true;
243+
}
244+
245+
// We only check CSS files in rustdoc.
246+
path.extension().map_or(false, |e| e == "css") && !is_in(path, "src", "librustdoc")
232247
}
248+
233249
let problematic_consts_strings: Vec<String> = (PROBLEMATIC_CONSTS.iter().map(u32::to_string))
234250
.chain(PROBLEMATIC_CONSTS.iter().map(|v| format!("{:x}", v)))
235251
.chain(PROBLEMATIC_CONSTS.iter().map(|v| format!("{:X}", v)))
236252
.collect();
237253
let problematic_regex = RegexSet::new(problematic_consts_strings.as_slice()).unwrap();
254+
238255
walk(path, skip, &mut |entry, contents| {
239256
let file = entry.path();
240257
let filename = file.file_name().unwrap().to_string_lossy();
241-
let extensions =
242-
[".rs", ".py", ".js", ".sh", ".c", ".cpp", ".h", ".md", ".css", ".ftl", ".goml"];
243-
if extensions.iter().all(|e| !filename.ends_with(e)) || filename.starts_with(".#") {
244-
return;
245-
}
246258

247259
let is_style_file = filename.ends_with(".css");
248260
let under_rustfmt = filename.ends_with(".rs") &&
@@ -253,11 +265,6 @@ pub fn check(path: &Path, bad: &mut bool) {
253265
a.ends_with("src/doc/book")
254266
});
255267

256-
if is_style_file && !is_in(file, "src", "librustdoc") {
257-
// We only check CSS files in rustdoc.
258-
return;
259-
}
260-
261268
if contents.is_empty() {
262269
tidy_error!(bad, "{}: empty file", file.display());
263270
}

0 commit comments

Comments
 (0)