Skip to content

Commit 060bad5

Browse files
brsonbrson
committed
Clean up tidy scripts, coverage, performance
This restructures tidy.py to walk the tree itself, and improves performance considerably by not loading entire files into buffers for licenseck. Splits build rules into 'tidy', 'tidy-basic', 'tidy-binaries', 'tidy-errors', 'tidy-features'.
1 parent aa1d59e commit 060bad5

File tree

28 files changed

+196
-106
lines changed

28 files changed

+196
-106
lines changed

‎mk/tests.mk

Lines changed: 2 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -235,30 +235,10 @@ cleantestlibs:
235235
######################################################################
236236

237237
ifdef CFG_NOTIDY
238+
.PHONY: tidy
238239
tidy:
239240
else
240241

241-
ALL_CS := $(wildcard $(S)src/rt/*.cpp \
242-
$(S)src/rt/*/*.cpp \
243-
$(S)src/rt/*/*/*.cpp \
244-
$(S)src/rustllvm/*.cpp)
245-
ALL_CS := $(filter-out $(S)src/rt/miniz.cpp \
246-
$(wildcard $(S)src/rt/hoedown/src/*.c) \
247-
$(wildcard $(S)src/rt/hoedown/bin/*.c) \
248-
,$(ALL_CS))
249-
ALL_HS := $(wildcard $(S)src/rt/*.h \
250-
$(S)src/rt/*/*.h \
251-
$(S)src/rt/*/*/*.h \
252-
$(S)src/rustllvm/*.h)
253-
ALL_HS := $(filter-out $(S)src/rt/valgrind/valgrind.h \
254-
$(S)src/rt/valgrind/memcheck.h \
255-
$(S)src/rt/msvc/typeof.h \
256-
$(S)src/rt/msvc/stdint.h \
257-
$(S)src/rt/msvc/inttypes.h \
258-
$(wildcard $(S)src/rt/hoedown/src/*.h) \
259-
$(wildcard $(S)src/rt/hoedown/bin/*.h) \
260-
,$(ALL_HS))
261-
262242
# Run the tidy script in multiple parts to avoid huge 'echo' commands
263243
.PHONY: tidy
264244
tidy: tidy-basic tidy-binaries tidy-errors tidy-features
@@ -268,30 +248,7 @@ endif
268248
.PHONY: tidy-basic
269249
tidy-basic:
270250
@$(call E, check: formatting)
271-
$(Q)find $(S)src -name '*.r[sc]' \
272-
-and -not -regex '^$(S)src/jemalloc.*' \
273-
-and -not -regex '^$(S)src/libuv.*' \
274-
-and -not -regex '^$(S)src/llvm.*' \
275-
-and -not -regex '^$(S)src/gyp.*' \
276-
-and -not -regex '^$(S)src/libbacktrace.*' \
277-
-print0 \
278-
| xargs -0 -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
279-
$(Q)find $(S)src/etc -name '*.py' \
280-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
281-
$(Q)find $(S)src/doc -name '*.js' \
282-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
283-
$(Q)find $(S)src/etc -name '*.sh' \
284-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
285-
$(Q)find $(S)src/etc -name '*.pl' \
286-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
287-
$(Q)find $(S)src/etc -name '*.c' \
288-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
289-
$(Q)find $(S)src/etc -name '*.h' \
290-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
291-
$(Q)echo $(ALL_CS) \
292-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
293-
$(Q)echo $(ALL_HS) \
294-
| xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
251+
$(Q) $(CFG_PYTHON) $(S)src/etc/tidy.py $(S)src/
295252

296253
.PHONY: tidy-binaries
297254
tidy-binaries:

‎src/etc/errorck.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313

1414
import sys, os, re
1515

16+
if len(sys.argv) < 2:
17+
print "usage: errorck.py <src-dir>"
18+
sys.exit(1)
19+
1620
src_dir = sys.argv[1]
1721

1822
errcode_map = { }

‎src/etc/featureck.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020

2121
import sys, os, re
2222

23+
if len(sys.argv) < 2:
24+
print "usage: featurkck.py <src-dir>"
25+
sys.exit(1)
26+
2327
src_dir = sys.argv[1]
2428

2529
# Features that are allowed to exist in both the language and the library

‎src/etc/licenseck.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,6 @@
3333
"""
3434

3535
exceptions = [
36-
"rt/rust_android_dummy.cpp", # BSD, chromium
37-
"rt/rust_android_dummy.h", # BSD, chromium
38-
"rt/isaac/randport.cpp", # public domain
39-
"rt/isaac/rand.h", # public domain
40-
"rt/isaac/standard.h", # public domain
4136
"libstd/sync/mpsc/mpsc_queue.rs", # BSD
4237
"libstd/sync/mpsc/spsc_queue.rs", # BSD
4338
"test/bench/shootout-binarytrees.rs", # BSD

‎src/etc/tidy.py

Lines changed: 155 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -43,78 +43,184 @@ def do_license_check(name, contents):
4343
if not check_license(name, contents):
4444
report_error_name_no(name, 1, "incorrect license")
4545

46-
47-
file_names = [s for s in sys.argv[1:] if (not s.endswith("_gen.rs"))
48-
and (not ".#" in s)]
49-
5046
current_name = ""
5147
current_contents = ""
5248
check_tab = True
5349
check_cr = True
5450
check_linelength = True
5551

52+
if len(sys.argv) < 2:
53+
print "usage: tidy.py <src-dir>"
54+
sys.exit(1)
55+
56+
src_dir = sys.argv[1]
5657

5758
try:
58-
for line in fileinput.input(file_names,
59+
count_rs = 0
60+
count_py = 0
61+
count_js = 0
62+
count_sh = 0
63+
count_pl = 0
64+
count_c = 0
65+
count_h = 0
66+
count_other = 0
67+
68+
count_lines = 0
69+
count_non_blank_lines = 0
70+
71+
def update_counts(current_name):
72+
global count_rs
73+
global count_py
74+
global count_js
75+
global count_sh
76+
global count_pl
77+
global count_c
78+
global count_h
79+
global count_other
80+
81+
if current_name.endswith(".rs"):
82+
count_rs += 1
83+
if current_name.endswith(".py"):
84+
count_py += 1
85+
if current_name.endswith(".js"):
86+
count_js += 1
87+
if current_name.endswith(".sh"):
88+
count_sh += 1
89+
if current_name.endswith(".pl"):
90+
count_pl += 1
91+
if current_name.endswith(".c"):
92+
count_c += 1
93+
if current_name.endswith(".h"):
94+
count_h += 1
95+
96+
all_paths = set()
97+
98+
for (dirpath, dirnames, filenames) in os.walk(src_dir):
99+
100+
# Skip some third-party directories
101+
if "src/jemalloc" in dirpath: continue
102+
if "src/llvm" in dirpath: continue
103+
if "src/gyp" in dirpath: continue
104+
if "src/libbacktrace" in dirpath: continue
105+
if "src/compiler-rt" in dirpath: continue
106+
if "src/rt/hoedown" in dirpath: continue
107+
if "src/rustllvm" in dirpath: continue
108+
if "src/rt/valgrind" in dirpath: continue
109+
if "src/rt/msvc" in dirpath: continue
110+
if "src/rust-installer" in dirpath: continue
111+
112+
def interesting_file(f):
113+
if "miniz.c" in f \
114+
or "jquery" in f \
115+
or "rust_android_dummy" in f:
116+
return False
117+
118+
if f.endswith(".rs") \
119+
or f.endswith(".py") \
120+
or f.endswith(".js") \
121+
or f.endswith(".sh") \
122+
or f.endswith(".pl") \
123+
or f.endswith(".c") \
124+
or f.endswith(".h") :
125+
return True
126+
else:
127+
return False
128+
129+
file_names = [os.path.join(dirpath, f) for f in filenames
130+
if interesting_file(f)
131+
and not f.endswith("_gen.rs")
132+
and not ".#" is f]
133+
134+
if not file_names:
135+
continue
136+
137+
for line in fileinput.input(file_names,
59138
openhook=fileinput.hook_encoded("utf-8")):
60139

61-
if "tidy.py" not in fileinput.filename():
140+
filename = fileinput.filename()
141+
142+
if "tidy.py" not in filename:
143+
if "TODO" in line:
144+
report_err("TODO is deprecated; use FIXME")
145+
match = re.match(r'^.*/(\*|/!?)\s*XXX', line)
146+
if match:
147+
report_err("XXX is no longer necessary, use FIXME")
148+
match = re.match(r'^.*//\s*(NOTE.*)$', line)
149+
if match and "TRAVIS" not in os.environ:
150+
m = match.group(1)
151+
if "snap" in m.lower():
152+
report_warn(match.group(1))
153+
match = re.match(r'^.*//\s*SNAP\s+(\w+)', line)
154+
if match:
155+
hsh = match.group(1)
156+
date, rev = snapshot.curr_snapshot_rev()
157+
if not hsh.startswith(rev):
158+
report_err("snapshot out of date (" + date
159+
+ "): " + line)
160+
else:
161+
if "SNAP" in line:
162+
report_warn("unmatched SNAP line: " + line)
163+
62164
if cr_flag in line:
63165
check_cr = False
64166
if tab_flag in line:
65167
check_tab = False
66168
if linelength_flag in line:
67169
check_linelength = False
68-
if "TODO" in line:
69-
report_err("TODO is deprecated; use FIXME")
70-
match = re.match(r'^.*/(\*|/!?)\s*XXX', line)
71-
if match:
72-
report_err("XXX is no longer necessary, use FIXME")
73-
match = re.match(r'^.*//\s*(NOTE.*)$', line)
74-
if match and "TRAVIS" not in os.environ:
75-
m = match.group(1)
76-
if "snap" in m.lower():
77-
report_warn(match.group(1))
78-
match = re.match(r'^.*//\s*SNAP\s+(\w+)', line)
79-
if match:
80-
hsh = match.group(1)
81-
date, rev = snapshot.curr_snapshot_rev()
82-
if not hsh.startswith(rev):
83-
report_err("snapshot out of date (" + date
84-
+ "): " + line)
85-
else:
86-
if "SNAP" in line:
87-
report_warn("unmatched SNAP line: " + line)
88-
89-
if check_tab and ('\t' in line and
90-
"Makefile" not in fileinput.filename()):
91-
report_err("tab character")
92-
if check_cr and not autocrlf and '\r' in line:
93-
report_err("CR character")
94-
if line.endswith(" \n") or line.endswith("\t\n"):
95-
report_err("trailing whitespace")
96-
line_len = len(line)-2 if autocrlf else len(line)-1
97-
98-
if check_linelength and line_len > cols:
99-
report_err("line longer than %d chars" % cols)
100-
101-
if fileinput.isfirstline() and current_name != "":
102-
do_license_check(current_name, current_contents)
103-
104-
if fileinput.isfirstline():
105-
current_name = fileinput.filename()
106-
current_contents = ""
107-
check_cr = True
108-
check_tab = True
109-
check_linelength = True
110170

111-
current_contents += line
171+
if check_tab and ('\t' in line and
172+
"Makefile" not in filename):
173+
report_err("tab character")
174+
if check_cr and not autocrlf and '\r' in line:
175+
report_err("CR character")
176+
if line.endswith(" \n") or line.endswith("\t\n"):
177+
report_err("trailing whitespace")
178+
line_len = len(line)-2 if autocrlf else len(line)-1
179+
180+
if check_linelength and line_len > cols:
181+
report_err("line longer than %d chars" % cols)
182+
183+
if fileinput.isfirstline():
184+
# This happens at the end of each file except the last.
185+
if current_name != "":
186+
update_counts(current_name)
187+
assert len(current_contents) > 0
188+
do_license_check(current_name, current_contents)
189+
190+
current_name = filename
191+
current_contents = ""
192+
check_cr = True
193+
check_tab = True
194+
check_linelength = True
195+
196+
# Put a reasonable limit on the amount of header data we use for
197+
# the licenseck
198+
if len(current_contents) < 1000:
199+
current_contents += line
200+
201+
count_lines += 1
202+
if line.strip():
203+
count_non_blank_lines += 1
112204

113205
if current_name != "":
206+
update_counts(current_name)
207+
assert len(current_contents) > 0
114208
do_license_check(current_name, current_contents)
115209

116210
except UnicodeDecodeError, e:
117211
report_err("UTF-8 decoding error " + str(e))
118212

213+
print
214+
print "* linted .rs files: " + str(count_rs)
215+
print "* linted .py files: " + str(count_py)
216+
print "* linted .js files: " + str(count_js)
217+
print "* linted .sh files: " + str(count_sh)
218+
print "* linted .pl files: " + str(count_pl)
219+
print "* linted .c files: " + str(count_c)
220+
print "* linted .h files: " + str(count_h)
221+
print "* other linted files: " + str(count_other)
222+
print "* total lines of code: " + str(count_lines)
223+
print "* total non-blank lines of code: " + str(count_non_blank_lines)
224+
print
119225

120226
sys.exit(err)

‎src/grammar/check.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/sh
22

3+
# ignore-license
4+
35
# Run the reference lexer against libsyntax and compare the tokens and spans.
46
# If "// ignore-lexer-test" is present in the file, it will be ignored.
57

‎src/test/run-make/c-dynamic-dylib/cfoo.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
// ignore-license
12
int foo() { return 0; }

‎src/test/run-make/c-dynamic-rlib/cfoo.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
// ignore-license
12
int foo() { return 0; }

‎src/test/run-make/c-link-to-rust-dylib/bar.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// ignore-license
12
void foo();
23

34
int main() {

‎src/test/run-make/c-link-to-rust-staticlib/bar.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// ignore-license
12
void foo();
23

34
int main() {

‎src/test/run-make/c-static-dylib/cfoo.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
// ignore-license
12
int foo() { return 0; }

‎src/test/run-make/c-static-rlib/cfoo.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
// ignore-license
12
int foo() { return 0; }

‎src/test/run-make/extern-fn-generic/test.c

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
1+
// ignore-license
12
#include <stdint.h>
23

34
typedef struct TestStruct {
4-
uint8_t x;
5-
int32_t y;
5+
uint8_t x;
6+
int32_t y;
67
} TestStruct;
78

89
typedef int callback(TestStruct s);
910

1011
uint32_t call(callback *c) {
11-
TestStruct s;
12-
s.x = 'a';
13-
s.y = 3;
12+
TestStruct s;
13+
s.x = 'a';
14+
s.y = 3;
1415

15-
return c(s);
16+
return c(s);
1617
}

‎src/test/run-make/extern-fn-mangle/test.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1+
// ignore-license
12
#include <stdint.h>
23

34
uint32_t foo();
45
uint32_t bar();
56

67
uint32_t add() {
7-
return foo() + bar();
8+
return foo() + bar();
89
}

0 commit comments

Comments
 (0)