auto merge of #8722 : graydon/rust/2013-08-23-test-shard, r=msullivan

This makes it relatively easy for us to split testsuite load between machines in buildbot. I've added buildbot-side support for setting up builders with -a.b suffixes (eg. linux-64-opt-vg-0.5, linux-64-opt-vg-1.5, linux-64-opt-vg-2.5, linux-64-opt-vg-3.5, linux-64-opt-vg-4.5 causes the valgrind-supervised testsuite to split 5 ways across hosts).

Author: bors

mk/tests.mk

@@ -73,16 +73,24 @@ TEST_RATCHET_NOISE_PERCENT=10.0
 
 # Whether to ratchet or merely save benchmarks
 ifdef CFG_RATCHET_BENCH
-CRATE_TEST_BENCH_ARGS=\
+CRATE_TEST_EXTRA_ARGS=\
   --test $(TEST_BENCH) \
   --ratchet-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) \
   --ratchet-noise-percent $(TEST_RATCHET_NOISE_PERCENT)
 else
-CRATE_TEST_BENCH_ARGS=\
+CRATE_TEST_EXTRA_ARGS=\
   --test $(TEST_BENCH) \
   --save-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4))
 endif
 
+# If we're sharding the testsuite between parallel testers,
+# pass this argument along to the compiletest and crate test
+# invocations.
+ifdef TEST_SHARD
+  CTEST_TESTARGS += --test-shard=$(TEST_SHARD)
+  CRATE_TEST_EXTRA_ARGS += --test-shard=$(TEST_SHARD)
+endif
+
 define DEF_TARGET_COMMANDS
 
 ifdef CFG_UNIXY_$(1)
@@ -401,7 +409,7 @@ $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
 	@$$(call E, run: $$<)
 	$$(Q)$$(call CFG_RUN_TEST_$(2),$$<,$(2),$(3)) $$(TESTARGS)	\
 	--logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),$(4)) \
-	$$(call CRATE_TEST_BENCH_ARGS,$(1),$(2),$(3),$(4)) \
+	$$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)) \
 	&& touch $$@
 endef
 
@@ -415,7 +423,7 @@ $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
 	@$(CFG_ADB) shell '(cd $(CFG_ADB_TEST_DIR); LD_LIBRARY_PATH=. \
 		./$$(notdir $$<) \
 		--logfile $(CFG_ADB_TEST_DIR)/check-stage$(1)-T-$(2)-H-$(3)-$(4).log \
-		$$(call CRATE_TEST_BENCH_ARGS,$(1),$(2),$(3),$(4)))' \
+		$$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)))' \
 		> tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
 	@cat tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
 	@touch tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).log

src/compiletest/common.rs

@@ -68,6 +68,11 @@ pub struct config {
     // Percent change in metrics to consider noise
     ratchet_noise_percent: Option<f64>,
 
+    // "Shard" of the testsuite to run: this has the form of
+    // two numbers (a,b), and causes only those tests with
+    // positional order equal to a mod b to run.
+    test_shard: Option<(uint,uint)>,
+
     // A command line to prefix program execution with,
     // for running under valgrind
     runtool: Option<~str>,

src/compiletest/compiletest.rs

@@ -75,6 +75,7 @@ pub fn parse_config(args: ~[~str]) -> config {
           optopt("", "target", "the target to build for", "TARGET"),
           optopt("", "adb-path", "path to the android debugger", "PATH"),
           optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
+          optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite", "A.B"),
           optflag("h", "help", "show this message"),
          ];
 
@@ -148,6 +149,7 @@ pub fn parse_config(args: ~[~str]) -> config {
                     ~"") { true }
                 else { false }
             } else { false },
+        test_shard: test::opt_shard(getopts::opt_maybe_str(matches, "test-shard")),
         verbose: getopts::opt_present(matches, "verbose")
     }
 }
@@ -172,6 +174,10 @@ pub fn log_config(config: &config) {
     logv(c, fmt!("adb_path: %s", config.adb_path));
     logv(c, fmt!("adb_test_dir: %s", config.adb_test_dir));
     logv(c, fmt!("adb_device_status: %b", config.adb_device_status));
+    match config.test_shard {
+        None => logv(c, ~"test_shard: (all)"),
+        Some((a,b)) => logv(c, fmt!("test_shard: %u.%u", a, b))
+    }
     logv(c, fmt!("verbose: %b", config.verbose));
     logv(c, fmt!("\n"));
 }
@@ -234,6 +240,7 @@ pub fn test_opts(config: &config) -> test::TestOpts {
         ratchet_metrics: config.ratchet_metrics.clone(),
         ratchet_noise_percent: config.ratchet_noise_percent.clone(),
         save_metrics: config.save_metrics.clone(),
+        test_shard: config.test_shard.clone()
     }
 }
 

src/libextra/test.rs

@@ -38,6 +38,7 @@ use std::task;
 use std::to_str::ToStr;
 use std::f64;
 use std::os;
+use std::uint;
 
 
 // The name of a test. By convention this follows the rules for rust
@@ -164,6 +165,7 @@ pub struct TestOpts {
     ratchet_metrics: Option<Path>,
     ratchet_noise_percent: Option<f64>,
     save_metrics: Option<Path>,
+    test_shard: Option<(uint,uint)>,
     logfile: Option<Path>
 }
 
@@ -184,7 +186,9 @@ fn optgroups() -> ~[getopts::groups::OptGroup] {
                      "Tests within N% of the recorded metrics will be \
                       considered as passing", "PERCENTAGE"),
       groups::optopt("", "logfile", "Write logs to the specified file instead \
-                          of stdout", "PATH")]
+                          of stdout", "PATH"),
+      groups::optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite",
+                     "A.B")]
 }
 
 fn usage(binary: &str, helpstr: &str) -> ! {
@@ -255,6 +259,9 @@ pub fn parse_opts(args: &[~str]) -> OptRes {
     let save_metrics = getopts::opt_maybe_str(&matches, "save-metrics");
     let save_metrics = save_metrics.map_move(|s| Path(s));
 
+    let test_shard = getopts::opt_maybe_str(&matches, "test-shard");
+    let test_shard = opt_shard(test_shard);
+
     let test_opts = TestOpts {
         filter: filter,
         run_ignored: run_ignored,
@@ -263,12 +270,29 @@ pub fn parse_opts(args: &[~str]) -> OptRes {
         ratchet_metrics: ratchet_metrics,
         ratchet_noise_percent: ratchet_noise_percent,
         save_metrics: save_metrics,
+        test_shard: test_shard,
         logfile: logfile
     };
 
     either::Left(test_opts)
 }
 
+pub fn opt_shard(maybestr: Option<~str>) -> Option<(uint,uint)> {
+    match maybestr {
+        None => None,
+        Some(s) => {
+            match s.split_iter('.').to_owned_vec() {
+                [a, b] => match (uint::from_str(a), uint::from_str(b)) {
+                    (Some(a), Some(b)) => Some((a,b)),
+                    _ => None
+                },
+                _ => None
+            }
+        }
+    }
+}
+
+
 #[deriving(Clone, Eq)]
 pub struct BenchSamples {
     ns_iter_summ: stats::Summary,
@@ -772,7 +796,15 @@ pub fn filter_tests(
     }
     sort::quick_sort(filtered, lteq);
 
-    filtered
+    // Shard the remaining tests, if sharding requested.
+    match opts.test_shard {
+        None => filtered,
+        Some((a,b)) =>
+            filtered.move_iter().enumerate()
+            .filter(|&(i,_)| i % b == a)
+            .map(|(_,t)| t)
+            .to_owned_vec()
+    }
 }
 
 struct TestFuture {
@@ -1234,6 +1266,7 @@ mod tests {
             ratchet_noise_percent: None,
             ratchet_metrics: None,
             save_metrics: None,
+            test_shard: None
         };
 
         let tests = ~[
@@ -1272,6 +1305,7 @@ mod tests {
             ratchet_noise_percent: None,
             ratchet_metrics: None,
             save_metrics: None,
+            test_shard: None
         };
 
         let names =