Update rustdoc testing to test all code blocks

It's too easy to forget the `rust` tag to have a code example tested, and it's
far more common to have testable code than untestable code.

This alters rustdoc to have only two directives, `ignore` and `should_fail`. The
`ignore` directive ignores the code block entirely, and the `should_fail`
directive has been fixed to only fail the test if the code execution fails, not
also compilation.

Author: alexcrichton

src/doc/rustdoc.md

@@ -100,34 +100,29 @@ rustdoc --test crate.rs
 
 ## Defining tests
 
-Rust documentation currently uses the markdown format, and code blocks can refer
-to any piece of code-related documentation, which isn't always rust. Because of
-this, only code blocks with the language of "rust" will be considered for
-testing.
+Rust documentation currently uses the markdown format, and rustdoc treats all
+code blocks as testable-by-default. In order to not run a test over a block of
+code, the `ignore` string can be added to the three-backtick form of markdown
+code block.
 
 ~~~
-```rust
+```
 // This is a testable code block
 ```
 
-```
+```ignore
 // This is not a testable code block
 ```
 
-    // This is not a testable code block (4-space indent)
+    // This is a testable code block (4-space indent)
 ~~~
 
-In addition to only testing "rust"-language code blocks, there are additional
-specifiers that can be used to dictate how a code block is tested:
+In addition to the `ignore` directive, you can specify that the test's execution
+should fail with the `should_fail` directive.
 
 ~~~
-```rust,ignore
-// This code block is ignored by rustdoc, but is passed through to the test
-// harness
-```
-
-```rust,should_fail
-// This code block is expected to generate a failure
+```should_fail
+// This code block is expected to generate a failure when run
 ```
 ~~~
 
@@ -143,7 +138,7 @@ that one can still write things like `#[deriving(Eq)]`).
 # the doc-generating tool.  In order to display them anyway in this particular
 # case, the character following the leading '#' is not a usual space like in
 # these first five lines but a non breakable one.
-# 
+#
 # // showing 'fib' in this documentation would just be tedious and detracts from
 # // what's actualy being documented.
 # fn fib(n: int) { n + 2 }
@@ -169,9 +164,6 @@ rustdoc --test lib.rs --test-args 'foo'
 
 // See what's possible when running tests
 rustdoc --test lib.rs --test-args '--help'
-
-// Run all ignored tests
-rustdoc --test lib.rs --test-args '--ignored'
 ~~~
 
 When testing a library, code examples will often show how functions are used,

src/librustdoc/html/markdown.rs

@@ -172,21 +172,23 @@ pub fn render(w: &mut io::Writer, s: &str) -> fmt::Result {
 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
     extern fn block(_ob: *buf, text: *buf, lang: *buf, opaque: *libc::c_void) {
         unsafe {
-            if text.is_null() || lang.is_null() { return }
-            let (test, shouldfail, ignore) =
+            if text.is_null() { return }
+            let (shouldfail, ignore) = if lang.is_null() {
+                (false, false)
+            } else {
                 vec::raw::buf_as_slice((*lang).data,
                                        (*lang).size as uint, |lang| {
                     let s = str::from_utf8(lang).unwrap();
-                    (s.contains("rust"), s.contains("should_fail"),
-                     s.contains("ignore"))
-                });
-            if !test { return }
+                    (s.contains("should_fail"), s.contains("ignore"))
+                })
+            };
+            if ignore { return }
             vec::raw::buf_as_slice((*text).data, (*text).size as uint, |text| {
                 let tests: &mut ::test::Collector = intrinsics::transmute(opaque);
                 let text = str::from_utf8(text).unwrap();
                 let mut lines = text.lines().map(|l| stripped_filtered_line(l).unwrap_or(l));
                 let text = lines.to_owned_vec().connect("\n");
-                tests.add_test(text, ignore, shouldfail);
+                tests.add_test(text, shouldfail);
             })
         }
     }

src/librustdoc/test.rs

@@ -94,7 +94,7 @@ pub fn run(input: &str, matches: &getopts::Matches) -> int {
     0
 }
 
-fn runtest(test: &str, cratename: &str, libs: HashSet<Path>) {
+fn runtest(test: &str, cratename: &str, libs: HashSet<Path>, should_fail: bool) {
     let test = maketest(test, cratename);
     let parsesess = parse::new_parse_sess();
     let input = driver::StrInput(test);
@@ -130,9 +130,10 @@ fn runtest(test: &str, cratename: &str, libs: HashSet<Path>) {
     match out {
         Err(e) => fail!("couldn't run the test: {}", e),
         Ok(out) => {
-            if !out.status.success() {
-                fail!("test executable failed:\n{}",
-                      str::from_utf8(out.error));
+            if should_fail && out.status.success() {
+                fail!("test executable succeeded when it should have failed");
+            } else if !should_fail && !out.status.success() {
+                fail!("test executable failed:\n{}", str::from_utf8(out.error));
             }
         }
     }
@@ -169,7 +170,7 @@ pub struct Collector {
 }
 
 impl Collector {
-    pub fn add_test(&mut self, test: &str, ignore: bool, should_fail: bool) {
+    pub fn add_test(&mut self, test: &str, should_fail: bool) {
         let test = test.to_owned();
         let name = format!("{}_{}", self.names.connect("::"), self.cnt);
         self.cnt += 1;
@@ -180,11 +181,11 @@ impl Collector {
         self.tests.push(test::TestDescAndFn {
             desc: test::TestDesc {
                 name: test::DynTestName(name),
-                ignore: ignore,
-                should_fail: should_fail,
+                ignore: false,
+                should_fail: false, // compiler failures are test failures
             },
             testfn: test::DynTestFn(proc() {
-                runtest(test, cratename, libs);
+                runtest(test, cratename, libs, should_fail);
             }),
         });
     }