Merge pull request #120 from ltratt/ignore_if

Replace `ignore` with `ignore-if`.

examples/rust_lang_tester/lang_tests/ignore.rs

@@ -1,4 +1,5 @@
-// ignore: this test is intentionally ignored
+// # Always ignore this test
+// ignore-if: echo 123 | grep 2
 // Compiler:
 //   status: success
 

examples/rust_lang_tester/lang_tests/not_ignore.rs

@@ -0,0 +1,10 @@
+// # Never ignore this test.
+// ignore-if: echo 123 | grep 4
+// Run-time:
+//   stdout:
+//     # an ignored comment
+//     check
+
+fn main() {
+    println!("check");
+}

examples/rust_lang_tester/run_tests.rs

@@ -17,6 +17,8 @@ fn main() {
     let tempdir = TempDir::new().unwrap();
     LangTester::new()
         .test_dir("examples/rust_lang_tester/lang_tests")
+        // Treat top-level lines beginning with "#" as comments.
+        .comment_prefix("#")
         // Only use files named `*.rs` as test files.
         .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("rs"))
         // Extract the first sequence of commented line(s) as the tests.

src/lib.rs

@@ -138,32 +138,31 @@
 //!   * `exec-arg: <string>` specifies a string which will be passed as an additional command-line
 //!     argument to the command (in addition to those specified by the `test_cmds` function).
 //!     Multiple `exec-arg`s can be specified, each adding an additional command-line argument.
-//!   * `stdin: <string>`, text to be passed to the command's `stdin`. If the command exits without
-//!     having consumed all of `<string>`, an error will be raised. Note, though, that operating
-//!     system file buffers can mean that the command *appears* to have consumed all of `<string>`
-//!     without it actually having done so.
+//!   * `stdin: <string>` specifies text to be passed to the command's `stdin`. If the command
+//!     exits without consuming all of `<string>`, an error will be raised. Note, though, that
+//!     operating system file buffers can mean that the command *appears* to have consumed all of
+//!     `<string>` without it actually having done so.
 //!
 //! Test commands can specify that a test should be rerun if one of the following (optional) is
 //! specified and it matches the test's output:
 //!
-//!   * `rerun-if-status`: follows the same format as the `status`.
-//!   * `rerun-if-stderr` and `rerun-if-stdout`: follow the same format as `stderr` and `stdout`.
+//!   * `rerun-if-status` follows the same format as the `status`.
+//!   * `rerun-if-stderr` and `rerun-if-stdout` follow the same format as `stderr` and `stdout`.
 //!
 //! These can be useful if tests are subject to intermittent errors (e.g. network failure) that
 //! should not be considered as a failure of the test itself. Test commands are rerun at most *n*
 //! times, which by default is specified as 3. If no `rerun-if-` is specified, then the first time
-//! a test fails, it will be reported to the users.
+//! a test fails, it will be reported to the user.
 //!
 //! The above file thus contains 4 meaningful tests, two specified by the user and two implied by
 //! defaults: the `Compiler` should succeed (e.g. return a `0` exit code when run on Unix), and
 //! its `stderr` output should warn about an unused variable on line 12; and the resulting binary
 //! should succeed produce `Hello world` on `stdout`.
 //!
-//! A file's tests can be ignored entirely if a test command `ignore` is defined:
+//! A file's tests can be ignored entirely with:
 //!
-//!   * `ignore: [<string>]`, specifies that this file should be ignored for the reason set out in
-//!     `<string>` (if any). Note that `<string>` is purely for user information and has no effect
-//!     on the running of tests.
+//!   * `ignore-if: <cmd>` defines a shell command that will be run to determine whether to ignore
+//!     this test or not. If `<cmd>` returns 0 the test will be ignored, otherwise it will be run.
 //!
 //! `lang_tester`'s output is deliberately similar to Rust's normal testing output. Running the
 //! example `rust_lang_tester` in this crate produces the following output:

src/parser.rs

@@ -6,20 +6,26 @@ use crate::{
 };
 
 /// Parse test data into a set of `Test`s.
-pub(crate) fn parse_tests(test_str: &str) -> Tests {
+pub(crate) fn parse_tests<'a>(comment_prefix: Option<&str>, test_str: &'a str) -> Tests<'a> {
     let lines = test_str.lines().collect::<Vec<_>>();
     let mut tests = HashMap::new();
     let mut line_off = 0;
-    let mut ignore = false;
+    let mut ignore_if = None;
     while line_off < lines.len() {
         let indent = indent_level(&lines, line_off);
         if indent == lines[line_off].len() {
             line_off += 1;
             continue;
         }
+        if let Some(cp) = comment_prefix {
+            if lines[line_off][indent..].starts_with(cp) {
+                line_off += 1;
+                continue;
+            }
+        }
         let (test_name, val) = key_val(&lines, line_off, indent);
-        if test_name == "ignore" {
-            ignore = true;
+        if test_name == "ignore-if" {
+            ignore_if = Some(val.into());
             line_off += 1;
             continue;
         }
@@ -46,7 +52,8 @@ pub(crate) fn parse_tests(test_str: &str) -> Tests {
                     if sub_indent == indent {
                         break;
                     }
-                    let (end_line_off, key, val) = key_multiline_val(&lines, line_off, sub_indent);
+                    let (end_line_off, key, val) =
+                        key_multiline_val(comment_prefix, &lines, line_off, sub_indent);
                     line_off = end_line_off;
                     match key {
                         "env-var" => {
@@ -120,7 +127,7 @@ pub(crate) fn parse_tests(test_str: &str) -> Tests {
             }
         }
     }
-    Tests { ignore, tests }
+    Tests { ignore_if, tests }
 }
 
 fn indent_level(lines: &[&str], line_off: usize) -> usize {
@@ -160,6 +167,7 @@ fn key_val<'a>(lines: &[&'a str], line_off: usize, indent: usize) -> (&'a str, &
 /// Turn one more lines of the format `key: val` (where `val` may spread over many lines) into its
 /// separate components.
 fn key_multiline_val<'a>(
+    comment_prefix: Option<&str>,
     lines: &[&'a str],
     mut line_off: usize,
     indent: usize,
@@ -179,6 +187,12 @@ fn key_multiline_val<'a>(
             if cur_indent <= indent {
                 break;
             }
+            if let Some(cp) = comment_prefix {
+                if lines[line_off][sub_indent..].starts_with(cp) {
+                    line_off += 1;
+                    continue;
+                }
+            }
             val.push(&lines[line_off][sub_indent..]);
             line_off += 1;
         }
@@ -201,22 +215,31 @@ mod test {
 
     #[test]
     fn test_key_multiline() {
-        assert_eq!(key_multiline_val(&["x:", ""], 0, 0), (2, "x", vec![]));
+        assert_eq!(key_multiline_val(None, &["x:", ""], 0, 0), (2, "x", vec![]));
         assert_eq!(
-            key_multiline_val(&["x: y", "  z", "a"], 0, 0),
+            key_multiline_val(None, &["x: y", "  z", "a"], 0, 0),
             (2, "x", vec!["y", "z"])
         );
         assert_eq!(
-            key_multiline_val(&["x:", "  z", "a"], 0, 0),
+            key_multiline_val(None, &["x:", "  z", "a"], 0, 0),
             (2, "x", vec!["z"])
         );
         assert_eq!(
-            key_multiline_val(&["x:", "  z  ", "  a  ", "  ", "b"], 0, 0),
+            key_multiline_val(None, &["x:", "  z  ", "  a  ", "  ", "b"], 0, 0),
             (4, "x", vec!["z  ", "a  "])
         );
         assert_eq!(
-            key_multiline_val(&["x:", "  z  ", "    a  ", "  ", "  b"], 0, 0),
+            key_multiline_val(None, &["x:", "  z  ", "    a  ", "  ", "  b"], 0, 0),
             (5, "x", vec!["z  ", "  a  ", "", "b"])
         );
+        assert_eq!(
+            key_multiline_val(
+                Some("#"),
+                &["x:", "  z  ", "    a  ", "  # c2", "  ", "  b"],
+                0,
+                0
+            ),
+            (6, "x", vec!["z  ", "  a  ", "", "b"])
+        );
     }
 }

src/tester.rs

@@ -61,6 +61,7 @@ struct LangTesterPooler {
     test_threads: usize,
     ignored: bool,
     nocapture: bool,
+    comment_prefix: Option<String>,
     test_extract: Option<Box<dyn Fn(&Path) -> String + RefUnwindSafe + Send + Sync>>,
     fm_options: Option<
         Box<
@@ -94,6 +95,7 @@ impl LangTester {
                 test_dir: None,
                 ignored: false,
                 nocapture: false,
+                comment_prefix: None,
                 test_threads: num_cpus::get(),
                 fm_options: None,
                 test_extract: None,
@@ -128,6 +130,16 @@ impl LangTester {
         self
     }
 
+    /// If set, defines what lines will be treated as comments if, ignoring the current level of
+    /// indentation, they begin with `comment_prefix`.
+    ///
+    /// This option defaults to `None`.
+    pub fn comment_prefix<S: AsRef<str>>(&mut self, comment_prefix: S) -> &mut Self {
+        let inner = Arc::get_mut(&mut self.inner).unwrap();
+        inner.comment_prefix = Some(comment_prefix.as_ref().to_owned());
+        self
+    }
+
     /// If `test_file_filter` is specified, only files for which it returns `true` will be
     /// considered tests. A common use of this is to filter files based on filename extensions
     /// e.g.:
@@ -550,7 +562,7 @@ impl<'a> TestCmd<'a> {
 
 /// A collection of tests.
 pub(crate) struct Tests<'a> {
-    pub ignore: bool,
+    pub ignore_if: Option<String>,
     pub tests: HashMap<String, TestCmd<'a>>,
 }
 
@@ -643,8 +655,22 @@ fn test_file(
                         return;
                     }
 
-                    let tests = parse_tests(&test_str);
-                    if (inner.ignored && !tests.ignore) || (!inner.ignored && tests.ignore) {
+                    let tests = parse_tests(inner.comment_prefix.as_deref(), &test_str);
+                    let ignore = if let Some(ignore_if) = tests.ignore_if {
+                        Command::new(env::var("SHELL").unwrap_or_else(|_| "/bin/sh".to_owned()))
+                            .args(["-c", &ignore_if])
+                            .stdin(process::Stdio::piped())
+                            .stderr(process::Stdio::piped())
+                            .stdout(process::Stdio::piped())
+                            .status()
+                            .unwrap_or_else(|_| {
+                                fatal(&format!("Couldn't run ignore-if '{ignore_if}'"))
+                            })
+                            .success()
+                    } else {
+                        false
+                    };
+                    if (inner.ignored && !ignore) || (!inner.ignored && ignore) {
                         write_ignored(test_fname.as_str(), "", inner);
                         num_ignored.fetch_add(1, Ordering::Relaxed);
                         return;