Skip to content

Commit 5eac9e1

Browse files
alexcrichtonalexcrichton
authored
Support multi-line run/fail directives in CLI tests (#1782)
Also add a `readme.wat` test explainer as to what's going on.
1 parent b146040 commit 5eac9e1

6 files changed

Lines changed: 172 additions & 67 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ tempfile = "3.1"
178178
wast = { path = 'crates/wast' }
179179
pretty_assertions = { workspace = true }
180180
libtest-mimic = { workspace = true }
181+
indexmap = { workspace = true }
181182

182183
[[test]]
183184
name = "cli"

‎tests/cli.rs‎

Lines changed: 90 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,12 @@
11
//! A test suite to test the `wasm-tools` CLI itself.
22
//!
3-
//! This test suite will look for `*.wat` files in the `tests/cli/**` directory,
4-
//! recursively. Each wat file must have a directive of the form:
5-
//!
6-
//! ;; RUN: ...
7-
//!
8-
//! where `...` is a space-separate set of command to pass to the `wasm-tools`
9-
//! CLI. The `%` argument is replaced with the path to the current file. For
10-
//! example:
11-
//!
12-
//! ;; RUN: dump %
13-
//!
14-
//! would execute `wasm-tools dump the-current-file.wat`. The `cli` directory
15-
//! additionally contains `*.stdout` and `*.stderr` files to assert the output
16-
//! of the subcommand. Files are not present if the stdout/stderr are empty.
17-
//!
18-
//! This also supports a limited form of piping along the lines of:
19-
//!
20-
//! ;; RUN: strip % | objdump
21-
//!
22-
//! where a `|` will execute the first subcommand and pipe its stdout into the
23-
//! stdin of the next command.
24-
//!
25-
//! Use `BLESS=1` in the environment to auto-update expectation files. Be sure
26-
//! to look at the diff!
3+
//! This test suite will look for `*.wat` and `*.wit` files in the
4+
//! `tests/cli/**` directory, recursively. For more information about supported
5+
//! directives and features of this test suite see the `tests/cli/readme.wat`
6+
//! file which has an explanatory comment at the top for what's going on.
277
28-
use anyhow::{anyhow, bail, Context, Result};
8+
use anyhow::{bail, Context, Result};
9+
use indexmap::IndexMap;
2910
use libtest_mimic::{Arguments, Trial};
3011
use pretty_assertions::StrComparison;
3112
use std::env;
@@ -59,54 +40,96 @@ fn main() {
5940
libtest_mimic::run(&args, trials).exit();
6041
}
6142

62-
fn wasm_tools_exe() -> Command {
63-
Command::new(env!("CARGO_BIN_EXE_wasm-tools"))
64-
}
65-
6643
fn run_test(test: &Path, bless: bool) -> Result<()> {
6744
let contents = std::fs::read_to_string(test)?;
68-
let (line, should_fail) = contents
45+
46+
let mut directives = contents
6947
.lines()
70-
.filter_map(|l| {
71-
let run = l.strip_prefix(";; RUN: ").or(l.strip_prefix("// RUN: "));
72-
let fail = l.strip_prefix(";; FAIL: ").or(l.strip_prefix("// FAIL: "));
73-
run.map(|l| (l, false)).or(fail.map(|l| (l, true)))
74-
})
75-
.next()
76-
.ok_or_else(|| anyhow!("no line found with `;; RUN: ` directive"))?;
77-
78-
let mut cmd = wasm_tools_exe();
79-
let mut stdin = None;
80-
let tempdir = TempDir::new()?;
81-
for arg in line.split_whitespace() {
82-
let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
83-
if arg == "|" {
84-
let output = execute(&mut cmd, stdin.as_deref(), false)?;
85-
stdin = Some(output.stdout);
86-
cmd = wasm_tools_exe();
87-
} else if arg == "%" {
88-
cmd.arg(test);
89-
} else {
90-
cmd.arg(arg);
48+
.enumerate()
49+
.filter(|(_, l)| !l.is_empty())
50+
.filter_map(|(i, l)| {
51+
l.strip_prefix("// ")
52+
.or(l.strip_prefix(";; "))
53+
.map(|l| (i + 1, l))
54+
});
55+
56+
let mut commands = IndexMap::new();
57+
58+
while let Some((i, line)) = directives.next() {
59+
let run = line.strip_prefix("RUN");
60+
let fail = line.strip_prefix("FAIL");
61+
let (directive, should_fail) = match run.map(|l| (l, false)).or(fail.map(|l| (l, true))) {
62+
Some(pair) => pair,
63+
None => continue,
64+
};
65+
let (cmd, name) = match directive.strip_prefix("[") {
66+
Some(prefix) => match prefix.find("]:") {
67+
Some(i) => (&prefix[i + 2..], &prefix[..i]),
68+
None => bail!("line {i}: failed to find `]:` after `[`"),
69+
},
70+
None => match directive.strip_prefix(":") {
71+
Some(cmd) => (cmd, ""),
72+
None => bail!("line {i}: failed to find `:` after `RUN` or `FAIL`"),
73+
},
74+
};
75+
let mut cmd = cmd.to_string();
76+
while cmd.ends_with("\\") {
77+
cmd.pop();
78+
match directives.next() {
79+
Some((_, line)) => cmd.push_str(line),
80+
None => bail!("line {i}: directive ends in `\\` but nothing on next line"),
81+
}
82+
}
83+
84+
match commands.insert(name, (cmd, should_fail)) {
85+
Some(_) => bail!("line {i}: duplicate directive named {name:?}"),
86+
None => {}
9187
}
9288
}
9389

94-
let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
95-
let extension = test.extension().unwrap().to_str().unwrap();
96-
assert_output(
97-
bless,
98-
&output.stdout,
99-
&test.with_extension(&format!("{extension}.stdout")),
100-
&tempdir,
101-
)
102-
.context("failed to check stdout expectation (auto-update with BLESS=1)")?;
103-
assert_output(
104-
bless,
105-
&output.stderr,
106-
&test.with_extension(&format!("{extension}.stderr")),
107-
&tempdir,
108-
)
109-
.context("failed to check stderr expectation (auto-update with BLESS=1)")?;
90+
if commands.is_empty() {
91+
bail!("failed to find `// RUN: ...` or `// FAIL: ...` at the top of this file");
92+
}
93+
let exe = Path::new(env!("CARGO_BIN_EXE_wasm-tools"));
94+
let tempdir = TempDir::new_in(exe.parent().unwrap())?;
95+
for (name, (line, should_fail)) in commands {
96+
let mut cmd = Command::new(exe);
97+
let mut stdin = None;
98+
for arg in line.split_whitespace() {
99+
let arg = arg.replace("%tmpdir", tempdir.path().to_str().unwrap());
100+
if arg == "|" {
101+
let output = execute(&mut cmd, stdin.as_deref(), false)?;
102+
stdin = Some(output.stdout);
103+
cmd = Command::new(exe);
104+
} else if arg == "%" {
105+
cmd.arg(test);
106+
} else {
107+
cmd.arg(arg);
108+
}
109+
}
110+
111+
let output = execute(&mut cmd, stdin.as_deref(), should_fail)?;
112+
let extension = test.extension().unwrap().to_str().unwrap();
113+
let extension = if name.is_empty() {
114+
extension.to_string()
115+
} else {
116+
format!("{extension}.{name}")
117+
};
118+
assert_output(
119+
bless,
120+
&output.stdout,
121+
&test.with_extension(&format!("{extension}.stdout")),
122+
&tempdir,
123+
)
124+
.context("failed to check stdout expectation (auto-update with BLESS=1)")?;
125+
assert_output(
126+
bless,
127+
&output.stderr,
128+
&test.with_extension(&format!("{extension}.stderr")),
129+
&tempdir,
130+
)
131+
.context("failed to check stderr expectation (auto-update with BLESS=1)")?;
132+
}
110133
Ok(())
111134
}
112135

‎tests/cli/readme.wat‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
;; This is intended to be a self-documenting test which explains what's
2+
;; possible in directives for this test suite in `tests/cli/*`. The purpose of
3+
;; this test suite is to make it as easy as dropping a file in this directory to
4+
;; test the `wasm-tools` CLI tool and its subcommands. The test file itself is
5+
;; generally the input to the test and what's being tested will be present in
6+
;; comments at the top of the file with directives.
7+
;;
8+
;; All test directives must come in comments at the start of the file:
9+
;;
10+
;; RUN: validate %
11+
;;
12+
;; The `RUN` prefix indicates that the specified `wasm-tools` subcommand should
13+
;; be executed. It's possible to have more than one test in a file by having
14+
;; named directives such as:
15+
;;
16+
;; RUN[validate-again]: validate %
17+
;;
18+
;; Directive names must be unique, so using `validate-again` would not be valid.
19+
;; Additionally you can't use an unprefixed directive more than once so using
20+
;; `RUN: ...` here again would not be allowed for example.
21+
;;
22+
;; You can also use the `FAIL` directive to indicate that the subcommand should
23+
;; fail rather than succeed.
24+
;;
25+
;; FAIL[should-fail]: validate % --features=-simd
26+
;;
27+
;; As you can see directives can have comments around them. Directives are
28+
;; identified as comment lines starting with `RUN` or `FAIL`.
29+
;;
30+
;; Within directives there are a few feature. First as you've seen the `%` value
31+
;; will be substituted with the current filename which means:
32+
;;
33+
;; RUN[subst]: validate %
34+
;;
35+
;; means to run `wasm-tools validate tests/cli/readme.wat` and test the result
36+
;; is successful.
37+
;;
38+
;; You can additionally use `|` to pipe commands together by feeding the stdout
39+
;; of the previous command into the stdin of the next command.
40+
;;
41+
;; RUN[pipe]: print % | validate
42+
;;
43+
;; Note that when piping commands the intermediate commands before the final
44+
;; one, in this case `print` being the intermediate, must all succeed.
45+
;;
46+
;; Tests also assert the stdout/stderr of the command being tested. For example
47+
;; if printing is tested:
48+
;;
49+
;; RUN[print]: print %
50+
;;
51+
;; then this tests that `tests/cli/readme.wat.print.stdout` is the result of
52+
;; `wasm-tools print tests/cli/readme.wat`. Note that this can be tedious to
53+
;; update so you can use the environment variable `BLESS=1` to automatically
54+
;; update all test assertions. This can then be reviewed after the test is
55+
;; passing for accuracy.
56+
;;
57+
;; Each test additionally can have a temporary directory available to it which
58+
;; is accessible with the `%tmpdir` substitution. For example:
59+
;;
60+
;; RUN[tmpdir]: print % -o %tmpdir/foo.wat | validate %tmpdir/foo.wat
61+
;;
62+
;; Note that temporary directories are persisted across tests in the same file,
63+
;; but different files all get different temporary directories.
64+
;;
65+
;; You can also split commands across multiple lines:
66+
;;
67+
;; RUN[multiline]: print % | \
68+
;; validate
69+
;;
70+
;; here the `\` character is deleted and the next line is concatenated.
71+
72+
73+
;; this is the contents of the test, mostly empty in this case.
74+
(module
75+
(type (func (result v128)))
76+
)

‎tests/cli/readme.wat.print.stdout‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
(module
2+
(type (;0;) (func (result v128)))
3+
)

‎tests/cli/readme.wat.should-fail.stderr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
error: SIMD support is not enabled (at offset 0xb)

0 commit comments

Comments
 (0)