EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
CLI: run --commands reads {"command", "params"} steps from a JSON or JSON Lines file, or standard input (#2251)
Marco Arnulfo2c683a1ae10a
diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rsindex 6e5b8d6cc..08c75828e 100644--- a/apps/effectcraft-cli/src/main.rs+++ b/apps/effectcraft-cli/src/main.rs@@ -7,6 +7,7 @@//! effectcraft-cli exec --list [--filter TEXT] [--schemas] list engine commands (as `commands`)//! effectcraft-cli run <id> <json> [<id> <json> …] run several commands in order//! effectcraft-cli run --args-file F the same, one argument per line of F+//! effectcraft-cli run --commands FILE|- steps {"command", "params"} from FILE or stdin//! effectcraft-cli props <comp> <layer> [--flat] [--time S] a layer's property tree (with paths)//! effectcraft-cli get <comp> <layer> <path> [--time S] read a property//! effectcraft-cli set <comp> <layer> <path> <value> [--time S] [--expression E]@@ -98,6 +99,10 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|seexec <command-id> [--params JSON] run one engine command (exec --list: list them, as `commands`)run <id> <json> [<id> <json> ...] run several commands in orderrun --args-file F the same, one argument per line of F (no command-line length limit)+ run --commands FILE|- steps from FILE, or standard input for -: a JSON array of+ {\"command\": ID, \"params\": {...}} (or \"id\" for \"command\"), or one+ such object per line (JSON Lines); at most 64 MiB and 100000 steps.+ Repeatable; run after the <id> <json> pairs and --args-fileprops <comp> <layer> [--flat] [--time S] a layer's property tree with pathsget <comp> <layer> <path> [--time S] read a propertyset <comp> <layer> <path> <value> [--time S] [--expression E]@@ -151,6 +156,7 @@ const VALUED: &[&str] = &["--depth","--eval","--args-file",+ "--commands",// render"--template","--values",@@ -334,6 +340,87 @@ impl From<String> for Failure {}}+/// Most bytes `run --commands` reads from a file or standard input: the steps are untrusted input.+const MAX_COMMANDS_BYTES: u64 = 64 * 1024 * 1024;+/// Most steps one `run --commands` takes.+const MAX_COMMANDS_STEPS: usize = 100_000;++/// The text of `--commands source`: the file `source`, or standard input for `-`, refused past+/// [`MAX_COMMANDS_BYTES`]. This reader, [`parse_commands`] and [`command_step`] are ported from+/// vectorcraft-cli's (storytold/vectorcraft#3630), with a tuple in place of its `Step` enum.+fn read_commands(source: &str) -> Result<String, String> {+ use std::io::Read;+ let mut bytes = vec![];+ let read = if source == "-" {+ std::io::stdin().lock().take(MAX_COMMANDS_BYTES + 1).read_to_end(&mut bytes)+ } else {+ std::fs::File::open(source).and_then(|f| f.take(MAX_COMMANDS_BYTES + 1).read_to_end(&mut bytes))+ };+ read.map_err(|e| format!("--commands {source}: {e}"))?;+ if bytes.len() as u64 > MAX_COMMANDS_BYTES {+ return Err(format!("--commands {source}: more than {} MiB of steps", MAX_COMMANDS_BYTES / (1024 * 1024)));+ }+ String::from_utf8(bytes).map_err(|_| format!("--commands {source}: not UTF-8 text"))+}++/// The steps of `--commands source` from its `text`: a JSON array of `{"command": ID, "params":+/// {…}}` objects (`"id"` may stand for `"command"`), or one such object per line (JSON Lines, blank+/// lines skipped). `first` is the run's index of the first of them, which an error names with the+/// entry's place in the text.+fn parse_commands(source: &str, text: &str, first: usize) -> Result<Vec<(String, Value)>, String> {+ // Notepad on Windows starts UTF-8 files with a byte order mark.+ let text = text.strip_prefix('\u{feff}').unwrap_or(text);+ let too_many = || format!("--commands {source}: more than {MAX_COMMANDS_STEPS} steps");+ // Each entry with where it is in the text: its line for JSON Lines.+ let mut entries: Vec<(Option<usize>, Value)> = vec![];+ if text.trim_start().starts_with('[') {+ let v: Value = serde_json::from_str(text).map_err(|e| format!("--commands {source}: {e}"))?;+ let Value::Array(items) = v else { return Err(format!("--commands {source}: not a JSON array")) };+ if items.len() > MAX_COMMANDS_STEPS {+ return Err(too_many());+ }+ entries.extend(items.into_iter().map(|v| (None, v)));+ } else {+ for (n, line) in text.lines().enumerate().filter(|(_, l)| !l.trim().is_empty()) {+ if entries.len() == MAX_COMMANDS_STEPS {+ return Err(too_many());+ }+ let v = serde_json::from_str(line).map_err(|e| format!("--commands {source}: step {} (line {}): {e}", first + entries.len(), n + 1))?;+ entries.push((Some(n + 1), v));+ }+ }+ entries+ .into_iter()+ .enumerate()+ .map(|(k, (line, v))| {+ command_step(v).map_err(|e| {+ let place = line.map_or_else(|| format!("entry {}", k + 1), |l| format!("line {l}"));+ format!("--commands {source}: step {} ({place}): {e}", first + k)+ })+ })+ .collect()+}++/// The command id and params of one `--commands` entry, `{"command": ID, "params": {…}}`.+fn command_step(v: Value) -> Result<(String, Value), String> {+ let Value::Object(mut o) = v else { return Err("a step must be an object such as {\"command\": \"comp.new\", \"params\": {}}".into()) };+ let id = match (o.remove("command"), o.remove("id")) {+ (Some(Value::String(id)), None) | (None, Some(Value::String(id))) => id,+ (Some(_), Some(_)) => return Err("give `command` or `id`, not both".into()),+ (None, None) => return Err("no `command`".into()),+ _ => return Err("`command` must be a string, a command id".into()),+ };+ let params = match o.remove("params") {+ None | Some(Value::Null) => json!({}),+ Some(p @ Value::Object(_)) => p,+ Some(_) => return Err(format!("`params` of {id} must be a JSON object")),+ };+ if let Some(key) = o.keys().next() {+ return Err(format!("unknown key `{key}`: a step has `command` (or `id`) and `params`"));+ }+ Ok((id, params))+}+fn usage_err<T>(m: &str) -> Result<T, Failure> {Err(Failure::Usage(m.to_string()))}@@ -524,14 +611,14 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> {let text = std::fs::read_to_string(f).map_err(|e| Failure::Usage(format!("--args-file {f}: {e}")))?;pairs.extend(text.lines().map(str::trim).filter(|l| !l.is_empty()).map(String::from));}- if pairs.is_empty() {- return usage_err("run needs <command-id> [<json>] pairs");+ let sources = args.opts("--commands");+ if pairs.is_empty() && sources.is_empty() {+ return usage_err("run needs <command-id> [<json>] pairs or --commands FILE|-");}- let mut b = backend(args, true)?;- let mut results = vec![];+ let mut steps = vec![];let mut i = 0;while i < pairs.len() {- let id = &pairs[i];+ let id = pairs[i].clone();let params = match pairs.get(i + 1).filter(|p| p.trim_start().starts_with('{')) {Some(p) => {i += 1;@@ -540,9 +627,26 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> {None => json!({}),};i += 1;- let r = b.exec(id, params).map_err(|e| Failure::Error(format!("{id}: {e}")))?;- if let Some(e) = failed_render(id, &r) {- return Err(Failure::Error(format!("{id}: {e}")));+ steps.push((id, params));+ }+ if sources.iter().filter(|s| **s == "-").count() > 1 {+ return usage_err("--commands - is given twice: standard input is read once");+ }+ for &source in &sources {+ let text = read_commands(source).map_err(Failure::Usage)?;+ steps.extend(parse_commands(source, &text, steps.len()).map_err(Failure::Usage)?);+ }+ if steps.is_empty() {+ return Err(Failure::Usage(format!("--commands {}: no commands", sources.join(", "))));+ }+ let mut b = backend(args, true)?;+ let mut results = vec![];+ // The step's index closes the message: a generated list repeats ids, and callers match+ // the leading `<id>: `.+ for (n, (id, params)) in steps.into_iter().enumerate() {+ let r = b.exec(&id, params).map_err(|e| Failure::Error(format!("{id}: {e} (step {n})")))?;+ if let Some(e) = failed_render(&id, &r) {+ return Err(Failure::Error(format!("{id}: {e} (step {n})")));}if !json_out {emit(&r, false);diff --git a/apps/effectcraft-cli/tests/it/cli.rs b/apps/effectcraft-cli/tests/it/cli.rsindex 538887d9b..f48673ab8 100644--- a/apps/effectcraft-cli/tests/it/cli.rs+++ b/apps/effectcraft-cli/tests/it/cli.rs@@ -133,6 +133,78 @@ fn run_reads_arguments_from_a_file() {assert!(String::from_utf8_lossy(&out.stderr).contains("--args-file no/such/file.txt"));}+/// Run with `stdin` piped in; returns (exit code, stdout, stderr).+fn run_with_stdin(args: &[&str], stdin: &str) -> (i32, String, String) {+ let mut child = bin().args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();+ child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap();+ let out = child.wait_with_output().unwrap();+ (out.status.code().unwrap_or(-1), String::from_utf8_lossy(&out.stdout).into_owned(), String::from_utf8_lossy(&out.stderr).into_owned())+}++/// `run --commands FILE|-` (#2251) reads `{"command", "params"}` steps from a JSON array or JSON+/// Lines, from a file or standard input, and runs them as the same positional pairs would: after+/// the pairs, so a generated list can follow a hand-written prefix. One command whose params are+/// over Linux's 128 KiB per-argument limit goes through.+#[test]+fn run_reads_commands_from_a_file_or_stdin() {+ let steps = [+ ("comp.new", json!({"name": "A", "width": 64, "height": 64})),+ ("layer.newSolid", json!({"width": 8, "height": 8, "name": "One"})),+ ("layer.newSolid", json!({"width": 8, "height": 8, "name": "Two"})),+ ("project.summary", json!({})),+ ];+ let pairs: Vec<String> = steps.iter().flat_map(|(id, p)| [id.to_string(), p.to_string()]).collect();+ let pairs: Vec<&str> = pairs.iter().map(String::as_str).collect();+ let expected = ok_json(&[&["run", "--empty"][..], &pairs[..]].concat());+ let layers = |v: &Value| v[3]["result"]["items"][0]["layers"].as_u64();+ assert_eq!(layers(&expected), Some(2), "{expected}");++ let objects: Vec<Value> = steps.iter().map(|(id, p)| json!({"command": id, "params": p})).collect();+ let array = tmp("commands.json");+ std::fs::write(&array, serde_json::to_string_pretty(&objects).unwrap()).unwrap();+ assert_eq!(ok_json(&["run", "--commands", array.to_str().unwrap(), "--empty"]), expected);++ let lines: String = objects.iter().map(|o| format!("{o}\n\n")).collect();+ let (code, out, err) = run_with_stdin(&["run", "--commands", "-", "--empty", "--json"], &lines);+ assert_eq!(code, 0, "{err}");+ assert_eq!(serde_json::from_str::<Value>(&out).unwrap(), expected);++ // Positional pairs first, then the file.+ let rest = tmp("commands-rest.jsonl");+ std::fs::write(&rest, objects[1..].iter().map(|o| format!("{o}\n")).collect::<String>()).unwrap();+ assert_eq!(ok_json(&[&["run", "--empty", "--commands", rest.to_str().unwrap()][..], &pairs[..2]].concat()), expected);++ let name = "n".repeat(140 * 1024);+ let big = tmp("commands-big.jsonl");+ std::fs::write(&big, format!("{}\n{}\n", json!({"command": "comp.new", "params": {"name": name}}), json!({"command": "project.summary"}))).unwrap();+ let v = ok_json(&["run", "--commands", big.to_str().unwrap(), "--empty"]);+ assert_eq!(v[1]["result"]["items"][0]["name"].as_str().map(str::len), Some(name.len()));++ for (input, args, error) in [+ ("", &["run", "--commands", "-", "--empty"][..], "--commands -: no commands"),+ (+ "{\"command\":\"project.summary\"}\n{\"command\":\"x\",\"param\":{}}\n",+ &["run", "project.summary", "{}", "--commands", "-", "--empty"],+ "step 2 (line 2): unknown key `param`",+ ),+ ("[{\"command\":\"project.summary\"}, {\"params\":{}}]", &["run", "--commands", "-", "--empty"], "step 1 (entry 2): no `command`"),+ ("{}", &["run", "--commands", "-", "--commands", "-", "--empty"], "standard input is read once"),+ ] {+ let (code, out, err) = run_with_stdin(args, input);+ assert_eq!(code, 2, "{input:?}: {err}");+ assert!(err.contains(error), "{input:?}: {err}");+ assert!(out.is_empty(), "{input:?}: nothing ran");+ }+ // An empty list after positional pairs is no error: there is still something to run.+ let (code, _, err) = run_with_stdin(&["run", "project.summary", "--commands", "-", "--empty"], "");+ assert_eq!(code, 0, "{err}");+ // A step that fails when it runs is named by its index too.+ let (code, out, err) = run_with_stdin(&["run", "--commands", "-", "--empty", "--json"], "{\"command\":\"project.summary\"}\n{\"command\":\"no.such\"}\n");+ assert_eq!(code, 1, "{err}");+ let e = serde_json::from_str::<Value>(&out).unwrap()["error"].as_str().map(String::from).unwrap_or_default();+ assert!(e.starts_with("no.such: ") && e.ends_with(" (step 1)"), "{e}");+}+#[test]fn render_frame_writes_png() {let out = tmp("f.png");diff --git a/docs/agents.md b/docs/agents.mdindex 5c58d24b7..02b5253cf 100644--- a/docs/agents.md+++ b/docs/agents.md@@ -677,6 +677,19 @@ effectcraft-cli exec file.importLottie '{"path":"anim.json"}' main.ecproj --saveeffectcraft-cli render main.ecproj --comp Main --out main.mov --format prores --progress # progress lines on stderr for a program that reads them```+A long `run` doesn't fit on a command line (Windows caps it at 32 767 characters, Linux one+argument at 128 KiB). `run --args-file F` reads the same `<id> <json>` arguments one per line;+`run --commands FILE` reads steps as a JSON array of `{"command": ID, "params": {…}}` objects, or one+such object per line (JSON Lines), and `--commands -` reads them from standard input. `--commands`+may be repeated; its steps run after the positional pairs and `--args-file`. A malformed step is a+usage error naming its index (from 0, as in the results) before anything runs, and the error of a step+that fails ends with `(step N)`:++```sh+effectcraft-cli run --commands steps.json --empty --save-as main.ecproj+generate-steps | effectcraft-cli run main.ecproj --commands - --save --json+```+Scripts written against After Effects' documented scripting API run with `script`:```sh