EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
#759: #include / //@include / #includepath in scripts
vangel76e29f34029ce1
diff --git a/crates/script/src/include.rs b/crates/script/src/include.rsnew file mode 100644index 000000000..3c397f53c--- /dev/null+++ b/crates/script/src/include.rs@@ -0,0 +1,173 @@+//! ExtendScript's preprocessor directives, from the JavaScript Tools Guide's description of their+//! behaviour: `#include "file"` (and `//@include "file"`) inlines a file in place of the line,+//! resolved against the including file's folder, then the folders the latest `#includepath "a;b"`+//! (or `//@includepath`) named; `#target`, `#targetengine`, `#strict`, `#script` and `#engine`+//! are dropped. Each file is inlined once (which also breaks include cycles), nesting is limited+//! to [`MAX_DEPTH`], files and their total size are capped, and reads follow the scripts' file+//! gate ([`crate::scriptfile::check_read`]). Directives are recognised on lines of their own.+//!+//! Errors in the expanded code are mapped back to the file and line they came from.++use std::collections::HashSet;+use std::path::{Path, PathBuf};++use effectcraft_engine::Session;++use crate::runtime::ScriptError;+use crate::scriptfile::{self, MAX_DEPTH, MAX_FILE_BYTES};++/// The most text a script and the files it includes may add up to.+const MAX_TOTAL_BYTES: usize = 4 * MAX_FILE_BYTES;++/// A script with its includes inlined.+pub(crate) struct Expanded {+ pub(crate) code: String,+ /// Per line of `code`: the file (index into `files`) and 1-based line it came from.+ lines: Vec<(usize, u32)>,+ /// The script itself, then each included file.+ files: Vec<String>,+}++impl Expanded {+ /// Point an error in the expanded code at the file and line it came from.+ pub(crate) fn locate(&self, e: &mut ScriptError) {+ if self.files.first() != Some(&e.file) {+ return;+ }+ let Some(&(file, line)) = e.line.and_then(|l| l.checked_sub(1)).and_then(|l| self.lines.get(l as usize)) else { return };+ if let Some(name) = self.files.get(file) {+ e.file = name.clone();+ e.line = Some(line);+ }+ }+}++enum Directive<'a> {+ Include(&'a str),+ IncludePath(&'a str),+ Ignored,+}++/// The preprocessor directive on `line`, if it is one.+fn directive(line: &str) -> Option<Directive<'_>> {+ let t = line.trim();+ let (rest, comment) = match t.strip_prefix("//@") {+ Some(r) => (r, true),+ None => (t.strip_prefix('#')?, false),+ };+ let end = rest.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(rest.len());+ let (word, arg) = rest.split_at(end);+ if !(arg.is_empty() || arg.starts_with(char::is_whitespace) || arg.starts_with(['"', '\''])) {+ return None;+ }+ let arg = unquote(arg);+ match word {+ "include" => Some(Directive::Include(arg)),+ "includepath" => Some(Directive::IncludePath(arg)),+ // `//@target aftereffects` and friends are plain comments already.+ "target" | "targetengine" | "strict" | "script" | "engine" if !comment => Some(Directive::Ignored),+ _ => None,+ }+}++/// A directive's argument: `"x"`, `'x'` or `x`, with an optional trailing `;`.+fn unquote(arg: &str) -> &str {+ let a = arg.trim();+ let a = a.strip_suffix(';').map_or(a, str::trim_end);+ for q in ['"', '\''] {+ if let Some(inner) = a.strip_prefix(q).and_then(|r| r.split(q).next()) {+ return inner;+ }+ }+ a+}++struct Walk<'a> {+ s: &'a Session,+ /// The running script's folder: the file gate lets it load files below it.+ root: Option<PathBuf>,+ /// The latest `#includepath`.+ search: Vec<PathBuf>,+ seen: HashSet<PathBuf>,+ total: usize,+ out: Expanded,+}++impl Walk<'_> {+ fn push(&mut self, line: &str, file: usize, n: usize) {+ if !self.out.lines.is_empty() {+ self.out.code.push('\n');+ }+ self.out.code.push_str(line);+ self.out.lines.push((file, u32::try_from(n + 1).unwrap_or(u32::MAX)));+ }++ /// Inline `code` (file `file`, in folder `dir`), `depth` includes deep.+ fn walk(&mut self, code: &str, file: usize, dir: Option<&Path>, depth: usize) -> Result<(), ScriptError> {+ for (n, line) in code.split('\n').enumerate() {+ let err = |w: &Self, message: String| ScriptError {+ message,+ line: Some(u32::try_from(n + 1).unwrap_or(u32::MAX)),+ column: None,+ file: w.out.files.get(file).cloned().unwrap_or_default(),+ };+ match directive(line) {+ None => self.push(line, file, n),+ Some(Directive::Ignored) => self.push("", file, n),+ Some(Directive::IncludePath(paths)) => {+ self.search =+ paths.split(';').map(str::trim).filter(|p| !p.is_empty()).filter_map(|p| scriptfile::resolve(self.s, p, dir, &[]).ok()).collect();+ self.push("", file, n);+ }+ Some(Directive::Include(name)) => {+ if name.is_empty() {+ return Err(err(self, "#include needs a file name".into()));+ }+ let path = scriptfile::resolve(self.s, name, dir, &self.search).map_err(|e| err(self, format!("#include {e}")))?;+ if !self.seen.insert(path.clone()) {+ // Already inlined (or the file including it): once is enough.+ self.push("", file, n);+ continue;+ }+ if depth >= MAX_DEPTH {+ return Err(err(self, format!("#include \"{name}\": includes are nested too deeply (at most {MAX_DEPTH})")));+ }+ scriptfile::check_read(self.s, &path, self.root.as_deref()).map_err(|e| err(self, e))?;+ let text = scriptfile::read(self.s, &path).map_err(|e| err(self, e))?;+ self.total = self.total.saturating_add(text.len());+ if self.total > MAX_TOTAL_BYTES {+ return Err(err(self, format!("#include \"{name}\": the included files exceed {} MiB", MAX_TOTAL_BYTES >> 20)));+ }+ // The included lines take the directive's place.+ self.out.files.push(path.to_string_lossy().to_string());+ let idx = self.out.files.len().saturating_sub(1);+ self.walk(&text, idx, path.parent(), depth + 1)?;+ }+ }+ }+ Ok(())+ }+}++/// `code` (the script `name`) with its directives applied, or `None` when it has none.+pub(crate) fn expand(s: &Session, code: &str, name: &str) -> Result<Option<Expanded>, ScriptError> {+ if !code.split('\n').any(|l| directive(l).is_some()) {+ return Ok(None);+ }+ let root = scriptfile::script_dir(s, name);+ let mut w = Walk {+ s,+ root: root.clone(),+ search: vec![],+ seen: HashSet::new(),+ total: code.len(),+ out: Expanded { code: String::with_capacity(code.len()), lines: vec![], files: vec![if name.is_empty() { "script" } else { name }.to_string()] },+ };+ if root.is_some() {+ w.seen.insert(crate::runtime::normalize(name));+ }+ // A script without a folder (the Script Console, code run by name) looks in the project's.+ let dir = root.or_else(|| crate::runtime::project_dir(s).map(|d| crate::runtime::normalize(&d)));+ w.walk(code, 0, dir.as_deref(), 0)?;+ Ok(Some(w.out))+}diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rsindex d1c9568c9..bce2f9329 100644--- a/crates/script/src/lib.rs+++ b/crates/script/src/lib.rs@@ -53,9 +53,11 @@#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]#![recursion_limit = "256"]+mod include;pub mod matchnames;mod model;mod runtime;+mod scriptfile;mod socket;mod ui;@@ -85,6 +87,8 @@ mod tests;#[cfg(test)]mod tests_ease_presets;#[cfg(test)]+mod tests_include;+#[cfg(test)]mod tests_panels;#[cfg(test)]mod tests_ui;diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rsindex 712999ee0..e5986ea4d 100644--- a/crates/script/src/runtime.rs+++ b/crates/script/src/runtime.rs@@ -291,13 +291,13 @@ fn native_task(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult<JsVOk(JsValue::from(id))}-fn project_dir(s: &Session) -> Option<String> {+pub(crate) fn project_dir(s: &Session) -> Option<String> {let p = s.path.as_deref()?;std::path::Path::new(p).parent().map(|d| d.to_string_lossy().to_string())}/// Lexically normalized absolute path (`a/./b/../c` → `a/c`).-fn normalize(p: &str) -> std::path::PathBuf {+pub(crate) fn normalize(p: &str) -> std::path::PathBuf {use std::path::{Component, Path, PathBuf};let path = Path::new(p);let abs = if path.is_absolute() { path.to_path_buf() } else { std::env::current_dir().unwrap_or_default().join(path) };@@ -587,7 +587,23 @@ pub(crate) fn outcome(output: Vec<String>, result: Result<J, ScriptError>) -> Ou/// events reach the host through [`crate::dispatch_ui`]); palettes and panels keep their host/// alive for their handlers. The Script Console (and the web build) run inline, where dialogs/// don't block.+///+/// ExtendScript's `#include` / `#includepath` are inlined first, and errors point at the file+/// and line they came from ([`crate::include`]).pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome {+ let expanded = match crate::include::expand(session, req.code, req.name) {+ Ok(x) => x,+ Err(e) => return Outcome { error: Some(e), ..Default::default() },+ };+ let Some(x) = expanded else { return run_expanded(session, req) };+ let mut out = run_expanded(session, &ScriptRequest { code: &x.code, name: req.name, console: req.console });+ if let Some(e) = out.error.as_mut() {+ x.locate(e);+ }+ out+}++fn run_expanded(session: &mut Session, req: &ScriptRequest) -> Outcome {if let Err(message) = check_nesting(req.code) {return Outcome { error: Some(ScriptError { message, file: req.name.into(), ..Default::default() }), ..Default::default() };}diff --git a/crates/script/src/scriptfile.rs b/crates/script/src/scriptfile.rsnew file mode 100644index 000000000..2b6afcfbe--- /dev/null+++ b/crates/script/src/scriptfile.rs@@ -0,0 +1,65 @@+//! Script files a running script loads (`#include`, `$.evalFile`): where a name resolves, whether+//! the script may read it, and the limits that keep hostile, huge or cyclic files from running+//! away.++use std::path::{Path, PathBuf};++use effectcraft_engine::Session;++use crate::runtime::{GATE, normalize, project_dir};++/// How deep loaded files may nest (a loaded file loading another…).+pub(crate) const MAX_DEPTH: usize = 32;+/// The largest single file a script may load.+pub(crate) const MAX_FILE_BYTES: usize = 16 << 20;++/// The folder of the script file `name` runs from: `None` for the Script Console and for scripts+/// run by name or from code, which have no file.+pub(crate) fn script_dir(s: &Session, name: &str) -> Option<PathBuf> {+ let parent = Path::new(name).parent().filter(|p| !p.as_os_str().is_empty())?;+ s.services.exists(name).then(|| normalize(&parent.to_string_lossy()))+}++/// Resolve `name`: absolute paths as they are; relative ones against `dir` (the loading script's+/// folder), then each of `search`. The first existing file wins; when none exists, the first+/// candidate, so the read error names it.+pub(crate) fn resolve(s: &Session, name: &str, dir: Option<&Path>, search: &[PathBuf]) -> Result<PathBuf, String> {+ if Path::new(name).is_absolute() {+ return Ok(normalize(name));+ }+ let candidates: Vec<PathBuf> = dir.into_iter().chain(search.iter().map(PathBuf::as_path)).map(|d| normalize(&d.join(name).to_string_lossy())).collect();+ candidates+ .iter()+ .find(|c| s.services.exists(&c.to_string_lossy()))+ .or(candidates.first())+ .cloned()+ .ok_or_else(|| format!("cannot find \"{name}\": the script has no folder to look in (use an absolute path or #includepath)"))+}++/// The read gate for loaded script files: files under the running script's own folder (`root`) or+/// the project's folder; anywhere with the Allow Scripts to Write Files preference.+pub(crate) fn check_read(s: &Session, path: &Path, root: Option<&Path>) -> Result<(), String> {+ if s.prefs.scripting.allow_scripts_write_files {+ return Ok(());+ }+ let project = project_dir(s).map(|d| normalize(&d));+ if root.into_iter().chain(project.as_deref()).any(|d| path.starts_with(d)) {+ return Ok(());+ }+ Err(format!("Unable to read {}: scripts can only load files from their own or the project's folder unless {GATE} is on", path.display()))+}++/// Read a script file as text (a UTF-8 byte order mark dropped), refusing files over+/// [`MAX_FILE_BYTES`].+pub(crate) fn read(s: &Session, path: &Path) -> Result<String, String> {+ let too_big = || format!("Unable to read {}: script files are limited to {} MiB", path.display(), MAX_FILE_BYTES >> 20);+ if std::fs::metadata(path).is_ok_and(|m| m.len() > MAX_FILE_BYTES as u64) {+ return Err(too_big());+ }+ let bytes = s.services.read_file(&path.to_string_lossy()).map_err(|e| format!("Unable to read {}: {e}", path.display()))?;+ if bytes.len() > MAX_FILE_BYTES {+ return Err(too_big());+ }+ let text = String::from_utf8_lossy(&bytes);+ Ok(text.strip_prefix('\u{feff}').unwrap_or(&text).to_string())+}diff --git a/crates/script/src/tests_include.rs b/crates/script/src/tests_include.rsnew file mode 100644index 000000000..3c7d10241--- /dev/null+++ b/crates/script/src/tests_include.rs@@ -0,0 +1,94 @@+//! #759: ExtendScript's `#include` / `//@include` / `#includepath` directives.++use std::path::{Path, PathBuf};++use effectcraft_engine::Session;+use serde_json::json;++use crate::{Outcome, run_code};++fn session() -> Session {+ let mut s = Session::default();+ crate::install(&mut s);+ s+}++fn tmp(tag: &str) -> PathBuf {+ let dir = std::env::temp_dir().join(format!("ec-include-{tag}-{}", std::process::id()));+ let _ = std::fs::remove_dir_all(&dir);+ std::fs::create_dir_all(&dir).unwrap();+ dir+}++fn write(path: &Path, text: &str) {+ std::fs::create_dir_all(path.parent().unwrap()).unwrap();+ std::fs::write(path, text).unwrap();+}++/// Run the script file `main` (its code read from disk, as File ▸ Scripts ▸ Run Script File does).+fn run_file(s: &mut Session, main: &Path) -> Outcome {+ let code = std::fs::read_to_string(main).unwrap();+ run_code(s, &code, &main.to_string_lossy())+}++#[test]+fn includes_are_inlined_relative_to_the_script() {+ let dir = tmp("ok");+ write(&dir.join("lib/math.jsx"), "#include \"util.jsxinc\"\nfunction twice(x) { return 2 * x; }");+ write(&dir.join("lib/util.jsxinc"), "// includes its includer back: inlined once only\n#include \"math.jsx\"\nvar unit = 'px';");+ write(&dir.join("vendor/extra.jsx"), "var extra = 40;");+ write(+ &dir.join("main.jsx"),+ "#target aftereffects\n#targetengine \"main\"\n#strict on\n//@include 'lib/math.jsx'\n#includepath \"vendor;lib\"\n#include \"extra.jsx\";\n#include \"util.jsxinc\"\n[twice(1) + extra, unit, $.fileName.split('/').pop()]",+ );+ let mut s = session();+ let o = run_file(&mut s, &dir.join("main.jsx"));+ assert!(o.error.is_none(), "{:?}", o.error);+ assert_eq!(o.result, json!([42, "px", "main.jsx"]));+ let _ = std::fs::remove_dir_all(&dir);+}++#[test]+fn errors_point_at_the_file_and_line_they_came_from() {+ let dir = tmp("err");+ write(&dir.join("lib.jsx"), "var a = 1;\nvar b = 2;\nnotDefined();");+ write(&dir.join("main.jsx"), "// main\n#include \"lib.jsx\"\nvar c = 3;");+ let mut s = session();+ let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");+ assert!(e.file.ends_with("lib.jsx") && e.line == Some(3), "{e:?}");+ // After the include, lines in the script itself keep their numbers.+ write(&dir.join("lib.jsx"), "var a = 1;\nvar b = 2;");+ write(&dir.join("main.jsx"), "// main\n#include \"lib.jsx\"\nvar c = 3;\nalsoNotDefined();");+ let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");+ assert!(e.file.ends_with("main.jsx") && e.line == Some(4), "{e:?}");+ // A missing file is a clear error on the directive's line.+ write(&dir.join("main.jsx"), "var x = 1;\n#include \"missing.jsx\"");+ let e = run_file(&mut s, &dir.join("main.jsx")).error.expect("an error");+ assert!(e.message.contains("missing.jsx") && e.line == Some(2), "{e:?}");+ let _ = std::fs::remove_dir_all(&dir);+}++#[test]+fn hostile_includes_are_bounded_and_gated() {+ let dir = tmp("hostile");+ // A chain deeper than the limit is an error, not a stack overflow.+ for i in 0..100 {+ write(&dir.join(format!("chain/{i}.jsx")), &format!("#include \"{}.jsx\"", i + 1));+ }+ write(&dir.join("chain/100.jsx"), "1");+ let mut s = session();+ let e = run_file(&mut s, &dir.join("chain/0.jsx")).error.expect("an error");+ assert!(e.message.contains("nested too deeply"), "{e:?}");+ // Files outside the script's (and the project's) folder need the file preference.+ write(&dir.join("outside.jsx"), "var secret = 7;");+ write(&dir.join("app/main.jsx"), "#include \"../outside.jsx\"\nsecret");+ let e = run_file(&mut s, &dir.join("app/main.jsx")).error.expect("blocked");+ assert!(e.message.contains("Allow Scripts to Write Files"), "{e:?}");+ // Code run without a file can't include relative to anything but the project's folder.+ let e = run_code(&mut s, "#include \"outside.jsx\"", "console").error.expect("no folder");+ assert!(e.message.contains("outside.jsx"), "{e:?}");+ s.prefs.scripting.allow_scripts_write_files = true;+ let o = run_file(&mut s, &dir.join("app/main.jsx"));+ assert_eq!(o.result, json!(7), "{:?}", o.error);+ let _ = std::fs::remove_dir_all(&dir);+}diff --git a/docs/plugins.md b/docs/plugins.mdindex d24c078b8..8583d8a5c 100644--- a/docs/plugins.md+++ b/docs/plugins.md@@ -14,7 +14,7 @@ Guide):| Effect plug-ins (C/C++ SDK, `.aex` / `.plugin`) | [Effect plug-ins](#effect-plug-ins): WebAssembly modules (plug-in API v1) or Rust; the `Plug-ins` folder next to the settings loads at start-up | Different ABI: After Effects SDK plug-ins can't run (G8) || Scripts folder: its scripts are listed in File ▸ Scripts | `Scripts/` in the settings folder (File ▸ Scripts ▸ Install Script File… copies a script there), listed in File ▸ Scripts with the bundled samples | Same || File ▸ Scripts ▸ Run Script File… (`.jsx`, `.jsxbin`) | `.jsx` / `.js` | Gap: `.jsxbin` (an undocumented binary encoding) |-| ExtendScript preprocessor directives (`#include`, `#target`, `#targetengine`) | Not understood: a script that starts with one doesn't parse | Gap |+| ExtendScript preprocessor directives (`#include`, `#target`, `#targetengine`) | `#include` / `//@include` / `#includepath` inline files relative to the script (once each, depth-limited, file gate applies); `#target`, `#targetengine`, `#strict` are ignored | Same || `Scripts/Startup` and `Scripts/Shutdown`: run in alphabetical order when the app starts and quits | Not run | Gap || `Scripts/ScriptUI Panels`: listed at the bottom of the Window menu, dockable like the app's own panels, `this` is the panel | `Scripts/ScriptUI Panels/` in the settings folder (File ▸ Scripts ▸ Install ScriptUI Panel… copies one there), listed at the bottom of the Window menu, docked like any panel, `this` is the panel; the panels open when the app quits open again at the next launch | Same. Where a panel was docked isn't kept: workspace layouts aren't saved between launches || Keyboard shortcuts for scripts (the Keyboard Shortcuts editor lists the scripts of the Scripts folder) | Shortcuts bind to commands and menu entries; the File ▸ Scripts and Window menu entries of scripts aren't offered | Gap |diff --git a/docs/scripting-parity.md b/docs/scripting-parity.mdindex fef88efbc..a85be904a 100644--- a/docs/scripting-parity.md+++ b/docs/scripting-parity.md@@ -22,7 +22,7 @@ our own scripts; no real-world script library has been run against both apps.| ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics | Same, dockable panels reopen at launch | good | `ui_scriptui`, `tests_ui*` || Scripts menu, ScriptUI Panels folder | yes | yes | same | [plugins.md](plugins.md) || Startup / Shutdown folders | yes | not run | missing | plugins.md |-| `#include`, `#target` directives | yes | a script starting with one doesn't parse | missing | plugins.md; common in real scripts |+| `#include`, `#target` directives | yes | `#include` / `//@include` / `#includepath` inline files relative to the including script (once each, nested at most 32 deep, under the scripts' file gate; errors name the included file and line); `#target`, `#targetengine`, `#strict` are ignored | good | `tests_include.rs` (#759); installed scripts and panels have no folder of their own, so they include from the project's folder or absolute paths || `.jsxbin` | yes | no | out of scope | No public specification || Keyboard shortcuts for scripts | yes | no | missing | plugins.md || Command-line rendering `aerender` | yes | `effectcraft-cli render`, `script`, `exec` | same purpose | `apps/effectcraft-cli` |