EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust
Key browser preview caches on a file generation, and hash bytes only when that key misses.
Vikram Bhaskarand3fd2c58d7a3
diff --git a/apps/effectcraft-web/src/app.rs b/apps/effectcraft-web/src/app.rsindex 1a6952408..68fd1a40c 100644--- a/apps/effectcraft-web/src/app.rs+++ b/apps/effectcraft-web/src/app.rs@@ -257,6 +257,7 @@ pub async fn start(canvas_id: String) -> Result<(), JsValue> {if n > 0 {app.frames.set_remote(Some(Arc::new(frames::WebFrames::start(n, !query_flag("nogpuworkers")))));}+ app.frames.set_virtual_media(files::STORE.media_epoch(), Arc::new(|path| files::STORE.file_identity(path)));let (tx, rx) = std::sync::mpsc::channel();api::set_sender(tx);app = app.with_control(rx);diff --git a/apps/effectcraft-web/src/frames.rs b/apps/effectcraft-web/src/frames.rsindex 523fd03c8..c7f350526 100644--- a/apps/effectcraft-web/src/frames.rs+++ b/apps/effectcraft-web/src/frames.rs@@ -416,6 +416,7 @@ fn post_reply(r: &FrameReply, px: Option<&[u8]>) {fn new_server() -> FrameServer {let mut s = FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions)));s.set_layer_store(Some(Arc::new(effectcraft_engine::render::PrefetchStore::default())));+ s.set_virtual_id(Some(Arc::new(|path| crate::files::STORE.file_identity(path))));s}diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rsindex b3a858def..a44a2eac7 100644--- a/apps/effectcraft-web/src/store.rs+++ b/apps/effectcraft-web/src/store.rs@@ -20,6 +20,7 @@use std::collections::BTreeMap;use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicU64, Ordering};use std::sync::{Arc, Mutex, OnceLock};use effectcraft_engine::config::{ConfigStore, FileOps};@@ -45,6 +46,8 @@ pub struct Entry {/// Which write this is: new with every write of the key, so a file replaced by another of/// the same size still reads as changed (workers are sent it again, #218).pub version: u64,+ /// FNV-1a of `data`, filled the first time a disk-cache key asks for it.+ hash: Option<u64>,}/// A queued write for the browser storage: `data` = `None` deletes the key.@@ -69,13 +72,13 @@ impl Mirror {/// An entry read from the browser's storage at startup (nothing to flush).pub fn load(&mut self, key: &str, data: Arc<[u8]>, modified: f64) {let version = self.next_version();- self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version });+ self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version, hash: None });}pub fn put(&mut self, key: &str, data: Arc<[u8]>, modified: f64, persist: bool) {let was_persisted = self.entries.get(key).is_some_and(|e| e.persist);let version = self.next_version();- self.entries.insert(key.to_string(), Entry { data, modified, persist, version });+ self.entries.insert(key.to_string(), Entry { data, modified, persist, version, hash: None });if persist || was_persisted {self.touch(key);}@@ -149,14 +152,31 @@ pub fn key_path(key: &str) -> Option<&str> {key.strip_prefix(FILES).filter(|p| p.starts_with('/'))}+/// FNV-1a 64. Computed when a disk-cache key first needs the file, not on every write.+fn content_hash(data: &[u8]) -> u64 {+ let mut h: u64 = 0xcbf2_9ce4_8422_2325;+ for &b in data {+ h ^= u64::from(b);+ h = h.wrapping_mul(0x0000_0100_0000_01b3);+ }+ h+}+/// The persistent store: a [`Mirror`] and a hook that wakes the flusher after changes.-#[derive(Default)]pub struct Store {pub mirror: Mutex<Mirror>,/// Called after every change that needs flushing (the web host schedules a flush).pub on_change: OnceLock<fn()>,/// Clock in milliseconds since the Unix epoch (tests pin it).pub clock: OnceLock<fn() -> f64>,+ /// Bumped when a virtual file is written. Preview cache lookups key on it; they do not hash.+ media_epoch: Arc<AtomicU64>,+}++impl Default for Store {+ fn default() -> Self {+ Self { mirror: Mutex::default(), on_change: OnceLock::new(), clock: OnceLock::new(), media_epoch: Arc::new(AtomicU64::new(0)) }+ }}impl Store {@@ -179,12 +199,40 @@ impl Store {/// Store a file under `path` (persisted unless `persist` is false: render outputs).pub fn put_file(&self, path: &str, data: Arc<[u8]>, persist: bool) {let now = self.now();- self.lock().put(&file_key(path), data, now, persist);+ let key = file_key(path);+ let replaced = self.lock().get(&key).is_none_or(|e| !Arc::ptr_eq(&e.data, &data));+ self.lock().put(&key, data, now, persist);+ if replaced {+ self.media_epoch.fetch_add(1, Ordering::Relaxed);+ }if persist {self.changed();}}+ /// The generation preview cache lookups key on. It changes when [`Self::put_file`] writes+ /// a different blob, and it is not a content hash.+ pub fn media_epoch(&self) -> Arc<AtomicU64> {+ self.media_epoch.clone()+ }++ /// Size and content hash of the file under `path`. The hash is stored on the entry so a+ /// later key does not walk the bytes again.+ pub fn file_identity(&self, path: &str) -> Option<(u64, u64)> {+ let key = file_key(path);+ let mut mirror = self.lock();+ let entry = mirror.entries.get(&key)?;+ if let Some(hash) = entry.hash {+ return Some((entry.data.len() as u64, hash));+ }+ let hash = content_hash(&entry.data);+ let len = entry.data.len() as u64;+ if let Some(entry) = mirror.entries.get_mut(&key) {+ entry.hash = Some(hash);+ }+ Some((len, hash))+ }+pub fn file(&self, path: &str) -> Option<Arc<[u8]>> {self.lock().get(&file_key(path)).map(|e| e.data.clone())}@@ -446,6 +494,11 @@ mod tests {assert_ne!(a, b);assert_eq!(&data[..], b"blue");assert_eq!(s.file_versioned("/collision.png").map(|f| f.1), Some(b), "reading doesn't change it");+ let epoch = s.media_epoch().load(Ordering::Relaxed);+ let id = s.file_identity("/collision.png").expect("identity");+ assert_eq!(id, (4, content_hash(b"blue")));+ assert_eq!(s.file_identity("/collision.png"), Some(id), "the hash is kept on the entry");+ assert!(epoch >= 2, "each new blob bumps the generation the preview keys on");}/// A write that failed is flushed again with the key's latest state (#215).diff --git a/crates/engine/src/remote.rs b/crates/engine/src/remote.rsindex 207367b07..7d592abe2 100644--- a/crates/engine/src/remote.rs+++ b/crates/engine/src/remote.rs@@ -263,6 +263,8 @@ pub struct FrameServer {/// The disk cache's layer buffers (see the module docs) and the project's salt.layer_store: Option<Arc<PrefetchStore>>,salt: u64,+ /// Size and content hash for footage paths with no native metadata (the browser file table).+ virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,}/// A frame whose readbacks are in flight.@@ -294,9 +296,16 @@ impl FrameServer {pending: None,layer_store: None,salt: 0,+ virtual_id: None,}}+ /// Identity of virtual footage files, used when native metadata cannot describe the path.+ /// Called from [`FrameServer::load`], not from the UI thread's cache lookup.+ pub fn set_virtual_id(&mut self, id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>) {+ self.virtual_id = id;+ }+/// The store behind the layer cache when a request asks for `layers` (the browser: the/// disk cache's layer buffers, fetched before the request).pub fn set_layer_store(&mut self, store: Option<Arc<PrefetchStore>>) {@@ -338,7 +347,7 @@ impl FrameServer {fn load(&mut self, revision: u64) -> Result<(), String> {let p = Project::from_json(&self.doc.to_string()).map_err(|e| e.to_string())?;- self.salt = effectcraft_render::disk_cache::footage_salt(&p);+ self.salt = effectcraft_render::disk_cache::footage_salt_with(&p, self.virtual_id.as_deref());self.cache.set_store_salt(self.salt);self.project = Some(Arc::new(p));self.revision = Some(revision);diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rsindex 4140b7923..f867a614a 100644--- a/crates/render/src/disk_cache.rs+++ b/crates/render/src/disk_cache.rs@@ -779,8 +779,12 @@ fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) {h.write(&[0xff]);}-/// A file's size and modification time (so edited footage gets new keys).-fn file_stamp(h: &mut Hash128, path: &str) {+/// Size and content hash of a path that has no native metadata (the browser file table).+pub type VirtualFileId<'a> = &'a (dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync);++/// A file's size and modification time (so edited footage gets new keys). When the path has no+/// native metadata, `virtual_id` supplies a durable size and content hash instead.+fn file_stamp(h: &mut Hash128, path: &str, virtual_id: Option<VirtualFileId<'_>>) {if path.is_empty() {return;}@@ -791,26 +795,36 @@ fn file_stamp(h: &mut Hash128, path: &str) {{h.write_u64(d.as_nanos() as u64);}+ return;+ }+ if let Some((len, hash)) = virtual_id.and_then(|id| id(path)) {+ h.write_u64(len);+ h.write_u64(hash);}}-fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) {- file_stamp(h, &f.path);+fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage, virtual_id: Option<VirtualFileId<'_>>) {+ file_stamp(h, &f.path, virtual_id);for p in &f.sequence {- file_stamp(h, p);+ file_stamp(h, p, virtual_id);}}/// Salt for layer cache keys on disk: the renderer version and every footage file's identity/// (layer keys name footage items, not file contents).pub fn footage_salt(project: &effectcraft_project::Project) -> u64 {+ footage_salt_with(project, None)+}++/// [`footage_salt`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe.+pub fn footage_salt_with(project: &effectcraft_project::Project, virtual_id: Option<VirtualFileId<'_>>) -> u64 {let mut h = Hash128::default();h.write(RENDER_VERSION.as_bytes());for (id, it) in &project.items {if let effectcraft_project::ItemKind::Footage(f) = &it.kind {h.write_u64(id.0);hash_debug(&mut h, f);- footage_stamp(&mut h, f);+ footage_stamp(&mut h, f, virtual_id);}}let v = h.finish();@@ -822,6 +836,11 @@ pub fn footage_salt(project: &effectcraft_project::Project) -> u64 {/// files' size and time). With expressions in play (which can read any comp) the whole project/// counts. Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) don't count.pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 {+ comp_content_key_with(project, comp, None)+}++/// [`comp_content_key`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe.+pub fn comp_content_key_with(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId, virtual_id: Option<VirtualFileId<'_>>) -> u128 {use effectcraft_project::{ItemKind, LayerSource};let mut h = Hash128::default();h.write(RENDER_VERSION.as_bytes());@@ -855,11 +874,11 @@ pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcrafk => hash_debug(&mut h, k),}if let ItemKind::Footage(f) = &it.kind {- footage_stamp(&mut h, f);+ footage_stamp(&mut h, f, virtual_id);}hash_debug(&mut h, &it.proxy);if let Some(px) = &it.proxy {- footage_stamp(&mut h, &px.footage);+ footage_stamp(&mut h, &px.footage, virtual_id);}}h.write_u64(comp.0);@@ -1082,4 +1101,29 @@ mod tests {assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img);}}++ /// A path with no native metadata takes its key from the caller's size and content hash (#1253).+ #[test]+ fn virtual_file_bytes_change_content_keys() {+ use effectcraft_project::build;+ use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project};+ use effectcraft_time::{FrameRate, Tick};++ let mut p = Project::default();+ let f = Footage { path: "/virtual.mov".into(), width: 16, height: 9, duration: Tick::from_seconds_f64(1.0), has_video: true, ..Default::default() };+ let fid = p.add_item("virtual.mov", effectcraft_color::Label::None, None, ItemKind::Footage(f));+ let mut comp = Comp::new(16, 9, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));+ comp.layers = vec![build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (16, 9), None)];+ let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into()));++ let id = |mark: u64| move |path: &str| (path == "/virtual.mov").then_some((8, mark));+ let (one, two) = (id(1), id(2));+ let a = footage_salt_with(&p, Some(&one));+ let b = footage_salt_with(&p, Some(&two));+ let ka = comp_content_key_with(&p, cid, Some(&one));+ let kb = comp_content_key_with(&p, cid, Some(&two));+ assert_ne!(a, b, "replaced virtual media must not reuse the salt");+ assert_ne!(ka, kb, "replaced virtual media must not reuse the frame key");+ assert_eq!(comp_content_key(&p, cid), comp_content_key_with(&p, cid, None));+ }}diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rsindex 059bc3a03..a0a893452 100644--- a/crates/ui-egui/src/frames.rs+++ b/crates/ui-egui/src/frames.rs@@ -24,7 +24,7 @@//! them (Info panel, eyedroppers, histograms).use std::collections::{BTreeSet, HashMap, HashSet, VecDeque};-use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};use std::sync::{Arc, Mutex};use effectcraft_engine::project::{Comp, ItemId, ItemKind, LayerSource, Node, Project, PropGroup};@@ -258,8 +258,10 @@ pub struct RenderSource {/// ([`Frames::identity`]).type Identity = (u64, Option<Arc<Project>>);-/// Content keys of (project revision, comp), computed once per revision.-type ContentKeys = Arc<Mutex<HashMap<(u64, u64), u128>>>;+/// Content keys of (project revision, comp, virtual-file generation). The generation is one+/// counter bumped when a browser file is replaced, so a lookup stays a map read. Hashing the+/// new bytes happens only when that generation misses.+type ContentKeys = Arc<Mutex<HashMap<(u64, u64, u64), u128>>>;#[cfg(not(target_arch = "wasm32"))]#[derive(Default)]@@ -268,16 +270,16 @@ struct ContentKeyJobs {pending: VecDeque<(Arc<Project>, u64, u64)>,}-fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 {- if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) {+fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64, epoch: u64, virtual_id: Option<disk_cache::VirtualFileId<'_>>) -> u128 {+ if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp, epoch)).copied()) {return k;}- let k = disk_cache::comp_content_key(project, ItemId(comp));+ let k = disk_cache::comp_content_key_with(project, ItemId(comp), virtual_id);if let Ok(mut m) = keys.lock() {if m.len() > 256 {m.clear();}- m.insert((revision, comp), k);+ m.insert((revision, comp, epoch), k);}k}@@ -363,8 +365,15 @@ pub fn opts_hash(o: &RenderOpts) -> u64 {}/// Disk key of a viewer frame (render options included: region of interest, draft…).-fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 {- let content = content_key(keys, project, key.revision, key.comp);+fn frame_disk_key(+ keys: &ContentKeys,+ project: &Project,+ key: &FrameKey,+ opts_hash: u64,+ epoch: u64,+ virtual_id: Option<disk_cache::VirtualFileId<'_>>,+) -> u128 {+ let content = content_key(keys, project, key.revision, key.comp, epoch, virtual_id);disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash)}@@ -441,6 +450,10 @@ pub struct Frames {/// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout.pub last_ms: Arc<Mutex<f64>>,content_keys: ContentKeys,+ /// Bumped when a virtual file is replaced. Lookups key on it instead of statting footage.+ virtual_epoch: Arc<AtomicU64>,+ /// Size and content hash for paths with no native metadata. Called only on a cache miss.+ virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,#[cfg(not(target_arch = "wasm32"))]content_key_jobs: Arc<Mutex<ContentKeyJobs>>,gpu_retired: Arc<AtomicBool>,@@ -477,6 +490,8 @@ impl Default for Frames {ctx: None,last_ms: Arc::new(Mutex::new(0.0)),content_keys: Arc::default(),+ virtual_epoch: Arc::new(AtomicU64::new(0)),+ virtual_id: None,#[cfg(not(target_arch = "wasm32"))]content_key_jobs: Arc::default(),gpu_retired: Arc::default(),@@ -768,9 +783,10 @@ impl Frames {if src.disk.is_none() && remote.is_none() {return vec![];}+ let epoch = self.virtual_epoch.load(Ordering::Relaxed);#[cfg(not(target_arch = "wasm32"))]let content = {- let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp)).copied();+ let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp, epoch)).copied();let Some(content) = known else {self.request_content_key(src.project.clone(), base.revision, base.comp);return vec![];@@ -778,7 +794,7 @@ impl Frames {content};#[cfg(target_arch = "wasm32")]- let content = content_key(&self.content_keys, &src.project, base.revision, base.comp);+ let content = content_key(&self.content_keys, &src.project, base.revision, base.comp, epoch, self.virtual_id.as_deref());let ram: HashSet<i64> = self.cached_frames(base).into_iter().collect();let oh = opts_hash(opts);(0..frames.clamp(0, 100_000))@@ -803,6 +819,7 @@ impl Frames {jobs.running = true;drop(jobs);let (jobs, keys, ctx) = (self.content_key_jobs.clone(), self.content_keys.clone(), self.ctx.clone());+ let (epoch, id) = (self.virtual_epoch.clone(), self.virtual_id.clone());let work = move || loop {let next = {let mut jobs = jobs.lock().unwrap_or_else(std::sync::PoisonError::into_inner);@@ -813,7 +830,9 @@ impl Frames {next};let Some((project, revision, comp)) = next else { return };- if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp))).is_err() {+ let epoch = epoch.load(Ordering::Relaxed);+ let id = id.clone();+ if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp, epoch, id.as_deref()))).is_err() {log::error!("could not compute a preview disk-cache key; the cache indicator will retry");}if let Some(ctx) = &ctx {@@ -964,10 +983,22 @@ impl Frames {ctx: self.ctx.clone(),last: self.last_ms.clone(),content_keys: self.content_keys.clone(),+ virtual_epoch: self.virtual_epoch.clone(),+ virtual_id: self.virtual_id.clone(),gpu_retired: self.gpu_retired.clone(),}}+ /// Browser file table: `epoch` bumps on each replacement, and `id` is the size and content+ /// hash used when a content key is rebuilt. Neither runs on a cache hit.+ pub fn set_virtual_media(&mut self, epoch: Arc<AtomicU64>, id: Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>) {+ self.virtual_epoch = epoch;+ self.virtual_id = Some(id);+ if let Ok(mut keys) = self.content_keys.lock() {+ keys.clear();+ }+ }+/// Render frames with `remote` from now on (`None`: on this thread again).pub fn set_remote(&mut self, remote: Option<Arc<dyn RemoteFrames>>) {self.remote = remote;@@ -1036,7 +1067,8 @@ impl Frames {let fw = self.worker();let busy = self.remote_busy.clone();let t0 = web_time::Instant::now();- let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts)));+ let epoch = w.virtual_epoch.load(Ordering::Relaxed);+ let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts), epoch, w.virtual_id.as_deref()));let rj = RemoteJob { project: job.src.project.clone(), revision: key.revision, comp: job.comp, t: job.t, opts: job.opts, disk_key };remote.start(rj,@@ -1093,6 +1125,8 @@ struct Worker {ctx: Option<egui::Context>,last: Arc<Mutex<f64>>,content_keys: ContentKeys,+ virtual_epoch: Arc<AtomicU64>,+ virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,gpu_retired: Arc<AtomicBool>,}@@ -1147,7 +1181,12 @@ impl Worker {job};let t0 = web_time::Instant::now();- let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts))));+ let epoch = self.virtual_epoch.load(Ordering::Relaxed);+ let disk = job+ .src+ .disk+ .as_ref()+ .map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts), epoch, self.virtual_id.as_deref())));let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k));let disk_hit = from_disk.is_some();let ci = match from_disk {@@ -1428,7 +1467,7 @@ mod tests {assert_eq!(frames.content_key_jobs.lock().unwrap().pending.len(), 1);release.send(()).unwrap();let until = Instant::now() + Duration::from_secs(3);- while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp)) {+ while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp, 0)) {assert!(Instant::now() < until);std::thread::sleep(Duration::from_millis(1));}