LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
M0.12: web persistence — library, edits and imported originals survive a reload
Brandon Thomasbaa30ca59025
diff --git a/Cargo.lock b/Cargo.lockindex 8ff6fc3ef..20ae00caf 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2229,9 +2229,11 @@ dependencies = ["js-sys","lightcraft-codecs","lightcraft-engine",+ "lightcraft-preview","lightcraft-raster","lightcraft-ui-egui","log",+ "serde","serde_json","wasm-bindgen","wasm-bindgen-futures",diff --git a/apps/lightcraft-web/Cargo.toml b/apps/lightcraft-web/Cargo.tomlindex b9c7bdf23..1f9f14c13 100644--- a/apps/lightcraft-web/Cargo.toml+++ b/apps/lightcraft-web/Cargo.toml@@ -17,6 +17,8 @@ lightcraft-engine = { workspace = true }lightcraft-ui-egui = { workspace = true }lightcraft-codecs = { workspace = true }lightcraft-raster = { workspace = true }+lightcraft-preview = { workspace = true }+serde = { workspace = true }serde_json = { workspace = true }log = { workspace = true }@@ -29,16 +31,42 @@ js-sys = { workspace = true }web-sys = { workspace = true, features = ["Blob","BlobPropertyBag",+ "DedicatedWorkerGlobalScope","Document",+ "DomStringList","Element",+ "EventTarget","File","FileList",+ "FileSystemCreateWritableOptions",+ "FileSystemDirectoryHandle",+ "FileSystemFileHandle",+ "FileSystemGetDirectoryOptions",+ "FileSystemGetFileOptions",+ "FileSystemRemoveOptions",+ "FileSystemWritableFileStream","HtmlAnchorElement","HtmlCanvasElement","HtmlElement","HtmlInputElement",+ "IdbDatabase",+ "IdbFactory",+ "IdbKeyRange",+ "IdbObjectStore",+ "IdbOpenDbRequest",+ "IdbRequest",+ "IdbTransaction",+ "IdbTransactionMode",+ "IdbVersionChangeEvent","Location",+ "MessageEvent",+ "Navigator","Performance",+ "StorageManager","Url","Window",+ "Worker",+ "WorkerOptions",+ "WorkerType",+ "WritableStream",] }diff --git a/apps/lightcraft-web/index.html b/apps/lightcraft-web/index.htmlindex ac1ed167a..bc0ad2037 100644--- a/apps/lightcraft-web/index.html+++ b/apps/lightcraft-web/index.html@@ -22,8 +22,16 @@<div id="lightcraft_loading">Loading LightCraft…</div><div id="lightcraft_status" aria-live="polite"></div><script type="module">- import init from "./lightcraft_web.js";- init().catch((e) => {+ import init, { start, command } from "./lightcraft_web.js";+ (async () => {+ // Compile once; the render workers (worker.js) instantiate the same compiled module.+ const module = await WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm"));+ globalThis.lightcraftModule = module;+ await init({ module_or_path: module });+ // automation: `await lightcraft.command("library.info", "{}")` → JSON text+ globalThis.lightcraft = { command };+ start();+ })().catch((e) => {console.error(e);const el = document.getElementById("lightcraft_loading");if (el) el.textContent = "LightCraft failed to load: " + e;diff --git a/apps/lightcraft-web/src/backend.rs b/apps/lightcraft-web/src/backend.rsnew file mode 100644index 000000000..50a03135f--- /dev/null+++ b/apps/lightcraft-web/src/backend.rs@@ -0,0 +1,324 @@+//! Browser storage, asynchronous: OPFS (Origin Private File System) when the page can write it+//! from the main thread (`FileSystemFileHandle.createWritable`), else IndexedDB. Used from the+//! window and from render workers alike. Keys are `/`-separated paths (OPFS directories; plain+//! IndexedDB keys).++use js_sys::{Reflect, Uint8Array};+use wasm_bindgen::prelude::*;+use wasm_bindgen::{JsCast, JsValue};+use wasm_bindgen_futures::JsFuture;+use web_sys::{FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, IdbDatabase};++const IDB_NAME: &str = "lightcraft";+const IDB_STORE: &str = "files";++#[derive(Clone)]+pub enum Backend {+ Opfs(FileSystemDirectoryHandle),+ Idb(IdbDatabase),+}++fn err(e: JsValue) -> String {+ e.as_string().or_else(|| e.dyn_ref::<js_sys::Error>().map(|e| String::from(e.message()))).unwrap_or_else(|| format!("{e:?}"))+}++/// `navigator.storage` in a window or a worker.+fn storage_manager() -> Option<web_sys::StorageManager> {+ let g = js_sys::global();+ let nav = Reflect::get(&g, &"navigator".into()).ok()?;+ let storage = Reflect::get(&nav, &"storage".into()).ok()?;+ (!storage.is_undefined()).then(|| storage.unchecked_into())+}++fn indexed_db() -> Option<web_sys::IdbFactory> {+ let f = Reflect::get(&js_sys::global(), &"indexedDB".into()).ok()?;+ (!f.is_undefined() && !f.is_null()).then(|| f.unchecked_into())+}++/// Can files be written through OPFS writable streams here?+fn opfs_writable() -> bool {+ let g = js_sys::global();+ Reflect::get(&g, &"FileSystemFileHandle".into())+ .ok()+ .filter(|c| !c.is_undefined())+ .and_then(|c| Reflect::get(&c, &"prototype".into()).ok())+ .and_then(|p| Reflect::get(&p, &"createWritable".into()).ok())+ .is_some_and(|f| f.is_function())+}++/// Resolve an IndexedDB request.+async fn idb_req(req: &web_sys::IdbRequest) -> Result<JsValue, String> {+ let r1 = req.clone();+ let p = js_sys::Promise::new(&mut |resolve, reject| {+ let r2 = r1.clone();+ let ok = Closure::once_into_js(move || {+ let _ = resolve.call1(&JsValue::NULL, &r2.result().unwrap_or(JsValue::UNDEFINED));+ });+ let fail = Closure::once_into_js(move |e: JsValue| {+ let _ = reject.call1(&JsValue::NULL, &e);+ });+ r1.set_onsuccess(Some(ok.unchecked_ref()));+ r1.set_onerror(Some(fail.unchecked_ref()));+ });+ JsFuture::from(p).await.map_err(err)+}++/// Wait for an IndexedDB transaction to commit.+async fn idb_done(tx: &web_sys::IdbTransaction) -> Result<(), String> {+ let t1 = tx.clone();+ let p = js_sys::Promise::new(&mut |resolve, reject| {+ let ok = Closure::once_into_js(move || {+ let _ = resolve.call0(&JsValue::NULL);+ });+ let rej = reject.clone();+ let fail = Closure::once_into_js(move |e: JsValue| {+ let _ = rej.call1(&JsValue::NULL, &e);+ });+ let abort = Closure::once_into_js(move |e: JsValue| {+ let _ = reject.call1(&JsValue::NULL, &e);+ });+ t1.set_oncomplete(Some(ok.unchecked_ref()));+ t1.set_onerror(Some(fail.unchecked_ref()));+ t1.set_onabort(Some(abort.unchecked_ref()));+ });+ JsFuture::from(p).await.map(|_| ()).map_err(err)+}++impl Backend {+ /// OPFS if writable from here (unless `prefer_idb`), else IndexedDB.+ pub async fn open(prefer_idb: bool) -> Result<Backend, String> {+ if !prefer_idb+ && opfs_writable()+ && let Some(sm) = storage_manager()+ {+ match JsFuture::from(sm.get_directory()).await {+ Ok(root) => return Ok(Backend::Opfs(root.unchecked_into())),+ Err(e) => log::warn!("OPFS unavailable ({}); using IndexedDB", err(e)),+ }+ }+ Self::open_idb().await+ }++ async fn open_idb() -> Result<Backend, String> {+ let f = indexed_db().ok_or("no IndexedDB")?;+ let req = f.open_with_u32(IDB_NAME, 1).map_err(err)?;+ let upgrade = Closure::<dyn FnMut(web_sys::IdbVersionChangeEvent)>::new(move |e: web_sys::IdbVersionChangeEvent| {+ if let Some(db) = e.target().and_then(|t| t.dyn_into::<web_sys::IdbOpenDbRequest>().ok()).and_then(|r| r.result().ok()) {+ let db: IdbDatabase = db.unchecked_into();+ if !db.object_store_names().contains(IDB_STORE) {+ let _ = db.create_object_store(IDB_STORE);+ }+ }+ });+ req.set_onupgradeneeded(Some(upgrade.as_ref().unchecked_ref()));+ let db = idb_req(&req).await?;+ drop(upgrade);+ Ok(Backend::Idb(db.unchecked_into()))+ }++ /// "opfs" or "idb" (workers open the same kind).+ pub fn kind(&self) -> &'static str {+ match self {+ Backend::Opfs(_) => "opfs",+ Backend::Idb(_) => "idb",+ }+ }++ /// The directory holding `path`'s last component (created when `create`).+ async fn parent(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<(FileSystemDirectoryHandle, String)>, String> {+ let mut parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();+ let name = parts.pop().ok_or("empty path")?.to_string();+ let mut dir = root.clone();+ for p in parts {+ let opts = FileSystemGetDirectoryOptions::new();+ opts.set_create(create);+ match JsFuture::from(dir.get_directory_handle_with_options(p, &opts)).await {+ Ok(d) => dir = d.unchecked_into(),+ Err(e) if !create => {+ let _ = e;+ return Ok(None);+ }+ Err(e) => return Err(err(e)),+ }+ }+ Ok(Some((dir, name)))+ }++ async fn opfs_file(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<FileSystemFileHandle>, String> {+ let Some((dir, name)) = Self::parent(root, path, create).await? else { return Ok(None) };+ let opts = FileSystemGetFileOptions::new();+ opts.set_create(create);+ match JsFuture::from(dir.get_file_handle_with_options(&name, &opts)).await {+ Ok(h) => Ok(Some(h.unchecked_into())),+ Err(_) if !create => Ok(None),+ Err(e) => Err(err(e)),+ }+ }++ fn idb_store(db: &IdbDatabase, write: bool) -> Result<(web_sys::IdbTransaction, web_sys::IdbObjectStore), String> {+ let mode = if write { web_sys::IdbTransactionMode::Readwrite } else { web_sys::IdbTransactionMode::Readonly };+ let tx = db.transaction_with_str_and_mode(IDB_STORE, mode).map_err(err)?;+ let store = tx.object_store(IDB_STORE).map_err(err)?;+ Ok((tx, store))+ }++ /// A whole file, or `None` if it doesn't exist.+ pub async fn read(&self, path: &str) -> Result<Option<Vec<u8>>, String> {+ match self {+ Backend::Opfs(root) => {+ let Some(h) = Self::opfs_file(root, path, false).await? else { return Ok(None) };+ let file: web_sys::File = JsFuture::from(h.get_file()).await.map_err(err)?.unchecked_into();+ let buf = JsFuture::from(file.array_buffer()).await.map_err(err)?;+ Ok(Some(Uint8Array::new(&buf).to_vec()))+ }+ Backend::Idb(db) => {+ let (_tx, store) = Self::idb_store(db, false)?;+ let v = idb_req(&store.get(&path.into()).map_err(err)?).await?;+ if v.is_undefined() || v.is_null() {+ return Ok(None);+ }+ Ok(Some(Uint8Array::new(&v).to_vec()))+ }+ }+ }++ /// Replace a file atomically (OPFS: the writable's swap file is committed on close).+ pub async fn write(&self, path: &str, data: &[u8]) -> Result<(), String> {+ self.write_at(path, None, data).await+ }++ /// Write `data` at `offset` keeping the bytes before it (`None`: replace the file). Atomic.+ pub async fn write_at(&self, path: &str, offset: Option<usize>, data: &[u8]) -> Result<(), String> {+ match self {+ Backend::Opfs(root) => {+ let h = Self::opfs_file(root, path, true).await?.ok_or("can't create file")?;+ let opts = web_sys::FileSystemCreateWritableOptions::new();+ opts.set_keep_existing_data(offset.is_some());+ let w: web_sys::FileSystemWritableFileStream =+ JsFuture::from(h.create_writable_with_options(&opts)).await.map_err(err)?.unchecked_into();+ let res = async {+ if let Some(off) = offset {+ JsFuture::from(w.truncate_with_f64(off as f64).map_err(err)?).await.map_err(err)?;+ JsFuture::from(w.seek_with_f64(off as f64).map_err(err)?).await.map_err(err)?;+ }+ JsFuture::from(w.write_with_js_u8_array(&Uint8Array::from(data)).map_err(err)?).await.map_err(err)?;+ Ok::<(), String>(())+ }+ .await;+ match res {+ Ok(()) => JsFuture::from(w.close()).await.map(|_| ()).map_err(err),+ Err(e) => {+ let _ = JsFuture::from(w.abort()).await;+ Err(e)+ }+ }+ }+ Backend::Idb(db) => {+ let (tx, store) = Self::idb_store(db, true)?;+ let value = match offset {+ // read-modify-write inside one transaction+ Some(off) => {+ let old = idb_req(&store.get(&path.into()).map_err(err)?).await?;+ let mut v = if old.is_undefined() || old.is_null() { Vec::new() } else { Uint8Array::new(&old).to_vec() };+ v.truncate(off);+ v.extend_from_slice(data);+ Uint8Array::from(&v[..])+ }+ None => Uint8Array::from(data),+ };+ store.put_with_key(&value, &path.into()).map_err(err)?;+ idb_done(&tx).await+ }+ }+ }++ #[allow(dead_code)] // thumbnail cache pruning (render workers)+ pub async fn remove(&self, path: &str) -> Result<(), String> {+ match self {+ Backend::Opfs(root) => {+ let Some((dir, name)) = Self::parent(root, path, false).await? else { return Ok(()) };+ match JsFuture::from(dir.remove_entry(&name)).await {+ Ok(_) => Ok(()),+ Err(e) if Reflect::get(&e, &"name".into()).ok().and_then(|n| n.as_string()).as_deref() == Some("NotFoundError") => Ok(()),+ Err(e) => Err(err(e)),+ }+ }+ Backend::Idb(db) => {+ let (tx, store) = Self::idb_store(db, true)?;+ store.delete(&path.into()).map_err(err)?;+ idb_done(&tx).await+ }+ }+ }++ /// Names of the files directly inside `dir`.+ #[allow(dead_code)] // thumbnail cache pruning (render workers)+ pub async fn list(&self, dir: &str) -> Result<Vec<String>, String> {+ match self {+ Backend::Opfs(root) => {+ // `parent` of a child path is `dir` itself+ let Some((handle, _)) = Self::parent(root, &format!("{dir}/-"), false).await? else { return Ok(Vec::new()) };+ dir_keys(&handle).await+ }+ Backend::Idb(db) => {+ let (_tx, store) = Self::idb_store(db, false)?;+ let prefix = format!("{dir}/");+ let range = web_sys::IdbKeyRange::bound(&prefix.as_str().into(), &format!("{prefix}\u{ffff}").into()).map_err(err)?;+ let keys = idb_req(&store.get_all_keys_with_key(&range).map_err(err)?).await?;+ let arr: js_sys::Array = keys.unchecked_into();+ Ok(arr+ .iter()+ .filter_map(|k| k.as_string())+ .filter_map(|k| k.strip_prefix(&prefix).filter(|r| !r.contains('/')).map(str::to_string))+ .collect())+ }+ }+ }++ /// Delete everything (the "reset" escape hatch: `?reset` in the URL).+ pub async fn clear(&self) -> Result<(), String> {+ match self {+ Backend::Opfs(root) => {+ for name in dir_keys(root).await? {+ let opts = web_sys::FileSystemRemoveOptions::new();+ opts.set_recursive(true);+ JsFuture::from(root.remove_entry_with_options(&name, &opts)).await.map_err(err)?;+ }+ Ok(())+ }+ Backend::Idb(db) => {+ let (tx, store) = Self::idb_store(db, true)?;+ store.clear().map_err(err)?;+ idb_done(&tx).await+ }+ }+ }+}++/// Names of the entries in an OPFS directory.+async fn dir_keys(dir: &FileSystemDirectoryHandle) -> Result<Vec<String>, String> {+ let keys = dir.keys();+ let mut out = Vec::new();+ loop {+ let next = JsFuture::from(keys.next().map_err(err)?).await.map_err(err)?;+ if Reflect::get(&next, &"done".into()).ok().is_some_and(|d| d.is_truthy()) {+ break;+ }+ if let Some(k) = Reflect::get(&next, &"value".into()).ok().and_then(|v| v.as_string()) {+ out.push(k);+ }+ }+ Ok(out)+}++/// Ask the browser not to evict our storage under pressure (best effort; fire and forget).+pub fn request_persistence() {+ if let Some(sm) = storage_manager()+ && let Ok(p) = sm.persist()+ {+ wasm_bindgen_futures::spawn_local(async move {+ let granted = JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy());+ log::info!("lightcraft: persistent storage {}", if granted { "granted" } else { "not granted (best effort)" });+ });+ }+}diff --git a/apps/lightcraft-web/src/bench.rs b/apps/lightcraft-web/src/bench.rsindex 7ede4a7ad..90b85af84 100644--- a/apps/lightcraft-web/src/bench.rs+++ b/apps/lightcraft-web/src/bench.rs@@ -65,7 +65,7 @@ impl Bench {if self.first_frame_ms.is_none() && self.frames > 1 {self.first_frame_ms = Some(now - self.start_ms);}- let idle = app.renderer.queued() == 0;+ let idle = app.renderer.queued() == 0 && app.renderer.in_flight() == 0;match self.phase {Phase::Idle => {let thumbs = app.renderer.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count();diff --git a/apps/lightcraft-web/src/files.rs b/apps/lightcraft-web/src/files.rsnew file mode 100644index 000000000..18acc93d3--- /dev/null+++ b/apps/lightcraft-web/src/files.rs@@ -0,0 +1,255 @@+//! The library's small files (catalog journal, presets, view state, prefs) on the web.+//!+//! Browser storage is asynchronous on the main thread (OPFS writable streams, IndexedDB), but the+//! catalog [`Store`] is synchronous. So the files are mirrored in memory: they are read once at+//! start-up ([`Files::preload`]), every write changes the mirror immediately and marks the file+//! dirty, and the host flushes dirty files to storage in the background ([`Files::take_dirty`],+//! about once per frame while anything is dirty).+//!+//! Durability: a write is in browser storage a few milliseconds after the command that made it+//! (one flush), not before the command returns as on the desktop. Each flushed file is replaced+//! atomically (OPFS swap file / one IndexedDB transaction), and files are flushed in the order they+//! were last modified, so the journal's recovery rules still hold: e.g. a new `catalog.snap` always+//! lands before the `catalog.log` reset that follows it.++use std::collections::BTreeMap;+use std::io;+use std::sync::{Arc, Mutex};++use lightcraft_engine::catalog::Store;++/// What has to be written for a dirty file.+#[derive(Clone, Debug, PartialEq)]+pub enum FlushOp {+ /// Replace the whole file.+ Write { name: String, data: Vec<u8> },+ /// Bytes appended since the last flush, starting at `offset` (the stored file's length).+ Append { name: String, offset: usize, data: Vec<u8> },+}++impl FlushOp {+ pub fn name(&self) -> &str {+ match self {+ FlushOp::Write { name, .. } | FlushOp::Append { name, .. } => name,+ }+ }+}++#[derive(Clone, Copy, Debug, PartialEq)]+enum Dirty {+ Rewrite,+ /// Only appended since the last flush; the stored file has `from` bytes.+ Append {+ from: usize,+ },+}++#[derive(Default)]+struct Mirror {+ files: BTreeMap<String, Vec<u8>>,+ /// Dirty files, least recently modified first.+ dirty: Vec<(String, Dirty)>,+}++impl Mirror {+ fn mark(&mut self, name: &str, d: Dirty) {+ let prev = self.dirty.iter().position(|(n, _)| n == name).map(|i| self.dirty.remove(i).1);+ let d = match (prev, d) {+ // appends after appends keep the original offset; anything else needs a rewrite+ (Some(Dirty::Append { from }), Dirty::Append { .. }) => Dirty::Append { from },+ (Some(Dirty::Rewrite), _) => Dirty::Rewrite,+ (_, d) => d,+ };+ self.dirty.push((name.to_string(), d));+ }+}++/// Shared in-memory mirror of the library files (cheap to clone).+#[derive(Clone, Default)]+pub struct Files(Arc<Mutex<Mirror>>);++impl Files {+ fn lock(&self) -> std::sync::MutexGuard<'_, Mirror> {+ self.0.lock().unwrap_or_else(|e| e.into_inner())+ }++ /// Set a file's stored content (start-up; not dirty).+ pub fn preload(&self, name: &str, data: Vec<u8>) {+ self.lock().files.insert(name.to_string(), data);+ }++ pub fn get(&self, name: &str) -> Option<Vec<u8>> {+ self.lock().files.get(name).cloned()+ }++ /// Replace a file.+ pub fn write(&self, name: &str, data: &[u8]) {+ let mut m = self.lock();+ m.files.insert(name.to_string(), data.to_vec());+ m.mark(name, Dirty::Rewrite);+ }++ pub fn append(&self, name: &str, data: &[u8]) {+ let mut m = self.lock();+ let f = m.files.entry(name.to_string()).or_default();+ let from = f.len();+ f.extend_from_slice(data);+ m.mark(name, Dirty::Append { from });+ }++ pub fn truncate(&self, name: &str, len: usize) {+ let mut m = self.lock();+ if let Some(f) = m.files.get_mut(name) {+ f.truncate(len);+ m.mark(name, Dirty::Rewrite);+ }+ }++ pub fn is_dirty(&self) -> bool {+ !self.lock().dirty.is_empty()+ }++ /// Everything to write, in modification order; the files count as clean afterwards (call+ /// [`Files::failed`] for ops that couldn't be written).+ pub fn take_dirty(&self) -> Vec<FlushOp> {+ let mut m = self.lock();+ let dirty = std::mem::take(&mut m.dirty);+ dirty+ .into_iter()+ .map(|(name, d)| {+ let data = m.files.get(&name).cloned().unwrap_or_default();+ match d {+ Dirty::Append { from } if from <= data.len() => FlushOp::Append { offset: from, data: data[from..].to_vec(), name },+ _ => FlushOp::Write { name, data },+ }+ })+ .collect()+ }++ /// A flush failed: write these files in full next time (keeps any newer modification order).+ pub fn failed(&self, ops: &[FlushOp]) {+ let mut m = self.lock();+ for op in ops.iter().rev() {+ if !m.dirty.iter().any(|(n, _)| n == op.name()) {+ m.dirty.insert(0, (op.name().to_string(), Dirty::Rewrite));+ } else if let Some(d) = m.dirty.iter_mut().find(|(n, _)| n == op.name()) {+ d.1 = Dirty::Rewrite;+ }+ }+ }++ /// A catalog [`Store`] over these files.+ pub fn store(&self) -> FilesStore {+ FilesStore(self.clone())+ }+}++/// [`Store`] implementation over [`Files`].+pub struct FilesStore(Files);++impl Store for FilesStore {+ fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {+ Ok(self.0.get(name))+ }+ fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {+ self.0.write(name, data);+ Ok(())+ }+ fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {+ self.0.append(name, data);+ Ok(())+ }+ fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {+ self.0.truncate(name, len as usize);+ Ok(())+ }+ fn describe(&self) -> String {+ "browser storage".into()+ }+}++/// Names of the library files read at start-up.+pub const LIBRARY_FILES: [&str; 6] = ["catalog.snap", "catalog.log", "presets.json", "view.json", "prefs.json", "ui.json"];++#[cfg(test)]+mod tests {+ use super::*;+ use lightcraft_engine::Session;+ use lightcraft_engine::library::LibraryStores;+ use serde_json::json;++ /// Apply flush ops to a "disk" map, as the backend would.+ fn flush(files: &Files, disk: &mut BTreeMap<String, Vec<u8>>) {+ for op in files.take_dirty() {+ match op {+ FlushOp::Write { name, data } => {+ disk.insert(name, data);+ }+ FlushOp::Append { name, offset, data } => {+ let f = disk.entry(name).or_default();+ assert_eq!(f.len(), offset, "append offset matches the stored length");+ f.extend_from_slice(&data);+ }+ }+ }+ }++ #[test]+ fn appends_coalesce_and_rewrites_win() {+ let f = Files::default();+ f.preload("log", b"ab".to_vec());+ f.append("log", b"c");+ f.append("log", b"d");+ assert_eq!(f.take_dirty(), vec![FlushOp::Append { name: "log".into(), offset: 2, data: b"cd".to_vec() }]);+ assert!(!f.is_dirty());+ f.write("snap", b"S");+ f.truncate("log", 1);+ f.append("log", b"x");+ let ops = f.take_dirty();+ assert_eq!(+ ops,+ vec![FlushOp::Write { name: "snap".into(), data: b"S".to_vec() }, FlushOp::Write { name: "log".into(), data: b"ax".to_vec() }]+ );+ f.failed(&ops);+ assert_eq!(f.take_dirty().len(), 2);+ }++ #[test]+ fn modification_order_puts_snapshot_before_log_reset() {+ let f = Files::default();+ f.append("catalog.log", b"1\n");+ f.write("catalog.snap", b"snap");+ f.write("catalog.log", b"");+ let names: Vec<_> = f.take_dirty().iter().map(|o| o.name().to_string()).collect();+ assert_eq!(names, ["catalog.snap", "catalog.log"]);+ }++ #[test]+ fn library_survives_reload_through_flushes() {+ let mut disk = BTreeMap::new();+ let open = |disk: &BTreeMap<String, Vec<u8>>| {+ let files = Files::default();+ for (k, v) in disk {+ files.preload(k, v.clone());+ }+ let mut s = Session::new();+ let stores = LibraryStores { dir: "browser".into(), catalog: Box::new(files.store()), files: Box::new(files.store()), on_disk: false };+ s.open_library_in(stores, true).unwrap();+ (s, files)+ };+ let (mut s, files) = open(&disk);+ assert!(!s.catalog.is_empty(), "seeded with the demo photos");+ flush(&files, &mut disk);+ let id = s.selection.active.unwrap();+ s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.25})).unwrap();+ s.execute("photo.rate", &json!({"rating": 4})).unwrap();+ s.save_view();+ flush(&files, &mut disk);+ let (s2, _) = open(&disk);+ let p = s2.catalog.photo(id).unwrap();+ assert_eq!(p.rating, 4);+ assert_eq!(p.develop.light.exposure, 1.25);+ assert_eq!(s2.catalog.len(), s.catalog.len(), "not seeded twice");+ assert_eq!(s2.selection.active, s.selection.active);+ }+}diff --git a/apps/lightcraft-web/src/lib.rs b/apps/lightcraft-web/src/lib.rsindex 0ab06bc94..a2c5b5898 100644--- a/apps/lightcraft-web/src/lib.rs+++ b/apps/lightcraft-web/src/lib.rs@@ -1,20 +1,24 @@//! LightCraft in the browser: the same egui UI as the desktop app, compiled to WASM.//!-//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP.-//! Differences from the desktop host:-//! - no filesystem: photos picked (File ▸ Add Photos…) or dropped onto the page are held in-//! memory ([`store::MemStore`]) and decoded from bytes;-//! - export downloads the file through the browser;+//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP (see+//! `docs/web.md`). Differences from the desktop host:+//! - the library lives in browser storage (OPFS, or IndexedDB where OPFS can't be written from the+//! main thread): catalog journal, presets, view state and prefs are mirrored in memory and+//! flushed in the background ([`files`]); imported originals are stored by content hash+//! ([`store`]); a new library starts with the procedural demo photos;//! - renders run inline on the main thread (one job per frame) — no worker threads yet;-//! - no persistence yet (UI prefs and the catalog reset on reload);-//! - the procedural demo library is loaded at start so the page is never empty.+//! - export downloads the file through the browser.//!//! `?bench` in the URL runs a scripted first-paint / slider-latency measurement and logs it to the-//! console (see [`bench`]).+//! console (see [`bench`]); `?store=idb` forces the IndexedDB backend, `?store=memory` disables+//! persistence and `?reset` clears the stored library.#![forbid(unsafe_code)]pub mod bench;+pub mod files;pub mod store;+#[cfg(target_arch = "wasm32")]+mod backend;#[cfg(target_arch = "wasm32")]mod web;diff --git a/apps/lightcraft-web/src/store.rs b/apps/lightcraft-web/src/store.rsindex acac328e2..4b395c57b 100644--- a/apps/lightcraft-web/src/store.rs+++ b/apps/lightcraft-web/src/store.rs@@ -1,6 +1,13 @@-//! In-memory "filesystem" for the web build: files picked or dropped in the browser are kept as-//! bytes under synthetic paths (`mem/<n>/<name>`), and the engine reads them through the same-//! [`FileLoader`]/[`FileProbe`] hooks the desktop app backs with `std::fs`.+//! Imported originals on the web. There is no filesystem: the bytes of every photo picked or+//! dropped in the browser are stored in browser storage as `originals/<content hash>`, and the+//! catalog refers to them by a synthetic path `web/<content hash>/<file name>` (so the decoders+//! still see the file's extension).+//!+//! The engine reads files synchronously ([`FileLoader`]/[`FileProbe`]), so the main thread keeps+//! the bytes it needs in a bounded memory cache ([`Originals`]). A miss (e.g. after a reload)+//! queues an asynchronous load from storage ([`Originals::take_wanted`]) and fails for now; the+//! host prefetches the active photo's original so commands that need its pixels (auto tone,+//! export) find it. Render workers read originals from storage themselves.use std::collections::HashMap;use std::sync::{Arc, Mutex};@@ -8,37 +15,119 @@ use std::sync::{Arc, Mutex};use lightcraft_engine::files::{load_bytes, probe_bytes};use lightcraft_engine::media::{FileLoader, FileProbe};+/// Prefix of the catalog paths of files kept in browser storage.+pub const PATH_PREFIX: &str = "web/";++/// Memory budget for original bytes on the main thread.+const MEM_BUDGET: usize = 768 << 20;++/// Catalog path for a stored original.+pub fn original_path(hash: &str, name: &str) -> String {+ let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");+ format!("{PATH_PREFIX}{hash}/{name}")+}++/// The content hash in a path made by [`original_path`].+pub fn hash_of_path(path: &str) -> Option<&str> {+ let (hash, _) = path.strip_prefix(PATH_PREFIX)?.split_once('/')?;+ (!hash.is_empty()).then_some(hash)+}++/// Storage key of an original.+pub fn storage_key(hash: &str) -> String {+ format!("originals/{hash}")+}++/// Hash of a file's bytes (the same as the catalog's `content_hash`).+pub fn content_hash(bytes: &[u8]) -> String {+ lightcraft_preview::hash_bytes(bytes).to_string()+}+#[derive(Default)]struct Inner {- files: HashMap<String, Arc<[u8]>>,- /// Paths added since the last [`MemStore::take_pending`] (to import on the next frame).+ /// hash → (bytes, last use).+ bytes: HashMap<String, (Arc<[u8]>, u64)>,+ total: usize,+ clock: u64,+ /// Catalog paths stored since the last [`Originals::take_pending`] (to import on the next frame).pending: Vec<String>,- next: u64,+ /// Hashes to load from storage.+ wanted: Vec<String>,+ /// Hashes being loaded.+ loading: std::collections::HashSet<String>,}-/// Shared, cloneable byte store. File reads in the browser are async, so additions arrive from-/// promise callbacks and are picked up by the app on its next frame.+/// Original bytes in memory, shared between the engine hooks and the host (cheap to clone).#[derive(Clone, Default)]-pub struct MemStore(Arc<Mutex<Inner>>);+pub struct Originals(Arc<Mutex<Inner>>);-impl MemStore {+impl Originals {fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {self.0.lock().unwrap_or_else(|e| e.into_inner())}- /// Store a file's bytes; returns its synthetic path and queues it for import.- pub fn add(&self, name: &str, bytes: Vec<u8>) -> String {+ /// Keep `bytes` (hash `hash`) in memory, evicting the least recently used others if needed.+ pub fn insert(&self, hash: &str, bytes: Arc<[u8]>) {let mut g = self.lock();- g.next += 1;- let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");- let path = format!("mem/{}/{name}", g.next);- g.files.insert(path.clone(), bytes.into());- g.pending.push(path.clone());+ g.clock += 1;+ let clock = g.clock;+ g.loading.remove(hash);+ if let Some(old) = g.bytes.insert(hash.to_string(), (bytes.clone(), clock)) {+ g.total -= old.0.len();+ }+ g.total += bytes.len();+ while g.total > MEM_BUDGET && g.bytes.len() > 1 {+ let Some(oldest) = g.bytes.iter().filter(|(h, _)| *h != hash).min_by_key(|(_, v)| v.1).map(|(h, _)| h.clone()) else { break };+ if let Some(v) = g.bytes.remove(&oldest) {+ g.total -= v.0.len();+ }+ }+ }++ /// A file was stored (bytes in storage and in memory): queue its path for import.+ pub fn added(&self, name: &str, hash: &str, bytes: Arc<[u8]>) -> String {+ self.insert(hash, bytes);+ let path = original_path(hash, name);+ self.lock().pending.push(path.clone());path}+ /// Bytes for a catalog path, if in memory. A miss for a stored original queues its load.pub fn get(&self, path: &str) -> Option<Arc<[u8]>> {- self.lock().files.get(path).cloned()+ let hash = hash_of_path(path)?;+ let mut g = self.lock();+ g.clock += 1;+ let clock = g.clock;+ if let Some(v) = g.bytes.get_mut(hash) {+ v.1 = clock;+ return Some(v.0.clone());+ }+ if !g.loading.contains(hash) && !g.wanted.iter().any(|w| w == hash) {+ g.wanted.push(hash.to_string());+ }+ None+ }++ pub fn contains(&self, path: &str) -> bool {+ hash_of_path(path).is_some_and(|h| self.lock().bytes.contains_key(h))+ }++ /// Ask for the original behind `path` to be loaded into memory (no-op if it's there).+ pub fn prefetch(&self, path: &str) {+ let _ = self.get(path);+ }++ /// Hashes to load from storage (they count as loading until [`Originals::insert`] or+ /// [`Originals::load_failed`]).+ pub fn take_wanted(&self) -> Vec<String> {+ let mut g = self.lock();+ let w = std::mem::take(&mut g.wanted);+ g.loading.extend(w.iter().cloned());+ w+ }++ pub fn load_failed(&self, hash: &str) {+ self.lock().loading.remove(hash);}/// Paths added since the last call.@@ -46,24 +135,22 @@ impl MemStore {std::mem::take(&mut self.lock().pending)}- pub fn len(&self) -> usize {- self.lock().files.len()- }-- pub fn is_empty(&self) -> bool {- self.len() == 0+ /// (originals in memory, bytes).+ pub fn usage(&self) -> (usize, usize) {+ let g = self.lock();+ (g.bytes.len(), g.total)}- /// Engine hooks that decode/probe from this store.+ /// Engine hooks that decode/probe from memory.pub fn hooks(&self) -> (FileLoader, FileProbe) {let s = self.clone();let loader: FileLoader = Arc::new(move |path: &str, max_edge: usize| {- let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;+ let bytes = s.get(path).ok_or_else(|| format!("{path}: the original is still loading from browser storage"))?;load_bytes(&bytes, max_edge)});let s = self.clone();let probe: FileProbe = Arc::new(move |path: &str| {- let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;+ let bytes = s.get(path).ok_or_else(|| format!("{path}: not loaded"))?;probe_bytes(path, &bytes)});(loader, probe)@@ -94,41 +181,54 @@ pub fn mime_for(name: &str) -> &'static str {}}+#[cfg(test)]+pub(crate) fn png_bytes(w: usize, h: usize) -> Vec<u8> {+ let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();+ let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();+ lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()+}+#[cfg(test)]mod tests {use super::*;use serde_json::json;- fn png_bytes(w: usize, h: usize) -> Vec<u8> {- let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();- let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();- lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()+ #[test]+ fn paths() {+ let p = original_path("abc123", "C:\\x\\a.png");+ assert_eq!(p, "web/abc123/a.png");+ assert_eq!(hash_of_path(&p), Some("abc123"));+ assert_eq!(hash_of_path("mem/1/a.png"), None);+ assert_eq!(storage_key("abc"), "originals/abc");}#[test]- fn add_take_get() {- let s = MemStore::default();- let a = s.add("C:\\x\\a.png", vec![1, 2]);- let b = s.add("a.png", vec![3]);- assert_ne!(a, b, "same name twice gets distinct paths");- assert!(a.ends_with("/a.png"));- assert_eq!(s.take_pending(), vec![a.clone(), b]);- assert!(s.take_pending().is_empty());- assert_eq!(&*s.get(&a).unwrap(), &[1, 2]);- assert_eq!(s.len(), 2);+ fn miss_queues_a_load_once() {+ let s = Originals::default();+ assert!(s.get("web/h1/a.png").is_none());+ assert!(s.get("web/h1/a.png").is_none());+ assert_eq!(s.take_wanted(), vec!["h1".to_string()]);+ assert!(s.get("web/h1/a.png").is_none(), "still loading");+ assert!(s.take_wanted().is_empty());+ s.insert("h1", Arc::from(vec![1u8, 2]));+ assert_eq!(&*s.get("web/h1/b.jpg").unwrap(), &[1, 2]);}#[test]fn import_and_render_from_memory() {- let store = MemStore::default();+ let store = Originals::default();let mut session = lightcraft_engine::Session::new();store.install(&mut session);- let path = store.add("tiny.png", png_bytes(40, 30));+ let bytes = png_bytes(40, 30);+ let hash = content_hash(&bytes);+ let path = store.added("tiny.png", &hash, bytes.into());+ assert_eq!(store.take_pending(), vec![path.clone()]);let r = session.execute("library.import", &json!({"paths": [path]})).unwrap();let id = lightcraft_engine::catalog::PhotoId(r["imported"][0].as_u64().unwrap());let p = session.catalog.photo(id).unwrap();assert_eq!((p.width, p.height), (40, 30));assert_eq!(p.file_name, "tiny.png");+ assert_eq!(p.content_hash.as_deref(), Some(hash.as_str()), "storage key = catalog content hash");let out = session.render_now(id, 20, 20).unwrap();assert_eq!(out.image.width, 20);}diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rsindex c902a2589..d11343b0b 100644--- a/apps/lightcraft-web/src/web.rs+++ b/apps/lightcraft-web/src/web.rs@@ -1,17 +1,32 @@-//! Browser host: starts eframe on `<canvas id="lightcraft_canvas">`, wires the in-memory store to-//! the file picker and drag-and-drop, and turns exports into downloads.+//! Browser host: opens the library in browser storage, starts eframe on+//! `<canvas id="lightcraft_canvas">`, wires the file picker and drag-and-drop, and turns exports+//! into downloads.++use std::cell::Cell;+use std::rc::Rc;+use std::sync::Arc;use lightcraft_engine::Session;-use lightcraft_ui_egui::{LightcraftApp, Services};+use lightcraft_engine::catalog::Source;+use lightcraft_engine::library::LibraryStores;+use lightcraft_ui_egui::{LightcraftApp, Services, UiState};use serde_json::json;use wasm_bindgen::JsCast;use wasm_bindgen::prelude::*;+use crate::backend::Backend;use crate::bench::Bench;-use crate::store::{MemStore, download_name};+use crate::files::{Files, FlushOp, LIBRARY_FILES};+use crate::store::{Originals, content_hash, download_name, storage_key};const ACCEPT: &str = ".jpg,.jpeg,.png,.tif,.tiff,.webp,.dng,.cr2,.nef,.arw,.psd,.jxl,.gif,.bmp";+/// Where the library files live in browser storage.+const LIBRARY_DIR: &str = "library";++/// How often view state and UI prefs are saved (ms).+const SAVE_EVERY_MS: f64 = 1000.0;+fn window() -> web_sys::Window {web_sys::window().expect("no window")}@@ -27,19 +42,62 @@ fn set_status(text: &str) {}}-/// Read a browser `File` into the store.-async fn read_file(store: MemStore, file: web_sys::File, ctx: egui::Context) {- match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {- Ok(buf) => {- store.add(&file.name(), js_sys::Uint8Array::new(&buf).to_vec());- ctx.request_repaint();+fn library_key(name: &str) -> String {+ format!("{LIBRARY_DIR}/{name}")+}++/// URL options (`?bench&store=idb&reset`).+struct Options {+ bench: bool,+ /// "opfs" (default: OPFS, falling back to IndexedDB), "idb" or "memory".+ store: String,+ reset: bool,+}++impl Options {+ fn from_url() -> Options {+ let q = window().location().search().unwrap_or_default();+ let params: Vec<(String, String)> = q+ .trim_start_matches('?')+ .split('&')+ .filter(|p| !p.is_empty())+ .map(|p| match p.split_once('=') {+ Some((k, v)) => (k.to_string(), v.to_string()),+ None => (p.to_string(), String::new()),+ })+ .collect();+ let get = |k: &str| params.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());+ Options {+ bench: get("bench").is_some(),+ store: get("store").filter(|s| s == "idb" || s == "memory").unwrap_or_else(|| "opfs".into()),+ reset: get("reset").is_some(),}+ }+}++/// Store a picked/dropped file (bytes into browser storage, then queue its import).+async fn store_file(originals: Originals, backend: Option<Backend>, name: String, bytes: Vec<u8>, ctx: egui::Context) {+ let hash = content_hash(&bytes);+ if let Some(b) = &backend+ && let Err(e) = b.write(&storage_key(&hash), &bytes).await+ {+ // still importable for this session; it won't survive a reload+ log::error!("storing {name} in browser storage failed: {e}");+ }+ originals.added(&name, &hash, Arc::from(bytes));+ ctx.request_repaint();+}++/// Read a browser `File` into storage.+async fn read_file(originals: Originals, backend: Option<Backend>, file: web_sys::File, ctx: egui::Context) {+ match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {+ Ok(buf) => store_file(originals, backend, file.name(), js_sys::Uint8Array::new(&buf).to_vec(), ctx).await,Err(e) => log::warn!("reading {}: {e:?}", file.name()),}}-/// Show the browser's open dialog; picked files land in the store asynchronously.-fn open_picker(store: MemStore, ctx: egui::Context) {+/// Show the browser's open dialog; picked files are imported asynchronously.+fn open_picker(originals: Originals, backend: Option<Backend>, ctx: egui::Context) {let Some(doc) = window().document() else { return };let Ok(input) = doc.create_element("input").map(|e| e.unchecked_into::<web_sys::HtmlInputElement>()) else { return };input.set_type("file");@@ -50,7 +108,7 @@ fn open_picker(store: MemStore, ctx: egui::Context) {if let Some(files) = inp.files() {for i in 0..files.length() {if let Some(f) = files.get(i) {- wasm_bindgen_futures::spawn_local(read_file(store.clone(), f, ctx.clone()));+ wasm_bindgen_futures::spawn_local(read_file(originals.clone(), backend.clone(), f, ctx.clone()));}}}@@ -83,10 +141,10 @@ fn download(path: &str, bytes: &[u8]) -> Result<(), String> {Ok(())}-fn services(store: MemStore, ctx: egui::Context) -> Services {+fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context) -> Services {Services {pick_files: Some(Box::new(move || {- open_picker(store.clone(), ctx.clone());+ open_picker(originals.clone(), backend.clone(), ctx.clone());Vec::new() // files arrive asynchronously and are imported on a later frame})),// Preset files: browser pickers are asynchronous; not wired on the web yet.@@ -99,44 +157,145 @@ fn services(store: MemStore, ctx: egui::Context) -> Services {}}+/// Write dirty library files to storage, in order, until none are left.+async fn flush(files: Files, backend: Backend, flushing: Rc<Cell<bool>>) {+ 'outer: loop {+ let ops = files.take_dirty();+ if ops.is_empty() {+ break;+ }+ for (i, op) in ops.iter().enumerate() {+ let r = match op {+ FlushOp::Write { name, data } => backend.write(&library_key(name), data).await,+ FlushOp::Append { name, offset, data } => backend.write_at(&library_key(name), Some(*offset), data).await,+ };+ if let Err(e) = r {+ log::error!("saving {} failed: {e}", op.name());+ files.failed(&ops[i..]);+ break 'outer;+ }+ }+ }+ flushing.set(false);+}++/// Everything loaded before the UI starts.+struct Boot {+ opts: Options,+ backend: Option<Backend>,+ files: Files,+}++async fn boot(opts: Options) -> Boot {+ let t0 = perf_now();+ let backend = if opts.store == "memory" {+ None+ } else {+ match Backend::open(opts.store == "idb").await {+ Ok(b) => Some(b),+ Err(e) => {+ log::error!("no browser storage ({e}); this session won't be saved");+ None+ }+ }+ };+ let files = Files::default();+ if let Some(b) = &backend {+ if opts.reset {+ match b.clear().await {+ Ok(()) => log::info!("lightcraft: storage cleared (?reset)"),+ Err(e) => log::error!("clearing storage: {e}"),+ }+ }+ for name in LIBRARY_FILES {+ match b.read(&library_key(name)).await {+ Ok(Some(data)) => files.preload(name, data),+ Ok(None) => {}+ Err(e) => log::error!("reading {name}: {e}"),+ }+ }+ crate::backend::request_persistence();+ }+ log::info!("lightcraft: storage {} opened in {:.0} ms", backend.as_ref().map_or("memory", |b| b.kind()), perf_now() - t0);+ Boot { opts, backend, files }+}+struct WebApp {app: LightcraftApp,- store: MemStore,+ originals: Originals,+ files: Files,+ backend: Option<Backend>,+ flushing: Rc<Cell<bool>>,bench: Option<Bench>,first_frame_logged: bool,+ last_save: f64,+ ui_written: Vec<u8>,}impl WebApp {- fn new(cc: &eframe::CreationContext<'_>, bench: bool) -> Self {- let store = MemStore::default();+ fn new(cc: &eframe::CreationContext<'_>, boot: Boot) -> Self {+ let Boot { opts, backend, files } = boot;+ let originals = Originals::default();let t = perf_now();- let mut session = Session::with_demo();- log::info!("lightcraft: wasm started at {:.0} ms; demo library built in {:.0} ms", t, perf_now() - t);- store.install(&mut session);+ let mut session = Session::new();+ originals.install(&mut session);+ let stores = LibraryStores {+ dir: format!("browser:{}/{LIBRARY_DIR}", backend.as_ref().map_or("memory", |b| b.kind())).into(),+ catalog: Box::new(files.store()),+ files: Box::new(files.store()),+ on_disk: false,+ };+ match session.open_library_in(stores, true).cloned() {+ Ok(r) => log::info!(+ "lightcraft: library {} in {:.0} ms ({} photos; snapshot seq {}, {} ops replayed)",+ if r.created { "created" } else { "loaded" },+ perf_now() - t,+ session.catalog.len(),+ r.snapshot_seq,+ r.replayed+ ),+ Err(e) => {+ log::error!("opening the library failed: {e}; starting a temporary one");+ session = Session::with_demo();+ originals.install(&mut session);+ }+ }// previews are large (≤ 2560 px, f32): keep few in a 32-bit address spacesession.media.preview_capacity = 3;- let mut app = LightcraftApp::new(session, services(store.clone(), cc.egui_ctx.clone()));+ let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), cc.egui_ctx.clone()));+ let ui_written = files.get("ui.json").unwrap_or_default();+ if let Ok(ui) = serde_json::from_slice::<UiState>(&ui_written) {+ app.ui = ui;+ }app.ui = app.ui.sanitized();+ CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone()));let origin = lightcraft_ui_egui::now_ms() - perf_now();- WebApp { app, store, bench: bench.then(|| Bench::new(origin)), first_frame_logged: false }+ WebApp {+ app,+ originals,+ files,+ backend,+ flushing: Rc::new(Cell::new(false)),+ bench: opts.bench.then(|| Bench::new(origin)),+ first_frame_logged: false,+ last_save: 0.0,+ ui_written,+ }}fn import_dropped(&mut self, ctx: &egui::Context) {let dropped = ctx.input(|i| i.raw.dropped_files.clone());for f in dropped {- let (store, ctx) = (self.store.clone(), ctx.clone());+ let (originals, backend, ctx) = (self.originals.clone(), self.backend.clone(), ctx.clone());wasm_bindgen_futures::spawn_local(async move {let name = f.path().to_string_lossy().to_string();match f.bytes_async().await {- Ok(bytes) => {- store.add(&name, bytes);- ctx.request_repaint();- }+ Ok(bytes) => store_file(originals, backend, name, bytes, ctx).await,Err(e) => log::warn!("reading dropped {name}: {e}"),}});}- let paths = self.store.take_pending();+ let paths = self.originals.take_pending();if !paths.is_empty() {let n = paths.len();match self.app.run("library.import", json!({"paths": paths})) {@@ -145,12 +304,114 @@ impl WebApp {}}}++ /// Load originals the main thread asked for (and the active photo's, ahead of time).+ fn load_originals(&mut self, ctx: &egui::Context) {+ if let Some(id) = self.app.session.selection.active+ && let Some(Source::File { path }) = self.app.session.catalog.photo(id).map(|p| &p.source)+ && !self.originals.contains(path)+ {+ self.originals.prefetch(path);+ }+ let Some(b) = &self.backend else { return };+ for hash in self.originals.take_wanted() {+ let (b, originals, ctx) = (b.clone(), self.originals.clone(), ctx.clone());+ wasm_bindgen_futures::spawn_local(async move {+ match b.read(&storage_key(&hash)).await {+ Ok(Some(bytes)) => {+ originals.insert(&hash, Arc::from(bytes));+ ctx.request_repaint();+ }+ Ok(None) => {+ log::warn!("original {hash} is not in browser storage");+ originals.load_failed(&hash);+ }+ Err(e) => {+ log::warn!("loading original {hash}: {e}");+ originals.load_failed(&hash);+ }+ }+ });+ }+ }++ /// Save view state and UI prefs now and then; flush dirty files.+ fn save(&mut self) {+ let now = perf_now();+ if now - self.last_save >= SAVE_EVERY_MS {+ self.last_save = now;+ self.app.session.save_view();+ if let Ok(ui) = serde_json::to_vec_pretty(&self.app.ui)+ && ui != self.ui_written+ {+ self.files.write("ui.json", &ui);+ self.ui_written = ui;+ }+ }+ if let Some(b) = &self.backend+ && self.files.is_dirty()+ && !self.flushing.get()+ {+ self.flushing.set(true);+ wasm_bindgen_futures::spawn_local(flush(self.files.clone(), b.clone(), self.flushing.clone()));+ }+ }+}++type Reply = (js_sys::Function, js_sys::Function);++thread_local! {+ /// Commands from JavaScript ([`command`]), run on the next frame.+ static INBOX: std::cell::RefCell<Vec<(String, String, Reply)>> = const { std::cell::RefCell::new(Vec::new()) };+ static CTX: std::cell::RefCell<Option<egui::Context>> = const { std::cell::RefCell::new(None) };+}++/// Run a command by id from JavaScript (automation, tests): `await command("library.info", "{}")`+/// resolves to the result as JSON text, or rejects with the error. Besides every engine/UI+/// command, `web.stats` reports the browser host's state (storage, originals in memory).+#[wasm_bindgen]+pub fn command(id: String, params: String) -> js_sys::Promise {+ js_sys::Promise::new(&mut |resolve, reject| {+ INBOX.with(|q| q.borrow_mut().push((id.clone(), params.clone(), (resolve, reject))));+ CTX.with(|c| c.borrow().as_ref().map(|c| c.request_repaint()));+ })+}++impl WebApp {+ fn web_stats(&self) -> serde_json::Value {+ let (orig_n, orig_bytes) = self.originals.usage();+ json!({+ "storage": self.backend.as_ref().map_or("memory", |b| b.kind()),+ "dirty": self.files.is_dirty(),+ "flushing": self.flushing.get(),+ "originalsInMemory": orig_n,+ "originalBytesInMemory": orig_bytes,+ "renderQueue": self.app.renderer.queued(),+ "rendersInFlight": self.app.renderer.in_flight(),+ "thumbTextures": self.app.renderer.thumb_textures(),+ })+ }++ fn run_inbox(&mut self) {+ let inbox = INBOX.with(|q| std::mem::take(&mut *q.borrow_mut()));+ for (id, params, (resolve, reject)) in inbox {+ let params: serde_json::Value = serde_json::from_str(¶ms).unwrap_or(json!({}));+ let r = if id == "web.stats" { Ok(self.web_stats()) } else { self.app.run(&id, params) };+ let _ = match r {+ Ok(v) => resolve.call1(&JsValue::NULL, &v.to_string().into()),+ Err(e) => reject.call1(&JsValue::NULL, &e.into()),+ };+ }+ }}impl eframe::App for WebApp {fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {+ self.run_inbox();self.import_dropped(ctx);+ self.load_originals(ctx);self.app.logic(ctx);+ self.save();if let Some(b) = self.bench.as_mut()&& let Some(report) = b.step(&mut self.app){@@ -176,11 +437,12 @@ impl eframe::App for WebApp {}}-#[wasm_bindgen(start)]+/// Start the app (called by `index.html` after instantiating the module).+#[wasm_bindgen]pub fn start() {eframe::WebLogger::init(log::LevelFilter::Info).ok();log::info!("lightcraft: wasm instantiated at {:.0} ms", perf_now());- let bench = window().location().search().is_ok_and(|q| q.contains("bench"));+ let opts = Options::from_url();wasm_bindgen_futures::spawn_local(async move {let Some(canvas) =window().document().and_then(|d| d.get_element_by_id("lightcraft_canvas")).and_then(|e| e.dyn_into::<web_sys::HtmlCanvasElement>().ok())@@ -188,8 +450,9 @@ pub fn start() {log::error!("missing <canvas id=\"lightcraft_canvas\">");return;};+ let boot = boot(opts).await;let runner = eframe::WebRunner::new();- let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, bench))))).await;+ let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot))))).await;match r {Ok(()) => {if let Some(el) = window().document().and_then(|d| d.get_element_by_id("lightcraft_loading")) {diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rsindex 3aac02034..e255b0e00 100644--- a/crates/engine/src/import.rs+++ b/crates/engine/src/import.rs@@ -152,7 +152,10 @@ fn copy_into_library(lib: &Path, src: &str, date: &str) -> Result<String, String/// Import files/folders. See the module docs.pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result<ImportReport> {- let lib_dir = s.library.as_ref().map(|l| l.dir.clone());+ // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of every+ // added file in its own storage already, so "copy" there means "add".+ let mode = if mode == ImportMode::Copy && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };+ let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());if mode == ImportMode::Copy && lib_dir.is_none() {return Err(crate::EngineError::Other("copying into the library needs an open library".into()));}diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rsindex e9979354c..d0fd0ac3b 100644--- a/crates/engine/src/library.rs+++ b/crates/engine/src/library.rs@@ -10,6 +10,9 @@//! Originals/ (photos imported with "copy into library")//! ```//!+//! The files live in [`Store`]s: a directory ([`FsStore`]) natively, or any other implementation+//! via [`Session::open_library_in`] (the browser build keeps them in OPFS or IndexedDB).+//!//! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning,//! so a crash loses at most the command in flight. The log is compacted into a snapshot when it//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`].@@ -36,13 +39,32 @@ pub fn default_dir() -> Option<PathBuf> {}pub struct Library {+ /// The library directory, or a descriptive pseudo-path when the library isn't on disk.pub dir: PathBuf,+ /// The library is a real directory: photos can be copied into `Originals/` and thumbnails are+ /// cached in `thumbs/`.+ pub on_disk: bool,journal: Journal,+ /// presets.json, view.json, prefs.json.+ files: Box<dyn Store>,/// What happened when the catalog was loaded.pub report: LoadReport,/// Last persistence error (shown by the UI; ops stay pending and are retried).pub last_error: Option<String>,presets_written: String,+ view_written: Vec<u8>,+}++/// Where a library's files live (see [`Session::open_library_in`]).+pub struct LibraryStores {+ /// Shown as the library's location (`library.info`).+ pub dir: PathBuf,+ /// The catalog journal (`catalog.snap`, `catalog.log`).+ pub catalog: Box<dyn Store>,+ /// presets.json, view.json, prefs.json.+ pub files: Box<dyn Store>,+ /// `dir` is a real directory (enables `Originals/` copies and the `thumbs/` disk cache).+ pub on_disk: bool,}#[derive(Default, Serialize, Deserialize)]@@ -87,8 +109,8 @@ fn presets_json(presets: &[Preset]) -> String {serde_json::to_string_pretty(&f).unwrap_or_default()}-fn write_atomic(dir: &Path, name: &str, data: &[u8]) -> std::io::Result<()> {- FsStore::open(dir)?.write_atomic(name, data)+fn read_json<T: serde::de::DeserializeOwned>(store: &mut dyn Store, name: &str) -> Option<T> {+ store.read(name).ok().flatten().and_then(|b| serde_json::from_slice(&b).ok())}impl Session {@@ -96,8 +118,16 @@ impl Session {/// `seed_demo`, a newly created library starts with the procedural demo photos.pub fn open_library(&mut self, dir: impl AsRef<Path>, seed_demo: bool) -> Result<&LoadReport> {let dir = dir.as_ref().to_path_buf();- let store = FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())))?;- let (mut journal, catalog, report) = Journal::open(Box::new(store))?;+ let open = || FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())));+ let stores = LibraryStores { catalog: Box::new(open()?), files: Box::new(open()?), on_disk: true, dir: dir.clone() };+ self.open_library_in(stores, seed_demo)+ }++ /// Open (or create) a library whose files live in `stores` (e.g. browser storage), replacing+ /// this session's catalog.+ pub fn open_library_in(&mut self, stores: LibraryStores, seed_demo: bool) -> Result<&LoadReport> {+ let LibraryStores { dir, catalog, mut files, on_disk } = stores;+ let (mut journal, catalog, report) = Journal::open(catalog)?;self.catalog = catalog;self.undo.clear();self.redo.clear();@@ -110,9 +140,7 @@ impl Session {journal.snapshot(&self.catalog)?;}// presets- if let Ok(bytes) = std::fs::read(dir.join("presets.json"))- && let Ok(f) = serde_json::from_slice::<PresetsFile>(&bytes)- {+ if let Some(f) = read_json::<PresetsFile>(files.as_mut(), "presets.json") {for p in &mut self.presets {p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite);}@@ -123,12 +151,9 @@ impl Session {}}// preferences- self.xmp =- std::fs::read(dir.join("prefs.json")).ok().and_then(|b| serde_json::from_slice::<PrefsFile>(&b).ok()).map(|p| p.xmp).unwrap_or_default();+ self.xmp = read_json::<PrefsFile>(files.as_mut(), "prefs.json").map(|p| p.xmp).unwrap_or_default();// view state- if let Ok(bytes) = std::fs::read(dir.join("view.json"))- && let Ok(v) = serde_json::from_slice::<ViewFile>(&bytes)- {+ if let Some(v) = read_json::<ViewFile>(files.as_mut(), "view.json") {self.source = v.source;self.filter = v.filter;self.sort = v.sort;@@ -142,8 +167,11 @@ impl Session {self.selection = Selection::single(*first);}let presets_written = presets_json(&self.presets);- self.media.attach_disk_cache(&dir.join("thumbs"));- self.library = Some(Library { dir, journal, report, last_error: None, presets_written });+ if on_disk {+ self.media.attach_disk_cache(&dir.join("thumbs"));+ }+ let view_written = self.view_json();+ self.library = Some(Library { dir, on_disk, journal, files, report, last_error: None, presets_written, view_written });Ok(&self.library.as_ref().expect("just set").report)}@@ -166,7 +194,7 @@ impl Session {}let presets = presets_json(&self.presets);if presets != lib.presets_written {- if let Err(e) = write_atomic(&lib.dir, "presets.json", presets.as_bytes()) {+ if let Err(e) = lib.files.write_atomic("presets.json", presets.as_bytes()) {log::error!("library: presets: {e}");} else {lib.presets_written = presets;@@ -190,20 +218,39 @@ impl Session {}let _ = self.end_interaction();self.persist()?;- let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };+ self.save_view();let Some(lib) = self.library.as_mut() else { return Ok(()) };lib.journal.snapshot(&self.catalog)?;- if let Ok(v) = serde_json::to_vec_pretty(&view) {- let _ = write_atomic(&lib.dir, "view.json", &v);- }Ok(())}+ fn view_json(&self) -> Vec<u8> {+ let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };+ serde_json::to_vec_pretty(&view).unwrap_or_default()+ }++ /// Save the view state (source, filter, sort, selection) if it changed since it was last+ /// written. Native hosts get this from [`Session::close_library`]; the browser host calls it+ /// periodically, since a tab can be closed without notice.+ pub fn save_view(&mut self) {+ if self.library.is_none() {+ return;+ }+ let v = self.view_json();+ let Some(lib) = self.library.as_mut() else { return };+ if v != lib.view_written {+ match lib.files.write_atomic("view.json", &v) {+ Ok(()) => lib.view_written = v,+ Err(e) => log::error!("library: view: {e}"),+ }+ }+ }+/// Save the library preferences (no-op for in-memory sessions).- pub fn save_prefs(&self) -> Result<()> {- let Some(lib) = &self.library else { return Ok(()) };+ pub fn save_prefs(&mut self) -> Result<()> {let v = serde_json::to_vec_pretty(&PrefsFile { xmp: self.xmp }).unwrap_or_default();- write_atomic(&lib.dir, "prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))+ let Some(lib) = self.library.as_mut() else { return Ok(()) };+ lib.files.write_atomic("prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))}/// Compact the log into a snapshot now.