Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

M0.12: web persistence — library, edits and imported originals survive a reload

Brandon Thomasbaa30ca59025
diff --git a/Cargo.lock b/Cargo.lock
index 8ff6fc3ef..20ae00caf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2229,9 +2229,11 @@ dependencies = [
"js-sys",
"lightcraft-codecs",
"lightcraft-engine",
+ "lightcraft-preview",
"lightcraft-raster",
"lightcraft-ui-egui",
"log",
+ "serde",
"serde_json",
"wasm-bindgen",
"wasm-bindgen-futures",
diff --git a/apps/lightcraft-web/Cargo.toml b/apps/lightcraft-web/Cargo.toml
index b9c7bdf23..1f9f14c13 100644
--- a/apps/lightcraft-web/Cargo.toml
+++ b/apps/lightcraft-web/Cargo.toml
@@ -17,6 +17,8 @@ lightcraft-engine = { workspace = true }
lightcraft-ui-egui = { workspace = true }
lightcraft-codecs = { workspace = true }
lightcraft-raster = { workspace = true }
+lightcraft-preview = { workspace = true }
+serde = { workspace = true }
serde_json = { workspace = true }
log = { workspace = true }
@@ -29,16 +31,42 @@ js-sys = { workspace = true }
web-sys = { workspace = true, features = [
"Blob",
"BlobPropertyBag",
+ "DedicatedWorkerGlobalScope",
"Document",
+ "DomStringList",
"Element",
+ "EventTarget",
"File",
"FileList",
+ "FileSystemCreateWritableOptions",
+ "FileSystemDirectoryHandle",
+ "FileSystemFileHandle",
+ "FileSystemGetDirectoryOptions",
+ "FileSystemGetFileOptions",
+ "FileSystemRemoveOptions",
+ "FileSystemWritableFileStream",
"HtmlAnchorElement",
"HtmlCanvasElement",
"HtmlElement",
"HtmlInputElement",
+ "IdbDatabase",
+ "IdbFactory",
+ "IdbKeyRange",
+ "IdbObjectStore",
+ "IdbOpenDbRequest",
+ "IdbRequest",
+ "IdbTransaction",
+ "IdbTransactionMode",
+ "IdbVersionChangeEvent",
"Location",
+ "MessageEvent",
+ "Navigator",
"Performance",
+ "StorageManager",
"Url",
"Window",
+ "Worker",
+ "WorkerOptions",
+ "WorkerType",
+ "WritableStream",
] }
diff --git a/apps/lightcraft-web/index.html b/apps/lightcraft-web/index.html
index ac1ed167a..bc0ad2037 100644
--- a/apps/lightcraft-web/index.html
+++ b/apps/lightcraft-web/index.html
@@ -22,8 +22,16 @@
<div id="lightcraft_loading">Loading LightCraft…</div>
<div id="lightcraft_status" aria-live="polite"></div>
<script type="module">
- import init from "./lightcraft_web.js";
- init().catch((e) => {
+ import init, { start, command } from "./lightcraft_web.js";
+ (async () => {
+ // Compile once; the render workers (worker.js) instantiate the same compiled module.
+ const module = await WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm"));
+ globalThis.lightcraftModule = module;
+ await init({ module_or_path: module });
+ // automation: `await lightcraft.command("library.info", "{}")` → JSON text
+ globalThis.lightcraft = { command };
+ start();
+ })().catch((e) => {
console.error(e);
const el = document.getElementById("lightcraft_loading");
if (el) el.textContent = "LightCraft failed to load: " + e;
diff --git a/apps/lightcraft-web/src/backend.rs b/apps/lightcraft-web/src/backend.rs
new file mode 100644
index 000000000..50a03135f
--- /dev/null
+++ b/apps/lightcraft-web/src/backend.rs
@@ -0,0 +1,324 @@
+//! Browser storage, asynchronous: OPFS (Origin Private File System) when the page can write it
+//! from the main thread (`FileSystemFileHandle.createWritable`), else IndexedDB. Used from the
+//! window and from render workers alike. Keys are `/`-separated paths (OPFS directories; plain
+//! IndexedDB keys).
+
+use js_sys::{Reflect, Uint8Array};
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::{JsCast, JsValue};
+use wasm_bindgen_futures::JsFuture;
+use web_sys::{FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, IdbDatabase};
+
+const IDB_NAME: &str = "lightcraft";
+const IDB_STORE: &str = "files";
+
+#[derive(Clone)]
+pub enum Backend {
+ Opfs(FileSystemDirectoryHandle),
+ Idb(IdbDatabase),
+}
+
+fn err(e: JsValue) -> String {
+ e.as_string().or_else(|| e.dyn_ref::<js_sys::Error>().map(|e| String::from(e.message()))).unwrap_or_else(|| format!("{e:?}"))
+}
+
+/// `navigator.storage` in a window or a worker.
+fn storage_manager() -> Option<web_sys::StorageManager> {
+ let g = js_sys::global();
+ let nav = Reflect::get(&g, &"navigator".into()).ok()?;
+ let storage = Reflect::get(&nav, &"storage".into()).ok()?;
+ (!storage.is_undefined()).then(|| storage.unchecked_into())
+}
+
+fn indexed_db() -> Option<web_sys::IdbFactory> {
+ let f = Reflect::get(&js_sys::global(), &"indexedDB".into()).ok()?;
+ (!f.is_undefined() && !f.is_null()).then(|| f.unchecked_into())
+}
+
+/// Can files be written through OPFS writable streams here?
+fn opfs_writable() -> bool {
+ let g = js_sys::global();
+ Reflect::get(&g, &"FileSystemFileHandle".into())
+ .ok()
+ .filter(|c| !c.is_undefined())
+ .and_then(|c| Reflect::get(&c, &"prototype".into()).ok())
+ .and_then(|p| Reflect::get(&p, &"createWritable".into()).ok())
+ .is_some_and(|f| f.is_function())
+}
+
+/// Resolve an IndexedDB request.
+async fn idb_req(req: &web_sys::IdbRequest) -> Result<JsValue, String> {
+ let r1 = req.clone();
+ let p = js_sys::Promise::new(&mut |resolve, reject| {
+ let r2 = r1.clone();
+ let ok = Closure::once_into_js(move || {
+ let _ = resolve.call1(&JsValue::NULL, &r2.result().unwrap_or(JsValue::UNDEFINED));
+ });
+ let fail = Closure::once_into_js(move |e: JsValue| {
+ let _ = reject.call1(&JsValue::NULL, &e);
+ });
+ r1.set_onsuccess(Some(ok.unchecked_ref()));
+ r1.set_onerror(Some(fail.unchecked_ref()));
+ });
+ JsFuture::from(p).await.map_err(err)
+}
+
+/// Wait for an IndexedDB transaction to commit.
+async fn idb_done(tx: &web_sys::IdbTransaction) -> Result<(), String> {
+ let t1 = tx.clone();
+ let p = js_sys::Promise::new(&mut |resolve, reject| {
+ let ok = Closure::once_into_js(move || {
+ let _ = resolve.call0(&JsValue::NULL);
+ });
+ let rej = reject.clone();
+ let fail = Closure::once_into_js(move |e: JsValue| {
+ let _ = rej.call1(&JsValue::NULL, &e);
+ });
+ let abort = Closure::once_into_js(move |e: JsValue| {
+ let _ = reject.call1(&JsValue::NULL, &e);
+ });
+ t1.set_oncomplete(Some(ok.unchecked_ref()));
+ t1.set_onerror(Some(fail.unchecked_ref()));
+ t1.set_onabort(Some(abort.unchecked_ref()));
+ });
+ JsFuture::from(p).await.map(|_| ()).map_err(err)
+}
+
+impl Backend {
+ /// OPFS if writable from here (unless `prefer_idb`), else IndexedDB.
+ pub async fn open(prefer_idb: bool) -> Result<Backend, String> {
+ if !prefer_idb
+ && opfs_writable()
+ && let Some(sm) = storage_manager()
+ {
+ match JsFuture::from(sm.get_directory()).await {
+ Ok(root) => return Ok(Backend::Opfs(root.unchecked_into())),
+ Err(e) => log::warn!("OPFS unavailable ({}); using IndexedDB", err(e)),
+ }
+ }
+ Self::open_idb().await
+ }
+
+ async fn open_idb() -> Result<Backend, String> {
+ let f = indexed_db().ok_or("no IndexedDB")?;
+ let req = f.open_with_u32(IDB_NAME, 1).map_err(err)?;
+ let upgrade = Closure::<dyn FnMut(web_sys::IdbVersionChangeEvent)>::new(move |e: web_sys::IdbVersionChangeEvent| {
+ if let Some(db) = e.target().and_then(|t| t.dyn_into::<web_sys::IdbOpenDbRequest>().ok()).and_then(|r| r.result().ok()) {
+ let db: IdbDatabase = db.unchecked_into();
+ if !db.object_store_names().contains(IDB_STORE) {
+ let _ = db.create_object_store(IDB_STORE);
+ }
+ }
+ });
+ req.set_onupgradeneeded(Some(upgrade.as_ref().unchecked_ref()));
+ let db = idb_req(&req).await?;
+ drop(upgrade);
+ Ok(Backend::Idb(db.unchecked_into()))
+ }
+
+ /// "opfs" or "idb" (workers open the same kind).
+ pub fn kind(&self) -> &'static str {
+ match self {
+ Backend::Opfs(_) => "opfs",
+ Backend::Idb(_) => "idb",
+ }
+ }
+
+ /// The directory holding `path`'s last component (created when `create`).
+ async fn parent(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<(FileSystemDirectoryHandle, String)>, String> {
+ let mut parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
+ let name = parts.pop().ok_or("empty path")?.to_string();
+ let mut dir = root.clone();
+ for p in parts {
+ let opts = FileSystemGetDirectoryOptions::new();
+ opts.set_create(create);
+ match JsFuture::from(dir.get_directory_handle_with_options(p, &opts)).await {
+ Ok(d) => dir = d.unchecked_into(),
+ Err(e) if !create => {
+ let _ = e;
+ return Ok(None);
+ }
+ Err(e) => return Err(err(e)),
+ }
+ }
+ Ok(Some((dir, name)))
+ }
+
+ async fn opfs_file(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<FileSystemFileHandle>, String> {
+ let Some((dir, name)) = Self::parent(root, path, create).await? else { return Ok(None) };
+ let opts = FileSystemGetFileOptions::new();
+ opts.set_create(create);
+ match JsFuture::from(dir.get_file_handle_with_options(&name, &opts)).await {
+ Ok(h) => Ok(Some(h.unchecked_into())),
+ Err(_) if !create => Ok(None),
+ Err(e) => Err(err(e)),
+ }
+ }
+
+ fn idb_store(db: &IdbDatabase, write: bool) -> Result<(web_sys::IdbTransaction, web_sys::IdbObjectStore), String> {
+ let mode = if write { web_sys::IdbTransactionMode::Readwrite } else { web_sys::IdbTransactionMode::Readonly };
+ let tx = db.transaction_with_str_and_mode(IDB_STORE, mode).map_err(err)?;
+ let store = tx.object_store(IDB_STORE).map_err(err)?;
+ Ok((tx, store))
+ }
+
+ /// A whole file, or `None` if it doesn't exist.
+ pub async fn read(&self, path: &str) -> Result<Option<Vec<u8>>, String> {
+ match self {
+ Backend::Opfs(root) => {
+ let Some(h) = Self::opfs_file(root, path, false).await? else { return Ok(None) };
+ let file: web_sys::File = JsFuture::from(h.get_file()).await.map_err(err)?.unchecked_into();
+ let buf = JsFuture::from(file.array_buffer()).await.map_err(err)?;
+ Ok(Some(Uint8Array::new(&buf).to_vec()))
+ }
+ Backend::Idb(db) => {
+ let (_tx, store) = Self::idb_store(db, false)?;
+ let v = idb_req(&store.get(&path.into()).map_err(err)?).await?;
+ if v.is_undefined() || v.is_null() {
+ return Ok(None);
+ }
+ Ok(Some(Uint8Array::new(&v).to_vec()))
+ }
+ }
+ }
+
+ /// Replace a file atomically (OPFS: the writable's swap file is committed on close).
+ pub async fn write(&self, path: &str, data: &[u8]) -> Result<(), String> {
+ self.write_at(path, None, data).await
+ }
+
+ /// Write `data` at `offset` keeping the bytes before it (`None`: replace the file). Atomic.
+ pub async fn write_at(&self, path: &str, offset: Option<usize>, data: &[u8]) -> Result<(), String> {
+ match self {
+ Backend::Opfs(root) => {
+ let h = Self::opfs_file(root, path, true).await?.ok_or("can't create file")?;
+ let opts = web_sys::FileSystemCreateWritableOptions::new();
+ opts.set_keep_existing_data(offset.is_some());
+ let w: web_sys::FileSystemWritableFileStream =
+ JsFuture::from(h.create_writable_with_options(&opts)).await.map_err(err)?.unchecked_into();
+ let res = async {
+ if let Some(off) = offset {
+ JsFuture::from(w.truncate_with_f64(off as f64).map_err(err)?).await.map_err(err)?;
+ JsFuture::from(w.seek_with_f64(off as f64).map_err(err)?).await.map_err(err)?;
+ }
+ JsFuture::from(w.write_with_js_u8_array(&Uint8Array::from(data)).map_err(err)?).await.map_err(err)?;
+ Ok::<(), String>(())
+ }
+ .await;
+ match res {
+ Ok(()) => JsFuture::from(w.close()).await.map(|_| ()).map_err(err),
+ Err(e) => {
+ let _ = JsFuture::from(w.abort()).await;
+ Err(e)
+ }
+ }
+ }
+ Backend::Idb(db) => {
+ let (tx, store) = Self::idb_store(db, true)?;
+ let value = match offset {
+ // read-modify-write inside one transaction
+ Some(off) => {
+ let old = idb_req(&store.get(&path.into()).map_err(err)?).await?;
+ let mut v = if old.is_undefined() || old.is_null() { Vec::new() } else { Uint8Array::new(&old).to_vec() };
+ v.truncate(off);
+ v.extend_from_slice(data);
+ Uint8Array::from(&v[..])
+ }
+ None => Uint8Array::from(data),
+ };
+ store.put_with_key(&value, &path.into()).map_err(err)?;
+ idb_done(&tx).await
+ }
+ }
+ }
+
+ #[allow(dead_code)] // thumbnail cache pruning (render workers)
+ pub async fn remove(&self, path: &str) -> Result<(), String> {
+ match self {
+ Backend::Opfs(root) => {
+ let Some((dir, name)) = Self::parent(root, path, false).await? else { return Ok(()) };
+ match JsFuture::from(dir.remove_entry(&name)).await {
+ Ok(_) => Ok(()),
+ Err(e) if Reflect::get(&e, &"name".into()).ok().and_then(|n| n.as_string()).as_deref() == Some("NotFoundError") => Ok(()),
+ Err(e) => Err(err(e)),
+ }
+ }
+ Backend::Idb(db) => {
+ let (tx, store) = Self::idb_store(db, true)?;
+ store.delete(&path.into()).map_err(err)?;
+ idb_done(&tx).await
+ }
+ }
+ }
+
+ /// Names of the files directly inside `dir`.
+ #[allow(dead_code)] // thumbnail cache pruning (render workers)
+ pub async fn list(&self, dir: &str) -> Result<Vec<String>, String> {
+ match self {
+ Backend::Opfs(root) => {
+ // `parent` of a child path is `dir` itself
+ let Some((handle, _)) = Self::parent(root, &format!("{dir}/-"), false).await? else { return Ok(Vec::new()) };
+ dir_keys(&handle).await
+ }
+ Backend::Idb(db) => {
+ let (_tx, store) = Self::idb_store(db, false)?;
+ let prefix = format!("{dir}/");
+ let range = web_sys::IdbKeyRange::bound(&prefix.as_str().into(), &format!("{prefix}\u{ffff}").into()).map_err(err)?;
+ let keys = idb_req(&store.get_all_keys_with_key(&range).map_err(err)?).await?;
+ let arr: js_sys::Array = keys.unchecked_into();
+ Ok(arr
+ .iter()
+ .filter_map(|k| k.as_string())
+ .filter_map(|k| k.strip_prefix(&prefix).filter(|r| !r.contains('/')).map(str::to_string))
+ .collect())
+ }
+ }
+ }
+
+ /// Delete everything (the "reset" escape hatch: `?reset` in the URL).
+ pub async fn clear(&self) -> Result<(), String> {
+ match self {
+ Backend::Opfs(root) => {
+ for name in dir_keys(root).await? {
+ let opts = web_sys::FileSystemRemoveOptions::new();
+ opts.set_recursive(true);
+ JsFuture::from(root.remove_entry_with_options(&name, &opts)).await.map_err(err)?;
+ }
+ Ok(())
+ }
+ Backend::Idb(db) => {
+ let (tx, store) = Self::idb_store(db, true)?;
+ store.clear().map_err(err)?;
+ idb_done(&tx).await
+ }
+ }
+ }
+}
+
+/// Names of the entries in an OPFS directory.
+async fn dir_keys(dir: &FileSystemDirectoryHandle) -> Result<Vec<String>, String> {
+ let keys = dir.keys();
+ let mut out = Vec::new();
+ loop {
+ let next = JsFuture::from(keys.next().map_err(err)?).await.map_err(err)?;
+ if Reflect::get(&next, &"done".into()).ok().is_some_and(|d| d.is_truthy()) {
+ break;
+ }
+ if let Some(k) = Reflect::get(&next, &"value".into()).ok().and_then(|v| v.as_string()) {
+ out.push(k);
+ }
+ }
+ Ok(out)
+}
+
+/// Ask the browser not to evict our storage under pressure (best effort; fire and forget).
+pub fn request_persistence() {
+ if let Some(sm) = storage_manager()
+ && let Ok(p) = sm.persist()
+ {
+ wasm_bindgen_futures::spawn_local(async move {
+ let granted = JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy());
+ log::info!("lightcraft: persistent storage {}", if granted { "granted" } else { "not granted (best effort)" });
+ });
+ }
+}
diff --git a/apps/lightcraft-web/src/bench.rs b/apps/lightcraft-web/src/bench.rs
index 7ede4a7ad..90b85af84 100644
--- a/apps/lightcraft-web/src/bench.rs
+++ b/apps/lightcraft-web/src/bench.rs
@@ -65,7 +65,7 @@ impl Bench {
if self.first_frame_ms.is_none() && self.frames > 1 {
self.first_frame_ms = Some(now - self.start_ms);
}
- let idle = app.renderer.queued() == 0;
+ let idle = app.renderer.queued() == 0 && app.renderer.in_flight() == 0;
match self.phase {
Phase::Idle => {
let thumbs = app.renderer.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count();
diff --git a/apps/lightcraft-web/src/files.rs b/apps/lightcraft-web/src/files.rs
new file mode 100644
index 000000000..18acc93d3
--- /dev/null
+++ b/apps/lightcraft-web/src/files.rs
@@ -0,0 +1,255 @@
+//! The library's small files (catalog journal, presets, view state, prefs) on the web.
+//!
+//! Browser storage is asynchronous on the main thread (OPFS writable streams, IndexedDB), but the
+//! catalog [`Store`] is synchronous. So the files are mirrored in memory: they are read once at
+//! start-up ([`Files::preload`]), every write changes the mirror immediately and marks the file
+//! dirty, and the host flushes dirty files to storage in the background ([`Files::take_dirty`],
+//! about once per frame while anything is dirty).
+//!
+//! Durability: a write is in browser storage a few milliseconds after the command that made it
+//! (one flush), not before the command returns as on the desktop. Each flushed file is replaced
+//! atomically (OPFS swap file / one IndexedDB transaction), and files are flushed in the order they
+//! were last modified, so the journal's recovery rules still hold: e.g. a new `catalog.snap` always
+//! lands before the `catalog.log` reset that follows it.
+
+use std::collections::BTreeMap;
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use lightcraft_engine::catalog::Store;
+
+/// What has to be written for a dirty file.
+#[derive(Clone, Debug, PartialEq)]
+pub enum FlushOp {
+ /// Replace the whole file.
+ Write { name: String, data: Vec<u8> },
+ /// Bytes appended since the last flush, starting at `offset` (the stored file's length).
+ Append { name: String, offset: usize, data: Vec<u8> },
+}
+
+impl FlushOp {
+ pub fn name(&self) -> &str {
+ match self {
+ FlushOp::Write { name, .. } | FlushOp::Append { name, .. } => name,
+ }
+ }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum Dirty {
+ Rewrite,
+ /// Only appended since the last flush; the stored file has `from` bytes.
+ Append {
+ from: usize,
+ },
+}
+
+#[derive(Default)]
+struct Mirror {
+ files: BTreeMap<String, Vec<u8>>,
+ /// Dirty files, least recently modified first.
+ dirty: Vec<(String, Dirty)>,
+}
+
+impl Mirror {
+ fn mark(&mut self, name: &str, d: Dirty) {
+ let prev = self.dirty.iter().position(|(n, _)| n == name).map(|i| self.dirty.remove(i).1);
+ let d = match (prev, d) {
+ // appends after appends keep the original offset; anything else needs a rewrite
+ (Some(Dirty::Append { from }), Dirty::Append { .. }) => Dirty::Append { from },
+ (Some(Dirty::Rewrite), _) => Dirty::Rewrite,
+ (_, d) => d,
+ };
+ self.dirty.push((name.to_string(), d));
+ }
+}
+
+/// Shared in-memory mirror of the library files (cheap to clone).
+#[derive(Clone, Default)]
+pub struct Files(Arc<Mutex<Mirror>>);
+
+impl Files {
+ fn lock(&self) -> std::sync::MutexGuard<'_, Mirror> {
+ self.0.lock().unwrap_or_else(|e| e.into_inner())
+ }
+
+ /// Set a file's stored content (start-up; not dirty).
+ pub fn preload(&self, name: &str, data: Vec<u8>) {
+ self.lock().files.insert(name.to_string(), data);
+ }
+
+ pub fn get(&self, name: &str) -> Option<Vec<u8>> {
+ self.lock().files.get(name).cloned()
+ }
+
+ /// Replace a file.
+ pub fn write(&self, name: &str, data: &[u8]) {
+ let mut m = self.lock();
+ m.files.insert(name.to_string(), data.to_vec());
+ m.mark(name, Dirty::Rewrite);
+ }
+
+ pub fn append(&self, name: &str, data: &[u8]) {
+ let mut m = self.lock();
+ let f = m.files.entry(name.to_string()).or_default();
+ let from = f.len();
+ f.extend_from_slice(data);
+ m.mark(name, Dirty::Append { from });
+ }
+
+ pub fn truncate(&self, name: &str, len: usize) {
+ let mut m = self.lock();
+ if let Some(f) = m.files.get_mut(name) {
+ f.truncate(len);
+ m.mark(name, Dirty::Rewrite);
+ }
+ }
+
+ pub fn is_dirty(&self) -> bool {
+ !self.lock().dirty.is_empty()
+ }
+
+ /// Everything to write, in modification order; the files count as clean afterwards (call
+ /// [`Files::failed`] for ops that couldn't be written).
+ pub fn take_dirty(&self) -> Vec<FlushOp> {
+ let mut m = self.lock();
+ let dirty = std::mem::take(&mut m.dirty);
+ dirty
+ .into_iter()
+ .map(|(name, d)| {
+ let data = m.files.get(&name).cloned().unwrap_or_default();
+ match d {
+ Dirty::Append { from } if from <= data.len() => FlushOp::Append { offset: from, data: data[from..].to_vec(), name },
+ _ => FlushOp::Write { name, data },
+ }
+ })
+ .collect()
+ }
+
+ /// A flush failed: write these files in full next time (keeps any newer modification order).
+ pub fn failed(&self, ops: &[FlushOp]) {
+ let mut m = self.lock();
+ for op in ops.iter().rev() {
+ if !m.dirty.iter().any(|(n, _)| n == op.name()) {
+ m.dirty.insert(0, (op.name().to_string(), Dirty::Rewrite));
+ } else if let Some(d) = m.dirty.iter_mut().find(|(n, _)| n == op.name()) {
+ d.1 = Dirty::Rewrite;
+ }
+ }
+ }
+
+ /// A catalog [`Store`] over these files.
+ pub fn store(&self) -> FilesStore {
+ FilesStore(self.clone())
+ }
+}
+
+/// [`Store`] implementation over [`Files`].
+pub struct FilesStore(Files);
+
+impl Store for FilesStore {
+ fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {
+ Ok(self.0.get(name))
+ }
+ fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
+ self.0.write(name, data);
+ Ok(())
+ }
+ fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
+ self.0.append(name, data);
+ Ok(())
+ }
+ fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {
+ self.0.truncate(name, len as usize);
+ Ok(())
+ }
+ fn describe(&self) -> String {
+ "browser storage".into()
+ }
+}
+
+/// Names of the library files read at start-up.
+pub const LIBRARY_FILES: [&str; 6] = ["catalog.snap", "catalog.log", "presets.json", "view.json", "prefs.json", "ui.json"];
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use lightcraft_engine::Session;
+ use lightcraft_engine::library::LibraryStores;
+ use serde_json::json;
+
+ /// Apply flush ops to a "disk" map, as the backend would.
+ fn flush(files: &Files, disk: &mut BTreeMap<String, Vec<u8>>) {
+ for op in files.take_dirty() {
+ match op {
+ FlushOp::Write { name, data } => {
+ disk.insert(name, data);
+ }
+ FlushOp::Append { name, offset, data } => {
+ let f = disk.entry(name).or_default();
+ assert_eq!(f.len(), offset, "append offset matches the stored length");
+ f.extend_from_slice(&data);
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn appends_coalesce_and_rewrites_win() {
+ let f = Files::default();
+ f.preload("log", b"ab".to_vec());
+ f.append("log", b"c");
+ f.append("log", b"d");
+ assert_eq!(f.take_dirty(), vec![FlushOp::Append { name: "log".into(), offset: 2, data: b"cd".to_vec() }]);
+ assert!(!f.is_dirty());
+ f.write("snap", b"S");
+ f.truncate("log", 1);
+ f.append("log", b"x");
+ let ops = f.take_dirty();
+ assert_eq!(
+ ops,
+ vec![FlushOp::Write { name: "snap".into(), data: b"S".to_vec() }, FlushOp::Write { name: "log".into(), data: b"ax".to_vec() }]
+ );
+ f.failed(&ops);
+ assert_eq!(f.take_dirty().len(), 2);
+ }
+
+ #[test]
+ fn modification_order_puts_snapshot_before_log_reset() {
+ let f = Files::default();
+ f.append("catalog.log", b"1\n");
+ f.write("catalog.snap", b"snap");
+ f.write("catalog.log", b"");
+ let names: Vec<_> = f.take_dirty().iter().map(|o| o.name().to_string()).collect();
+ assert_eq!(names, ["catalog.snap", "catalog.log"]);
+ }
+
+ #[test]
+ fn library_survives_reload_through_flushes() {
+ let mut disk = BTreeMap::new();
+ let open = |disk: &BTreeMap<String, Vec<u8>>| {
+ let files = Files::default();
+ for (k, v) in disk {
+ files.preload(k, v.clone());
+ }
+ let mut s = Session::new();
+ let stores = LibraryStores { dir: "browser".into(), catalog: Box::new(files.store()), files: Box::new(files.store()), on_disk: false };
+ s.open_library_in(stores, true).unwrap();
+ (s, files)
+ };
+ let (mut s, files) = open(&disk);
+ assert!(!s.catalog.is_empty(), "seeded with the demo photos");
+ flush(&files, &mut disk);
+ let id = s.selection.active.unwrap();
+ s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.25})).unwrap();
+ s.execute("photo.rate", &json!({"rating": 4})).unwrap();
+ s.save_view();
+ flush(&files, &mut disk);
+ let (s2, _) = open(&disk);
+ let p = s2.catalog.photo(id).unwrap();
+ assert_eq!(p.rating, 4);
+ assert_eq!(p.develop.light.exposure, 1.25);
+ assert_eq!(s2.catalog.len(), s.catalog.len(), "not seeded twice");
+ assert_eq!(s2.selection.active, s.selection.active);
+ }
+}
diff --git a/apps/lightcraft-web/src/lib.rs b/apps/lightcraft-web/src/lib.rs
index 0ab06bc94..a2c5b5898 100644
--- a/apps/lightcraft-web/src/lib.rs
+++ b/apps/lightcraft-web/src/lib.rs
@@ -1,20 +1,24 @@
//! LightCraft in the browser: the same egui UI as the desktop app, compiled to WASM.
//!
-//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP.
-//! Differences from the desktop host:
-//! - no filesystem: photos picked (File ▸ Add Photos…) or dropped onto the page are held in
-//! memory ([`store::MemStore`]) and decoded from bytes;
-//! - export downloads the file through the browser;
+//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP (see
+//! `docs/web.md`). Differences from the desktop host:
+//! - the library lives in browser storage (OPFS, or IndexedDB where OPFS can't be written from the
+//! main thread): catalog journal, presets, view state and prefs are mirrored in memory and
+//! flushed in the background ([`files`]); imported originals are stored by content hash
+//! ([`store`]); a new library starts with the procedural demo photos;
//! - renders run inline on the main thread (one job per frame) — no worker threads yet;
-//! - no persistence yet (UI prefs and the catalog reset on reload);
-//! - the procedural demo library is loaded at start so the page is never empty.
+//! - export downloads the file through the browser.
//!
//! `?bench` in the URL runs a scripted first-paint / slider-latency measurement and logs it to the
-//! console (see [`bench`]).
+//! console (see [`bench`]); `?store=idb` forces the IndexedDB backend, `?store=memory` disables
+//! persistence and `?reset` clears the stored library.
#![forbid(unsafe_code)]
pub mod bench;
+pub mod files;
pub mod store;
+#[cfg(target_arch = "wasm32")]
+mod backend;
#[cfg(target_arch = "wasm32")]
mod web;
diff --git a/apps/lightcraft-web/src/store.rs b/apps/lightcraft-web/src/store.rs
index acac328e2..4b395c57b 100644
--- a/apps/lightcraft-web/src/store.rs
+++ b/apps/lightcraft-web/src/store.rs
@@ -1,6 +1,13 @@
-//! In-memory "filesystem" for the web build: files picked or dropped in the browser are kept as
-//! bytes under synthetic paths (`mem/<n>/<name>`), and the engine reads them through the same
-//! [`FileLoader`]/[`FileProbe`] hooks the desktop app backs with `std::fs`.
+//! Imported originals on the web. There is no filesystem: the bytes of every photo picked or
+//! dropped in the browser are stored in browser storage as `originals/<content hash>`, and the
+//! catalog refers to them by a synthetic path `web/<content hash>/<file name>` (so the decoders
+//! still see the file's extension).
+//!
+//! The engine reads files synchronously ([`FileLoader`]/[`FileProbe`]), so the main thread keeps
+//! the bytes it needs in a bounded memory cache ([`Originals`]). A miss (e.g. after a reload)
+//! queues an asynchronous load from storage ([`Originals::take_wanted`]) and fails for now; the
+//! host prefetches the active photo's original so commands that need its pixels (auto tone,
+//! export) find it. Render workers read originals from storage themselves.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
@@ -8,37 +15,119 @@ use std::sync::{Arc, Mutex};
use lightcraft_engine::files::{load_bytes, probe_bytes};
use lightcraft_engine::media::{FileLoader, FileProbe};
+/// Prefix of the catalog paths of files kept in browser storage.
+pub const PATH_PREFIX: &str = "web/";
+
+/// Memory budget for original bytes on the main thread.
+const MEM_BUDGET: usize = 768 << 20;
+
+/// Catalog path for a stored original.
+pub fn original_path(hash: &str, name: &str) -> String {
+ let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");
+ format!("{PATH_PREFIX}{hash}/{name}")
+}
+
+/// The content hash in a path made by [`original_path`].
+pub fn hash_of_path(path: &str) -> Option<&str> {
+ let (hash, _) = path.strip_prefix(PATH_PREFIX)?.split_once('/')?;
+ (!hash.is_empty()).then_some(hash)
+}
+
+/// Storage key of an original.
+pub fn storage_key(hash: &str) -> String {
+ format!("originals/{hash}")
+}
+
+/// Hash of a file's bytes (the same as the catalog's `content_hash`).
+pub fn content_hash(bytes: &[u8]) -> String {
+ lightcraft_preview::hash_bytes(bytes).to_string()
+}
+
#[derive(Default)]
struct Inner {
- files: HashMap<String, Arc<[u8]>>,
- /// Paths added since the last [`MemStore::take_pending`] (to import on the next frame).
+ /// hash → (bytes, last use).
+ bytes: HashMap<String, (Arc<[u8]>, u64)>,
+ total: usize,
+ clock: u64,
+ /// Catalog paths stored since the last [`Originals::take_pending`] (to import on the next frame).
pending: Vec<String>,
- next: u64,
+ /// Hashes to load from storage.
+ wanted: Vec<String>,
+ /// Hashes being loaded.
+ loading: std::collections::HashSet<String>,
}
-/// Shared, cloneable byte store. File reads in the browser are async, so additions arrive from
-/// promise callbacks and are picked up by the app on its next frame.
+/// Original bytes in memory, shared between the engine hooks and the host (cheap to clone).
#[derive(Clone, Default)]
-pub struct MemStore(Arc<Mutex<Inner>>);
+pub struct Originals(Arc<Mutex<Inner>>);
-impl MemStore {
+impl Originals {
fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
self.0.lock().unwrap_or_else(|e| e.into_inner())
}
- /// Store a file's bytes; returns its synthetic path and queues it for import.
- pub fn add(&self, name: &str, bytes: Vec<u8>) -> String {
+ /// Keep `bytes` (hash `hash`) in memory, evicting the least recently used others if needed.
+ pub fn insert(&self, hash: &str, bytes: Arc<[u8]>) {
let mut g = self.lock();
- g.next += 1;
- let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");
- let path = format!("mem/{}/{name}", g.next);
- g.files.insert(path.clone(), bytes.into());
- g.pending.push(path.clone());
+ g.clock += 1;
+ let clock = g.clock;
+ g.loading.remove(hash);
+ if let Some(old) = g.bytes.insert(hash.to_string(), (bytes.clone(), clock)) {
+ g.total -= old.0.len();
+ }
+ g.total += bytes.len();
+ while g.total > MEM_BUDGET && g.bytes.len() > 1 {
+ let Some(oldest) = g.bytes.iter().filter(|(h, _)| *h != hash).min_by_key(|(_, v)| v.1).map(|(h, _)| h.clone()) else { break };
+ if let Some(v) = g.bytes.remove(&oldest) {
+ g.total -= v.0.len();
+ }
+ }
+ }
+
+ /// A file was stored (bytes in storage and in memory): queue its path for import.
+ pub fn added(&self, name: &str, hash: &str, bytes: Arc<[u8]>) -> String {
+ self.insert(hash, bytes);
+ let path = original_path(hash, name);
+ self.lock().pending.push(path.clone());
path
}
+ /// Bytes for a catalog path, if in memory. A miss for a stored original queues its load.
pub fn get(&self, path: &str) -> Option<Arc<[u8]>> {
- self.lock().files.get(path).cloned()
+ let hash = hash_of_path(path)?;
+ let mut g = self.lock();
+ g.clock += 1;
+ let clock = g.clock;
+ if let Some(v) = g.bytes.get_mut(hash) {
+ v.1 = clock;
+ return Some(v.0.clone());
+ }
+ if !g.loading.contains(hash) && !g.wanted.iter().any(|w| w == hash) {
+ g.wanted.push(hash.to_string());
+ }
+ None
+ }
+
+ pub fn contains(&self, path: &str) -> bool {
+ hash_of_path(path).is_some_and(|h| self.lock().bytes.contains_key(h))
+ }
+
+ /// Ask for the original behind `path` to be loaded into memory (no-op if it's there).
+ pub fn prefetch(&self, path: &str) {
+ let _ = self.get(path);
+ }
+
+ /// Hashes to load from storage (they count as loading until [`Originals::insert`] or
+ /// [`Originals::load_failed`]).
+ pub fn take_wanted(&self) -> Vec<String> {
+ let mut g = self.lock();
+ let w = std::mem::take(&mut g.wanted);
+ g.loading.extend(w.iter().cloned());
+ w
+ }
+
+ pub fn load_failed(&self, hash: &str) {
+ self.lock().loading.remove(hash);
}
/// Paths added since the last call.
@@ -46,24 +135,22 @@ impl MemStore {
std::mem::take(&mut self.lock().pending)
}
- pub fn len(&self) -> usize {
- self.lock().files.len()
- }
-
- pub fn is_empty(&self) -> bool {
- self.len() == 0
+ /// (originals in memory, bytes).
+ pub fn usage(&self) -> (usize, usize) {
+ let g = self.lock();
+ (g.bytes.len(), g.total)
}
- /// Engine hooks that decode/probe from this store.
+ /// Engine hooks that decode/probe from memory.
pub fn hooks(&self) -> (FileLoader, FileProbe) {
let s = self.clone();
let loader: FileLoader = Arc::new(move |path: &str, max_edge: usize| {
- let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;
+ let bytes = s.get(path).ok_or_else(|| format!("{path}: the original is still loading from browser storage"))?;
load_bytes(&bytes, max_edge)
});
let s = self.clone();
let probe: FileProbe = Arc::new(move |path: &str| {
- let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;
+ let bytes = s.get(path).ok_or_else(|| format!("{path}: not loaded"))?;
probe_bytes(path, &bytes)
});
(loader, probe)
@@ -94,41 +181,54 @@ pub fn mime_for(name: &str) -> &'static str {
}
}
+#[cfg(test)]
+pub(crate) fn png_bytes(w: usize, h: usize) -> Vec<u8> {
+ let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();
+ let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();
+ lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()
+}
+
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
- fn png_bytes(w: usize, h: usize) -> Vec<u8> {
- let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();
- let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();
- lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()
+ #[test]
+ fn paths() {
+ let p = original_path("abc123", "C:\\x\\a.png");
+ assert_eq!(p, "web/abc123/a.png");
+ assert_eq!(hash_of_path(&p), Some("abc123"));
+ assert_eq!(hash_of_path("mem/1/a.png"), None);
+ assert_eq!(storage_key("abc"), "originals/abc");
}
#[test]
- fn add_take_get() {
- let s = MemStore::default();
- let a = s.add("C:\\x\\a.png", vec![1, 2]);
- let b = s.add("a.png", vec![3]);
- assert_ne!(a, b, "same name twice gets distinct paths");
- assert!(a.ends_with("/a.png"));
- assert_eq!(s.take_pending(), vec![a.clone(), b]);
- assert!(s.take_pending().is_empty());
- assert_eq!(&*s.get(&a).unwrap(), &[1, 2]);
- assert_eq!(s.len(), 2);
+ fn miss_queues_a_load_once() {
+ let s = Originals::default();
+ assert!(s.get("web/h1/a.png").is_none());
+ assert!(s.get("web/h1/a.png").is_none());
+ assert_eq!(s.take_wanted(), vec!["h1".to_string()]);
+ assert!(s.get("web/h1/a.png").is_none(), "still loading");
+ assert!(s.take_wanted().is_empty());
+ s.insert("h1", Arc::from(vec![1u8, 2]));
+ assert_eq!(&*s.get("web/h1/b.jpg").unwrap(), &[1, 2]);
}
#[test]
fn import_and_render_from_memory() {
- let store = MemStore::default();
+ let store = Originals::default();
let mut session = lightcraft_engine::Session::new();
store.install(&mut session);
- let path = store.add("tiny.png", png_bytes(40, 30));
+ let bytes = png_bytes(40, 30);
+ let hash = content_hash(&bytes);
+ let path = store.added("tiny.png", &hash, bytes.into());
+ assert_eq!(store.take_pending(), vec![path.clone()]);
let r = session.execute("library.import", &json!({"paths": [path]})).unwrap();
let id = lightcraft_engine::catalog::PhotoId(r["imported"][0].as_u64().unwrap());
let p = session.catalog.photo(id).unwrap();
assert_eq!((p.width, p.height), (40, 30));
assert_eq!(p.file_name, "tiny.png");
+ assert_eq!(p.content_hash.as_deref(), Some(hash.as_str()), "storage key = catalog content hash");
let out = session.render_now(id, 20, 20).unwrap();
assert_eq!(out.image.width, 20);
}
diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rs
index c902a2589..d11343b0b 100644
--- a/apps/lightcraft-web/src/web.rs
+++ b/apps/lightcraft-web/src/web.rs
@@ -1,17 +1,32 @@
-//! Browser host: starts eframe on `<canvas id="lightcraft_canvas">`, wires the in-memory store to
-//! the file picker and drag-and-drop, and turns exports into downloads.
+//! Browser host: opens the library in browser storage, starts eframe on
+//! `<canvas id="lightcraft_canvas">`, wires the file picker and drag-and-drop, and turns exports
+//! into downloads.
+
+use std::cell::Cell;
+use std::rc::Rc;
+use std::sync::Arc;
use lightcraft_engine::Session;
-use lightcraft_ui_egui::{LightcraftApp, Services};
+use lightcraft_engine::catalog::Source;
+use lightcraft_engine::library::LibraryStores;
+use lightcraft_ui_egui::{LightcraftApp, Services, UiState};
use serde_json::json;
use wasm_bindgen::JsCast;
use wasm_bindgen::prelude::*;
+use crate::backend::Backend;
use crate::bench::Bench;
-use crate::store::{MemStore, download_name};
+use crate::files::{Files, FlushOp, LIBRARY_FILES};
+use crate::store::{Originals, content_hash, download_name, storage_key};
const ACCEPT: &str = ".jpg,.jpeg,.png,.tif,.tiff,.webp,.dng,.cr2,.nef,.arw,.psd,.jxl,.gif,.bmp";
+/// Where the library files live in browser storage.
+const LIBRARY_DIR: &str = "library";
+
+/// How often view state and UI prefs are saved (ms).
+const SAVE_EVERY_MS: f64 = 1000.0;
+
fn window() -> web_sys::Window {
web_sys::window().expect("no window")
}
@@ -27,19 +42,62 @@ fn set_status(text: &str) {
}
}
-/// Read a browser `File` into the store.
-async fn read_file(store: MemStore, file: web_sys::File, ctx: egui::Context) {
- match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {
- Ok(buf) => {
- store.add(&file.name(), js_sys::Uint8Array::new(&buf).to_vec());
- ctx.request_repaint();
+fn library_key(name: &str) -> String {
+ format!("{LIBRARY_DIR}/{name}")
+}
+
+/// URL options (`?bench&store=idb&reset`).
+struct Options {
+ bench: bool,
+ /// "opfs" (default: OPFS, falling back to IndexedDB), "idb" or "memory".
+ store: String,
+ reset: bool,
+}
+
+impl Options {
+ fn from_url() -> Options {
+ let q = window().location().search().unwrap_or_default();
+ let params: Vec<(String, String)> = q
+ .trim_start_matches('?')
+ .split('&')
+ .filter(|p| !p.is_empty())
+ .map(|p| match p.split_once('=') {
+ Some((k, v)) => (k.to_string(), v.to_string()),
+ None => (p.to_string(), String::new()),
+ })
+ .collect();
+ let get = |k: &str| params.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
+ Options {
+ bench: get("bench").is_some(),
+ store: get("store").filter(|s| s == "idb" || s == "memory").unwrap_or_else(|| "opfs".into()),
+ reset: get("reset").is_some(),
}
+ }
+}
+
+/// Store a picked/dropped file (bytes into browser storage, then queue its import).
+async fn store_file(originals: Originals, backend: Option<Backend>, name: String, bytes: Vec<u8>, ctx: egui::Context) {
+ let hash = content_hash(&bytes);
+ if let Some(b) = &backend
+ && let Err(e) = b.write(&storage_key(&hash), &bytes).await
+ {
+ // still importable for this session; it won't survive a reload
+ log::error!("storing {name} in browser storage failed: {e}");
+ }
+ originals.added(&name, &hash, Arc::from(bytes));
+ ctx.request_repaint();
+}
+
+/// Read a browser `File` into storage.
+async fn read_file(originals: Originals, backend: Option<Backend>, file: web_sys::File, ctx: egui::Context) {
+ match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {
+ Ok(buf) => store_file(originals, backend, file.name(), js_sys::Uint8Array::new(&buf).to_vec(), ctx).await,
Err(e) => log::warn!("reading {}: {e:?}", file.name()),
}
}
-/// Show the browser's open dialog; picked files land in the store asynchronously.
-fn open_picker(store: MemStore, ctx: egui::Context) {
+/// Show the browser's open dialog; picked files are imported asynchronously.
+fn open_picker(originals: Originals, backend: Option<Backend>, ctx: egui::Context) {
let Some(doc) = window().document() else { return };
let Ok(input) = doc.create_element("input").map(|e| e.unchecked_into::<web_sys::HtmlInputElement>()) else { return };
input.set_type("file");
@@ -50,7 +108,7 @@ fn open_picker(store: MemStore, ctx: egui::Context) {
if let Some(files) = inp.files() {
for i in 0..files.length() {
if let Some(f) = files.get(i) {
- wasm_bindgen_futures::spawn_local(read_file(store.clone(), f, ctx.clone()));
+ wasm_bindgen_futures::spawn_local(read_file(originals.clone(), backend.clone(), f, ctx.clone()));
}
}
}
@@ -83,10 +141,10 @@ fn download(path: &str, bytes: &[u8]) -> Result<(), String> {
Ok(())
}
-fn services(store: MemStore, ctx: egui::Context) -> Services {
+fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context) -> Services {
Services {
pick_files: Some(Box::new(move || {
- open_picker(store.clone(), ctx.clone());
+ open_picker(originals.clone(), backend.clone(), ctx.clone());
Vec::new() // files arrive asynchronously and are imported on a later frame
})),
// Preset files: browser pickers are asynchronous; not wired on the web yet.
@@ -99,44 +157,145 @@ fn services(store: MemStore, ctx: egui::Context) -> Services {
}
}
+/// Write dirty library files to storage, in order, until none are left.
+async fn flush(files: Files, backend: Backend, flushing: Rc<Cell<bool>>) {
+ 'outer: loop {
+ let ops = files.take_dirty();
+ if ops.is_empty() {
+ break;
+ }
+ for (i, op) in ops.iter().enumerate() {
+ let r = match op {
+ FlushOp::Write { name, data } => backend.write(&library_key(name), data).await,
+ FlushOp::Append { name, offset, data } => backend.write_at(&library_key(name), Some(*offset), data).await,
+ };
+ if let Err(e) = r {
+ log::error!("saving {} failed: {e}", op.name());
+ files.failed(&ops[i..]);
+ break 'outer;
+ }
+ }
+ }
+ flushing.set(false);
+}
+
+/// Everything loaded before the UI starts.
+struct Boot {
+ opts: Options,
+ backend: Option<Backend>,
+ files: Files,
+}
+
+async fn boot(opts: Options) -> Boot {
+ let t0 = perf_now();
+ let backend = if opts.store == "memory" {
+ None
+ } else {
+ match Backend::open(opts.store == "idb").await {
+ Ok(b) => Some(b),
+ Err(e) => {
+ log::error!("no browser storage ({e}); this session won't be saved");
+ None
+ }
+ }
+ };
+ let files = Files::default();
+ if let Some(b) = &backend {
+ if opts.reset {
+ match b.clear().await {
+ Ok(()) => log::info!("lightcraft: storage cleared (?reset)"),
+ Err(e) => log::error!("clearing storage: {e}"),
+ }
+ }
+ for name in LIBRARY_FILES {
+ match b.read(&library_key(name)).await {
+ Ok(Some(data)) => files.preload(name, data),
+ Ok(None) => {}
+ Err(e) => log::error!("reading {name}: {e}"),
+ }
+ }
+ crate::backend::request_persistence();
+ }
+ log::info!("lightcraft: storage {} opened in {:.0} ms", backend.as_ref().map_or("memory", |b| b.kind()), perf_now() - t0);
+ Boot { opts, backend, files }
+}
+
struct WebApp {
app: LightcraftApp,
- store: MemStore,
+ originals: Originals,
+ files: Files,
+ backend: Option<Backend>,
+ flushing: Rc<Cell<bool>>,
bench: Option<Bench>,
first_frame_logged: bool,
+ last_save: f64,
+ ui_written: Vec<u8>,
}
impl WebApp {
- fn new(cc: &eframe::CreationContext<'_>, bench: bool) -> Self {
- let store = MemStore::default();
+ fn new(cc: &eframe::CreationContext<'_>, boot: Boot) -> Self {
+ let Boot { opts, backend, files } = boot;
+ let originals = Originals::default();
let t = perf_now();
- let mut session = Session::with_demo();
- log::info!("lightcraft: wasm started at {:.0} ms; demo library built in {:.0} ms", t, perf_now() - t);
- store.install(&mut session);
+ let mut session = Session::new();
+ originals.install(&mut session);
+ let stores = LibraryStores {
+ dir: format!("browser:{}/{LIBRARY_DIR}", backend.as_ref().map_or("memory", |b| b.kind())).into(),
+ catalog: Box::new(files.store()),
+ files: Box::new(files.store()),
+ on_disk: false,
+ };
+ match session.open_library_in(stores, true).cloned() {
+ Ok(r) => log::info!(
+ "lightcraft: library {} in {:.0} ms ({} photos; snapshot seq {}, {} ops replayed)",
+ if r.created { "created" } else { "loaded" },
+ perf_now() - t,
+ session.catalog.len(),
+ r.snapshot_seq,
+ r.replayed
+ ),
+ Err(e) => {
+ log::error!("opening the library failed: {e}; starting a temporary one");
+ session = Session::with_demo();
+ originals.install(&mut session);
+ }
+ }
// previews are large (≤ 2560 px, f32): keep few in a 32-bit address space
session.media.preview_capacity = 3;
- let mut app = LightcraftApp::new(session, services(store.clone(), cc.egui_ctx.clone()));
+ let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), cc.egui_ctx.clone()));
+ let ui_written = files.get("ui.json").unwrap_or_default();
+ if let Ok(ui) = serde_json::from_slice::<UiState>(&ui_written) {
+ app.ui = ui;
+ }
app.ui = app.ui.sanitized();
+ CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone()));
let origin = lightcraft_ui_egui::now_ms() - perf_now();
- WebApp { app, store, bench: bench.then(|| Bench::new(origin)), first_frame_logged: false }
+ WebApp {
+ app,
+ originals,
+ files,
+ backend,
+ flushing: Rc::new(Cell::new(false)),
+ bench: opts.bench.then(|| Bench::new(origin)),
+ first_frame_logged: false,
+ last_save: 0.0,
+ ui_written,
+ }
}
fn import_dropped(&mut self, ctx: &egui::Context) {
let dropped = ctx.input(|i| i.raw.dropped_files.clone());
for f in dropped {
- let (store, ctx) = (self.store.clone(), ctx.clone());
+ let (originals, backend, ctx) = (self.originals.clone(), self.backend.clone(), ctx.clone());
wasm_bindgen_futures::spawn_local(async move {
let name = f.path().to_string_lossy().to_string();
match f.bytes_async().await {
- Ok(bytes) => {
- store.add(&name, bytes);
- ctx.request_repaint();
- }
+ Ok(bytes) => store_file(originals, backend, name, bytes, ctx).await,
Err(e) => log::warn!("reading dropped {name}: {e}"),
}
});
}
- let paths = self.store.take_pending();
+ let paths = self.originals.take_pending();
if !paths.is_empty() {
let n = paths.len();
match self.app.run("library.import", json!({"paths": paths})) {
@@ -145,12 +304,114 @@ impl WebApp {
}
}
}
+
+ /// Load originals the main thread asked for (and the active photo's, ahead of time).
+ fn load_originals(&mut self, ctx: &egui::Context) {
+ if let Some(id) = self.app.session.selection.active
+ && let Some(Source::File { path }) = self.app.session.catalog.photo(id).map(|p| &p.source)
+ && !self.originals.contains(path)
+ {
+ self.originals.prefetch(path);
+ }
+ let Some(b) = &self.backend else { return };
+ for hash in self.originals.take_wanted() {
+ let (b, originals, ctx) = (b.clone(), self.originals.clone(), ctx.clone());
+ wasm_bindgen_futures::spawn_local(async move {
+ match b.read(&storage_key(&hash)).await {
+ Ok(Some(bytes)) => {
+ originals.insert(&hash, Arc::from(bytes));
+ ctx.request_repaint();
+ }
+ Ok(None) => {
+ log::warn!("original {hash} is not in browser storage");
+ originals.load_failed(&hash);
+ }
+ Err(e) => {
+ log::warn!("loading original {hash}: {e}");
+ originals.load_failed(&hash);
+ }
+ }
+ });
+ }
+ }
+
+ /// Save view state and UI prefs now and then; flush dirty files.
+ fn save(&mut self) {
+ let now = perf_now();
+ if now - self.last_save >= SAVE_EVERY_MS {
+ self.last_save = now;
+ self.app.session.save_view();
+ if let Ok(ui) = serde_json::to_vec_pretty(&self.app.ui)
+ && ui != self.ui_written
+ {
+ self.files.write("ui.json", &ui);
+ self.ui_written = ui;
+ }
+ }
+ if let Some(b) = &self.backend
+ && self.files.is_dirty()
+ && !self.flushing.get()
+ {
+ self.flushing.set(true);
+ wasm_bindgen_futures::spawn_local(flush(self.files.clone(), b.clone(), self.flushing.clone()));
+ }
+ }
+}
+
+type Reply = (js_sys::Function, js_sys::Function);
+
+thread_local! {
+ /// Commands from JavaScript ([`command`]), run on the next frame.
+ static INBOX: std::cell::RefCell<Vec<(String, String, Reply)>> = const { std::cell::RefCell::new(Vec::new()) };
+ static CTX: std::cell::RefCell<Option<egui::Context>> = const { std::cell::RefCell::new(None) };
+}
+
+/// Run a command by id from JavaScript (automation, tests): `await command("library.info", "{}")`
+/// resolves to the result as JSON text, or rejects with the error. Besides every engine/UI
+/// command, `web.stats` reports the browser host's state (storage, originals in memory).
+#[wasm_bindgen]
+pub fn command(id: String, params: String) -> js_sys::Promise {
+ js_sys::Promise::new(&mut |resolve, reject| {
+ INBOX.with(|q| q.borrow_mut().push((id.clone(), params.clone(), (resolve, reject))));
+ CTX.with(|c| c.borrow().as_ref().map(|c| c.request_repaint()));
+ })
+}
+
+impl WebApp {
+ fn web_stats(&self) -> serde_json::Value {
+ let (orig_n, orig_bytes) = self.originals.usage();
+ json!({
+ "storage": self.backend.as_ref().map_or("memory", |b| b.kind()),
+ "dirty": self.files.is_dirty(),
+ "flushing": self.flushing.get(),
+ "originalsInMemory": orig_n,
+ "originalBytesInMemory": orig_bytes,
+ "renderQueue": self.app.renderer.queued(),
+ "rendersInFlight": self.app.renderer.in_flight(),
+ "thumbTextures": self.app.renderer.thumb_textures(),
+ })
+ }
+
+ fn run_inbox(&mut self) {
+ let inbox = INBOX.with(|q| std::mem::take(&mut *q.borrow_mut()));
+ for (id, params, (resolve, reject)) in inbox {
+ let params: serde_json::Value = serde_json::from_str(&params).unwrap_or(json!({}));
+ let r = if id == "web.stats" { Ok(self.web_stats()) } else { self.app.run(&id, params) };
+ let _ = match r {
+ Ok(v) => resolve.call1(&JsValue::NULL, &v.to_string().into()),
+ Err(e) => reject.call1(&JsValue::NULL, &e.into()),
+ };
+ }
+ }
}
impl eframe::App for WebApp {
fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
+ self.run_inbox();
self.import_dropped(ctx);
+ self.load_originals(ctx);
self.app.logic(ctx);
+ self.save();
if let Some(b) = self.bench.as_mut()
&& let Some(report) = b.step(&mut self.app)
{
@@ -176,11 +437,12 @@ impl eframe::App for WebApp {
}
}
-#[wasm_bindgen(start)]
+/// Start the app (called by `index.html` after instantiating the module).
+#[wasm_bindgen]
pub fn start() {
eframe::WebLogger::init(log::LevelFilter::Info).ok();
log::info!("lightcraft: wasm instantiated at {:.0} ms", perf_now());
- let bench = window().location().search().is_ok_and(|q| q.contains("bench"));
+ let opts = Options::from_url();
wasm_bindgen_futures::spawn_local(async move {
let Some(canvas) =
window().document().and_then(|d| d.get_element_by_id("lightcraft_canvas")).and_then(|e| e.dyn_into::<web_sys::HtmlCanvasElement>().ok())
@@ -188,8 +450,9 @@ pub fn start() {
log::error!("missing <canvas id=\"lightcraft_canvas\">");
return;
};
+ let boot = boot(opts).await;
let runner = eframe::WebRunner::new();
- let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, bench))))).await;
+ let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot))))).await;
match r {
Ok(()) => {
if let Some(el) = window().document().and_then(|d| d.get_element_by_id("lightcraft_loading")) {
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs
index 3aac02034..e255b0e00 100644
--- a/crates/engine/src/import.rs
+++ b/crates/engine/src/import.rs
@@ -152,7 +152,10 @@ fn copy_into_library(lib: &Path, src: &str, date: &str) -> Result<String, String
/// Import files/folders. See the module docs.
pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result<ImportReport> {
- let lib_dir = s.library.as_ref().map(|l| l.dir.clone());
+ // Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of every
+ // added file in its own storage already, so "copy" there means "add".
+ let mode = if mode == ImportMode::Copy && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };
+ let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
if mode == ImportMode::Copy && lib_dir.is_none() {
return Err(crate::EngineError::Other("copying into the library needs an open library".into()));
}
diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs
index e9979354c..d0fd0ac3b 100644
--- a/crates/engine/src/library.rs
+++ b/crates/engine/src/library.rs
@@ -10,6 +10,9 @@
//! Originals/ (photos imported with "copy into library")
//! ```
//!
+//! The files live in [`Store`]s: a directory ([`FsStore`]) natively, or any other implementation
+//! via [`Session::open_library_in`] (the browser build keeps them in OPFS or IndexedDB).
+//!
//! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning,
//! so a crash loses at most the command in flight. The log is compacted into a snapshot when it
//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`].
@@ -36,13 +39,32 @@ pub fn default_dir() -> Option<PathBuf> {
}
pub struct Library {
+ /// The library directory, or a descriptive pseudo-path when the library isn't on disk.
pub dir: PathBuf,
+ /// The library is a real directory: photos can be copied into `Originals/` and thumbnails are
+ /// cached in `thumbs/`.
+ pub on_disk: bool,
journal: Journal,
+ /// presets.json, view.json, prefs.json.
+ files: Box<dyn Store>,
/// What happened when the catalog was loaded.
pub report: LoadReport,
/// Last persistence error (shown by the UI; ops stay pending and are retried).
pub last_error: Option<String>,
presets_written: String,
+ view_written: Vec<u8>,
+}
+
+/// Where a library's files live (see [`Session::open_library_in`]).
+pub struct LibraryStores {
+ /// Shown as the library's location (`library.info`).
+ pub dir: PathBuf,
+ /// The catalog journal (`catalog.snap`, `catalog.log`).
+ pub catalog: Box<dyn Store>,
+ /// presets.json, view.json, prefs.json.
+ pub files: Box<dyn Store>,
+ /// `dir` is a real directory (enables `Originals/` copies and the `thumbs/` disk cache).
+ pub on_disk: bool,
}
#[derive(Default, Serialize, Deserialize)]
@@ -87,8 +109,8 @@ fn presets_json(presets: &[Preset]) -> String {
serde_json::to_string_pretty(&f).unwrap_or_default()
}
-fn write_atomic(dir: &Path, name: &str, data: &[u8]) -> std::io::Result<()> {
- FsStore::open(dir)?.write_atomic(name, data)
+fn read_json<T: serde::de::DeserializeOwned>(store: &mut dyn Store, name: &str) -> Option<T> {
+ store.read(name).ok().flatten().and_then(|b| serde_json::from_slice(&b).ok())
}
impl Session {
@@ -96,8 +118,16 @@ impl Session {
/// `seed_demo`, a newly created library starts with the procedural demo photos.
pub fn open_library(&mut self, dir: impl AsRef<Path>, seed_demo: bool) -> Result<&LoadReport> {
let dir = dir.as_ref().to_path_buf();
- let store = FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())))?;
- let (mut journal, catalog, report) = Journal::open(Box::new(store))?;
+ let open = || FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())));
+ let stores = LibraryStores { catalog: Box::new(open()?), files: Box::new(open()?), on_disk: true, dir: dir.clone() };
+ self.open_library_in(stores, seed_demo)
+ }
+
+ /// Open (or create) a library whose files live in `stores` (e.g. browser storage), replacing
+ /// this session's catalog.
+ pub fn open_library_in(&mut self, stores: LibraryStores, seed_demo: bool) -> Result<&LoadReport> {
+ let LibraryStores { dir, catalog, mut files, on_disk } = stores;
+ let (mut journal, catalog, report) = Journal::open(catalog)?;
self.catalog = catalog;
self.undo.clear();
self.redo.clear();
@@ -110,9 +140,7 @@ impl Session {
journal.snapshot(&self.catalog)?;
}
// presets
- if let Ok(bytes) = std::fs::read(dir.join("presets.json"))
- && let Ok(f) = serde_json::from_slice::<PresetsFile>(&bytes)
- {
+ if let Some(f) = read_json::<PresetsFile>(files.as_mut(), "presets.json") {
for p in &mut self.presets {
p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite);
}
@@ -123,12 +151,9 @@ impl Session {
}
}
// preferences
- self.xmp =
- std::fs::read(dir.join("prefs.json")).ok().and_then(|b| serde_json::from_slice::<PrefsFile>(&b).ok()).map(|p| p.xmp).unwrap_or_default();
+ self.xmp = read_json::<PrefsFile>(files.as_mut(), "prefs.json").map(|p| p.xmp).unwrap_or_default();
// view state
- if let Ok(bytes) = std::fs::read(dir.join("view.json"))
- && let Ok(v) = serde_json::from_slice::<ViewFile>(&bytes)
- {
+ if let Some(v) = read_json::<ViewFile>(files.as_mut(), "view.json") {
self.source = v.source;
self.filter = v.filter;
self.sort = v.sort;
@@ -142,8 +167,11 @@ impl Session {
self.selection = Selection::single(*first);
}
let presets_written = presets_json(&self.presets);
- self.media.attach_disk_cache(&dir.join("thumbs"));
- self.library = Some(Library { dir, journal, report, last_error: None, presets_written });
+ if on_disk {
+ self.media.attach_disk_cache(&dir.join("thumbs"));
+ }
+ let view_written = self.view_json();
+ self.library = Some(Library { dir, on_disk, journal, files, report, last_error: None, presets_written, view_written });
Ok(&self.library.as_ref().expect("just set").report)
}
@@ -166,7 +194,7 @@ impl Session {
}
let presets = presets_json(&self.presets);
if presets != lib.presets_written {
- if let Err(e) = write_atomic(&lib.dir, "presets.json", presets.as_bytes()) {
+ if let Err(e) = lib.files.write_atomic("presets.json", presets.as_bytes()) {
log::error!("library: presets: {e}");
} else {
lib.presets_written = presets;
@@ -190,20 +218,39 @@ impl Session {
}
let _ = self.end_interaction();
self.persist()?;
- let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };
+ self.save_view();
let Some(lib) = self.library.as_mut() else { return Ok(()) };
lib.journal.snapshot(&self.catalog)?;
- if let Ok(v) = serde_json::to_vec_pretty(&view) {
- let _ = write_atomic(&lib.dir, "view.json", &v);
- }
Ok(())
}
+ fn view_json(&self) -> Vec<u8> {
+ let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };
+ serde_json::to_vec_pretty(&view).unwrap_or_default()
+ }
+
+ /// Save the view state (source, filter, sort, selection) if it changed since it was last
+ /// written. Native hosts get this from [`Session::close_library`]; the browser host calls it
+ /// periodically, since a tab can be closed without notice.
+ pub fn save_view(&mut self) {
+ if self.library.is_none() {
+ return;
+ }
+ let v = self.view_json();
+ let Some(lib) = self.library.as_mut() else { return };
+ if v != lib.view_written {
+ match lib.files.write_atomic("view.json", &v) {
+ Ok(()) => lib.view_written = v,
+ Err(e) => log::error!("library: view: {e}"),
+ }
+ }
+ }
+
/// Save the library preferences (no-op for in-memory sessions).
- pub fn save_prefs(&self) -> Result<()> {
- let Some(lib) = &self.library else { return Ok(()) };
+ pub fn save_prefs(&mut self) -> Result<()> {
let v = serde_json::to_vec_pretty(&PrefsFile { xmp: self.xmp }).unwrap_or_default();
- write_atomic(&lib.dir, "prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))
+ let Some(lib) = self.library.as_mut() else { return Ok(()) };
+ lib.files.write_atomic("prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))
}
/// Compact the log into a snapshot now.