Skip to content

LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust

LRC-LIB-CATALOG-IMPORT: address SQLite, worker & archive review

Adrian D'souza866d02e577cb
diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml
index f59523f02..c2d4e8a80 100644
--- a/crates/engine/Cargo.toml
+++ b/crates/engine/Cargo.toml
@@ -29,7 +29,6 @@ serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
log = { workspace = true }
-sqlite-core = "=0.12.0"
ab_glyph = { workspace = true }
lightcraft-segment = { workspace = true, optional = true }
diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs
index 4895eee73..1e94226e3 100644
--- a/crates/engine/src/import.rs
+++ b/crates/engine/src/import.rs
@@ -610,6 +610,78 @@ impl Prepared {
pub fn is_empty(&self) -> bool {
self.items.is_empty()
}
+
+ /// Recheck Add-only entries against the owner catalog immediately before commit.
+ /// Preparation may have run while another import added the same path or content.
+ pub fn revalidate_add(&mut self, s: &Session) {
+ let mut paths = HashMap::new();
+ let mut hashes = HashMap::new();
+ for photo in s.catalog.photos() {
+ if let Source::File { path } = &photo.source {
+ paths.insert(normalize_import_path(path), photo.id);
+ }
+ if let Some(hash) = &photo.content_hash {
+ hashes.insert(hash.clone(), photo.id);
+ }
+ }
+ let mut items = Vec::with_capacity(self.items.len());
+ for item in std::mem::take(&mut self.items) {
+ let PreparedItem::Ready(ready) = item else {
+ items.push(item);
+ continue;
+ };
+ let ReadyFile { path, stored, info, sidecar, placed } = *ready;
+ let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied());
+ let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied());
+ if let Some(existing) = existing_path.or(existing_hash) {
+ if let Some(placed) = &placed {
+ crate::import_move::rollback(placed);
+ }
+ items.push(PreparedItem::Duplicate {
+ path,
+ existing: Some(existing),
+ reason: if existing_path.is_some() { "path" } else { "content" },
+ hash: info.content_hash,
+ });
+ } else {
+ items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })));
+ }
+ }
+ self.items = items;
+ }
+}
+
+fn normalize_import_path(path: &str) -> String {
+ let path = path.replace('\\', "/");
+ if cfg!(windows) { path.to_lowercase() } else { path }
+}
+
+#[cfg(test)]
+mod prepared_tests {
+ use super::*;
+
+ #[test]
+ fn revalidate_add_only_converts_matching_ready_item() {
+ let mut s = Session::new();
+ let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg");
+ let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg");
+ let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now");
+ s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap();
+ let ready = |path: &Path| ReadyFile {
+ path: path.to_string_lossy().into(),
+ stored: path.to_string_lossy().into(),
+ info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() },
+ sidecar: None,
+ placed: None,
+ };
+ let mut prepared = Prepared {
+ scanned: 2,
+ items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))],
+ };
+ prepared.revalidate_add(&s);
+ assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. })));
+ assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_))));
+ }
}
impl ImportJob {
diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs
index 4ca8b5bc6..122bde8fa 100644
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@@ -25,7 +25,10 @@ pub mod guard;
pub mod import;
mod import_move;
pub mod library;
+mod lightroom_archive;
pub mod lightroom_catalog;
+pub mod lightroom_job;
+mod lightroom_sqlite;
pub mod media;
pub mod memory;
pub mod merge;
@@ -162,6 +165,8 @@ pub struct Session {
pub active_spot: Option<usize>,
/// The persistent library this session writes to (`None` = in-memory only).
pub library: Option<library::Library>,
+ /// Identity of current library opening; refreshed only after a successful open.
+ pub(crate) library_identity: Arc<()>,
/// XMP sidecar preferences (persisted with the library).
pub xmp: sidecar::XmpPrefs,
/// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json.
@@ -249,6 +254,7 @@ impl Session {
segmenter: segment::Segmenter::default(),
active_spot: None,
library: None,
+ library_identity: Arc::new(()),
xmp: sidecar::XmpPrefs::default(),
last_export: None,
export_presets: Vec::new(),
diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs
index 6a63e378e..a3f5be336 100644
--- a/crates/engine/src/library.rs
+++ b/crates/engine/src/library.rs
@@ -399,6 +399,7 @@ impl Session {
lib.forgot_local = Some(plan);
}
}
+ self.library_identity = std::sync::Arc::new(());
Ok(())
}
@@ -491,12 +492,16 @@ impl Session {
lib.retry_at = None;
lib.last_error = None;
}
- match (persisted, snapshot) {
+ let result = match (persisted, snapshot) {
(_, Ok(())) => Ok(()),
// neither the log nor the snapshot took the queued ops
(Err(EngineError::NotSaved(e)), Err(s)) if lib.journal.seq() == before => Err(EngineError::NotSaved(format!("{e}; snapshot: {s}"))),
(_, Err(s)) => Err(s.into()),
+ };
+ if result.is_ok() {
+ self.library_identity = std::sync::Arc::new(());
}
+ result
}
fn view_json(&self) -> Vec<u8> {
diff --git a/crates/engine/src/lightroom_archive.rs b/crates/engine/src/lightroom_archive.rs
new file mode 100644
index 000000000..ecb53e25c
--- /dev/null
+++ b/crates/engine/src/lightroom_archive.rs
@@ -0,0 +1,654 @@
+//! Bounded, deduplicated persistence for Lightroom import source data.
+//!
+//! Archives use a deterministic JSON representation wrapped in zlib. The JSON
+//! writer streams through a bounded compressor, so an import cannot first build
+//! an unbounded archive buffer. Blob columns are omitted by the catalog reader;
+//! decoded XMP,
+//! settings, history, snapshots, identities, paths, collections & membership
+//! remain available for recovery.
+
+use std::collections::BTreeMap;
+use std::fs::{self, File};
+use std::io::{self, Read, Write};
+use std::path::{Path, PathBuf};
+
+use miniz_oxide::deflate::{
+ core::{CompressorOxide, create_comp_flags_from_zip_params},
+ stream::deflate,
+};
+use miniz_oxide::{MZFlush, MZStatus};
+use serde::ser::{Serialize, SerializeMap, SerializeSeq, SerializeStruct, Serializer};
+use serde_json::Value;
+
+use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};
+
+/// Maximum size of one complete archive on disk, including format header.
+pub const MAX_ARCHIVE_BYTES: u64 = 32 * 1024 * 1024;
+/// Maximum total size retained in managed archives.
+pub const MAX_TOTAL_ARCHIVE_BYTES: u64 = 128 * 1024 * 1024;
+/// Maximum number of managed archives retained.
+pub const MAX_MANAGED_ARCHIVES: usize = 8;
+
+const FORMAT_VERSION: u8 = 1;
+const HEADER: &[u8] = b"LC-LRARCH\0\x01Z";
+const NAME_PREFIX: &str = "lightroom-import-";
+const NAME_SUFFIX: &str = ".lca";
+const MAX_SCAN: usize = 1024;
+const IO_BUFFER: usize = 16 * 1024;
+
+/// Store compact source data below `directory` and return its managed archive.
+///
+/// The operation is safe to call before catalog mutation. It rejects a
+/// symlink directory, writes only `lightroom-import-<number>.lca`, compares the
+/// complete bounded byte stream with recent managed archives, and retains at
+/// most [`MAX_MANAGED_ARCHIVES`] and [`MAX_TOTAL_ARCHIVE_BYTES`]. Legacy
+/// `lightroom-import-*.json` files are deliberately ignored and preserved for
+/// an explicit caller warning.
+pub fn store(directory: &Path, data: &CatalogImport) -> Result<PathBuf, String> {
+ prepare_directory(directory)?;
+ let mut managed = managed_files(directory)?;
+ managed.sort_by_key(|entry| entry.index);
+
+ let temporary = temporary_path(directory)?;
+ write_archive(&temporary, data).map_err(|error| {
+ let _ = remove_managed_file(&temporary);
+ error
+ })?;
+
+ let recent = managed.iter().rev().take(MAX_MANAGED_ARCHIVES);
+ for entry in recent {
+ let same = match same_bounded(&temporary, &entry.path) {
+ Ok(same) => same,
+ Err(error) => {
+ let _ = remove_managed_file(&temporary);
+ return Err(error.to_string());
+ }
+ };
+ if same {
+ remove_managed_file(&temporary).map_err(|e| e.to_string())?;
+ let matched = entry.path.clone();
+ let mut all = managed.clone();
+ prune_existing(directory, &mut all, Some(&matched))?;
+ return Ok(matched);
+ }
+ }
+
+ let index = managed.last().map_or(0, |entry| entry.index.saturating_add(1));
+ if managed.last().is_some_and(|entry| entry.index == u64::MAX) {
+ let _ = remove_managed_file(&temporary);
+ return Err("managed Lightroom archive index is exhausted".into());
+ }
+ let destination = match next_destination(directory, index) {
+ Ok(path) => path,
+ Err(error) => {
+ let _ = remove_managed_file(&temporary);
+ return Err(error);
+ }
+ };
+ let size = match fs::metadata(&temporary) {
+ Ok(meta) => meta.len(),
+ Err(error) => {
+ let _ = remove_managed_file(&temporary);
+ return Err(error.to_string());
+ }
+ };
+ let new_entry = Managed { index, path: destination.clone(), size };
+ let mut after = managed;
+ after.push(new_entry);
+ after.sort_by_key(|entry| entry.index);
+ let retirees = match retirement_set(&after) {
+ Ok(retirees) => retirees,
+ Err(error) => {
+ let _ = remove_managed_file(&temporary);
+ return Err(error);
+ }
+ };
+ let quarantined = match quarantine(&retirees) {
+ Ok(quarantined) => quarantined,
+ Err(error) => {
+ let _ = remove_managed_file(&temporary);
+ return Err(error);
+ }
+ };
+
+ if let Err(error) = publish_new(&temporary, &destination) {
+ rollback_quarantine(&quarantined);
+ let _ = remove_managed_file(&temporary);
+ return Err(error.to_string());
+ }
+ if let Err(error) = remove_managed_file(&temporary) {
+ return Err(format!("archive published but temporary cleanup failed: {error}"));
+ }
+ for (old, held) in quarantined {
+ if let Err(error) = remove_managed_file(&held) {
+ return Err(format!("archive published but could not retire {}: {error}", old.display()));
+ }
+ }
+ Ok(destination)
+}
+
+#[derive(Clone, Debug)]
+struct Managed {
+ index: u64,
+ path: PathBuf,
+ size: u64,
+}
+
+fn prepare_directory(directory: &Path) -> Result<(), String> {
+ match fs::symlink_metadata(directory) {
+ Ok(meta) => {
+ if meta.file_type().is_symlink() || !meta.is_dir() {
+ return Err(format!("archive directory is not a real directory: {}", directory.display()));
+ }
+ }
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {
+ fs::create_dir_all(directory).map_err(|e| e.to_string())?;
+ let meta = fs::symlink_metadata(directory).map_err(|e| e.to_string())?;
+ if meta.file_type().is_symlink() || !meta.is_dir() {
+ return Err(format!("archive directory is not a real directory: {}", directory.display()));
+ }
+ }
+ Err(error) => return Err(error.to_string()),
+ }
+ Ok(())
+}
+
+fn managed_files(directory: &Path) -> Result<Vec<Managed>, String> {
+ let mut files = Vec::new();
+ for (seen, result) in fs::read_dir(directory).map_err(|e| e.to_string())?.enumerate() {
+ if seen >= MAX_SCAN {
+ return Err(format!("too many entries in archive directory (limit {MAX_SCAN})"));
+ }
+ let entry = result.map_err(|e| e.to_string())?;
+ let Some(index) = parse_managed_name(&entry.file_name()) else { continue };
+ let meta = fs::symlink_metadata(entry.path()).map_err(|e| e.to_string())?;
+ if meta.file_type().is_symlink() || !meta.is_file() {
+ continue;
+ }
+ files.push(Managed { index, path: entry.path(), size: meta.len() });
+ }
+ Ok(files)
+}
+
+fn parse_managed_name(name: &std::ffi::OsStr) -> Option<u64> {
+ let name = name.to_str()?;
+ let digits = name.strip_prefix(NAME_PREFIX)?.strip_suffix(NAME_SUFFIX)?;
+ if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
+ return None;
+ }
+ digits.parse().ok()
+}
+
+fn temporary_path(directory: &Path) -> Result<PathBuf, String> {
+ let pid = std::process::id();
+ for slot in 0..MAX_SCAN {
+ let path = directory.join(format!(".lightroom-import-{pid}-{slot}.tmp"));
+ match fs::symlink_metadata(&path) {
+ Ok(_) => continue,
+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),
+ Err(error) => return Err(error.to_string()),
+ }
+ }
+ Err("no safe temporary archive name available".into())
+}
+
+fn next_destination(directory: &Path, mut index: u64) -> Result<PathBuf, String> {
+ for _ in 0..MAX_SCAN {
+ let path = directory.join(format!("{NAME_PREFIX}{index}{NAME_SUFFIX}"));
+ match fs::symlink_metadata(&path) {
+ Ok(_) => index = index.checked_add(1).ok_or("managed archive index is exhausted")?,
+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),
+ Err(error) => return Err(error.to_string()),
+ }
+ }
+ Err("no safe managed archive name available".into())
+}
+
+fn write_archive(path: &Path, data: &CatalogImport) -> Result<(), String> {
+ lightcraft_catalog::safe_file::write_atomic_with(path, &mut |raw| {
+ let mut bounded = BoundedWriter { inner: raw, total: 0 };
+ bounded.write_all(HEADER)?;
+ let mut compressor = ZlibWriter::new(&mut bounded);
+ serde_json::to_writer(&mut compressor, &ArchiveRef(data)).map_err(json_io)?;
+ compressor.finish()?;
+ Ok(())
+ })
+ .map(|_| ())
+ .map_err(|error| format!("could not write Lightroom archive: {error}"))
+}
+
+fn json_io(error: serde_json::Error) -> io::Error {
+ io::Error::new(io::ErrorKind::InvalidData, error.to_string())
+}
+
+struct BoundedWriter<'a> {
+ inner: &'a mut dyn Write,
+ total: u64,
+}
+
+impl Write for BoundedWriter<'_> {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ let next = self.total.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;
+ if next > MAX_ARCHIVE_BYTES {
+ return Err(limit_error());
+ }
+ self.inner.write_all(bytes)?;
+ self.total = next;
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ self.inner.flush()
+ }
+}
+
+fn limit_error() -> io::Error {
+ io::Error::new(io::ErrorKind::InvalidData, "Lightroom archive exceeds 32 MiB")
+}
+
+struct ZlibWriter<'a, 'b> {
+ sink: &'a mut BoundedWriter<'b>,
+ compressor: CompressorOxide,
+ plain: u64,
+}
+
+impl<'a, 'b> ZlibWriter<'a, 'b> {
+ fn new(sink: &'a mut BoundedWriter<'b>) -> ZlibWriter<'a, 'b> {
+ let flags = create_comp_flags_from_zip_params(6, 15, 0);
+ ZlibWriter { sink, compressor: CompressorOxide::new(flags), plain: 0 }
+ }
+
+ fn run(&mut self, input: &[u8], flush: MZFlush) -> io::Result<()> {
+ let mut rest = input;
+ loop {
+ let mut output = [0u8; IO_BUFFER];
+ let result = deflate(&mut self.compressor, rest, &mut output, flush);
+ self.sink.write_all(&output[..result.bytes_written])?;
+ match result.status {
+ Ok(MZStatus::StreamEnd) => return Ok(()),
+ Ok(MZStatus::Ok) => {
+ let consumed = result.bytes_consumed;
+ if consumed > rest.len() {
+ return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressor progress"));
+ }
+ rest = &rest[consumed..];
+ if rest.is_empty() && flush == MZFlush::None {
+ return Ok(());
+ }
+ if consumed == 0 && result.bytes_written == 0 {
+ return Err(io::Error::new(io::ErrorKind::InvalidData, "compressor made no progress"));
+ }
+ }
+ Ok(status) => {
+ return Err(io::Error::new(io::ErrorKind::InvalidData, format!("unsupported compressor status: {status:?}")));
+ }
+ Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("compression failed: {error:?}"))),
+ }
+ }
+ }
+
+ fn finish(&mut self) -> io::Result<()> {
+ self.run(&[], MZFlush::Finish)
+ }
+}
+
+impl Write for ZlibWriter<'_, '_> {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ if bytes.is_empty() {
+ return Ok(0);
+ }
+ let next = self.plain.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;
+ if next > MAX_ARCHIVE_BYTES {
+ return Err(limit_error());
+ }
+ self.run(bytes, MZFlush::None)?;
+ self.plain = next;
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+fn same_bounded(left: &Path, right: &Path) -> io::Result<bool> {
+ let left_meta = fs::symlink_metadata(left)?;
+ let right_meta = fs::symlink_metadata(right)?;
+ if left_meta.file_type().is_symlink() || right_meta.file_type().is_symlink() || !left_meta.is_file() || !right_meta.is_file() {
+ return Ok(false);
+ }
+ let left_len = left_meta.len();
+ let right_len = right_meta.len();
+ if left_len != right_len || left_len > MAX_ARCHIVE_BYTES {
+ return Ok(false);
+ }
+ let mut left = File::open(left)?;
+ let mut right = File::open(right)?;
+ let mut left_buf = [0u8; IO_BUFFER];
+ let mut right_buf = [0u8; IO_BUFFER];
+ loop {
+ let left_n = left.read(&mut left_buf)?;
+ let right_n = right.read(&mut right_buf)?;
+ if left_n != right_n {
+ return Ok(false);
+ }
+ if left_n == 0 {
+ return Ok(true);
+ }
+ if left_buf[..left_n] != right_buf[..right_n] {
+ return Ok(false);
+ }
+ }
+}
+
+fn retirement_set(entries: &[Managed]) -> Result<Vec<Managed>, String> {
+ retirement_set_protected(entries, None)
+}
+
+fn retirement_set_protected(entries: &[Managed], protected: Option<&Path>) -> Result<Vec<Managed>, String> {
+ let mut ordered = entries.to_vec();
+ ordered.sort_by_key(|entry| entry.index);
+ let mut total = ordered.iter().try_fold(0u64, |sum, entry| sum.checked_add(entry.size)).ok_or("archive size overflow")?;
+ let mut retirees = Vec::new();
+ while ordered.len() > MAX_MANAGED_ARCHIVES || total > MAX_TOTAL_ARCHIVE_BYTES {
+ let Some(position) = ordered.iter().position(|entry| protected.map_or(true, |path| entry.path != path)) else {
+ return Err("cannot satisfy archive retention limit".into());
+ };
+ let oldest = ordered.remove(position);
+ total = total.saturating_sub(oldest.size);
+ retirees.push(oldest);
+ }
+ Ok(retirees)
+}
+
+fn prune_existing(directory: &Path, entries: &mut [Managed], protected: Option<&Path>) -> Result<(), String> {
+ let retirees = retirement_set_protected(entries, protected)?;
+ let quarantined = quarantine(&retirees)?;
+ for (old, held) in quarantined {
+ if let Err(error) = remove_managed_file(&held) {
+ return Err(format!("could not retire {} from {}: {error}", old.display(), directory.display()));
+ }
+ }
+ Ok(())
+}
+
+fn quarantine(entries: &[Managed]) -> Result<Vec<(PathBuf, PathBuf)>, String> {
+ let mut moved = Vec::new();
+ let pid = std::process::id();
+ for (slot, entry) in entries.iter().enumerate() {
+ let held = entry.path.with_file_name(format!(".lightroom-retire-{pid}-{slot}.tmp"));
+ match fs::symlink_metadata(&held) {
+ Ok(_) => {
+ rollback_quarantine(&moved);
+ return Err(format!("retention temporary path already exists: {}", held.display()));
+ }
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}
+ Err(error) => {
+ rollback_quarantine(&moved);
+ return Err(error.to_string());
+ }
+ }
+ let meta = fs::symlink_metadata(&entry.path).map_err(|error| {
+ rollback_quarantine(&moved);
+ error.to_string()
+ })?;
+ if meta.file_type().is_symlink() || !meta.is_file() {
+ rollback_quarantine(&moved);
+ return Err(format!("managed archive changed into non-file: {}", entry.path.display()));
+ }
+ if let Err(error) = fs::rename(&entry.path, &held) {
+ rollback_quarantine(&moved);
+ return Err(error.to_string());
+ }
+ moved.push((entry.path.clone(), held));
+ }
+ Ok(moved)
+}
+
+fn rollback_quarantine(moved: &[(PathBuf, PathBuf)]) {
+ for (original, held) in moved.iter().rev() {
+ let _ = fs::rename(held, original);
+ }
+}
+
+fn publish_new(temporary: &Path, destination: &Path) -> io::Result<()> {
+ match fs::hard_link(temporary, destination) {
+ Ok(()) => Ok(()),
+ Err(error) => Err(io::Error::new(error.kind(), format!("cannot atomically publish archive: {error}"))),
+ }
+}
+
+fn remove_managed_file(path: &Path) -> io::Result<()> {
+ let meta = fs::symlink_metadata(path)?;
+ if meta.file_type().is_symlink() || !meta.is_file() {
+ return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to remove non-regular archive path"));
+ }
+ fs::remove_file(path)
+}
+
+struct ArchiveRef<'a>(&'a CatalogImport);
+
+impl Serialize for ArchiveRef<'_> {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut out = serializer.serialize_struct("CatalogImportArchive", 7)?;
+ out.serialize_field("version", &FORMAT_VERSION)?;
+ out.serialize_field("source", &self.0.source)?;
+ out.serialize_field("photos", &PhotoSeq(&self.0.photos))?;
+ out.serialize_field("collections", &RowSeq(&self.0.collections))?;
+ out.serialize_field("members", &RowSeq(&self.0.members))?;
+ out.serialize_field("collection_content", &RowSeq(&self.0.collection_content))?;
+ out.serialize_field("warnings", &self.0.warnings)?;
+ out.end()
+ }
+}
+
+struct PhotoSeq<'a>(&'a [CatalogPhoto]);
+impl Serialize for PhotoSeq<'_> {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut out = serializer.serialize_seq(Some(self.0.len()))?;
+ for photo in self.0 {
+ out.serialize_element(&PhotoRef(photo))?;
+ }
+ out.end()
+ }
+}
+
+struct PhotoRef<'a>(&'a CatalogPhoto);
+impl Serialize for PhotoRef<'_> {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let p = self.0;
+ let mut out = serializer.serialize_struct("CatalogPhotoArchive", 8)?;
+ out.serialize_field("source_id", &p.source_id)?;
+ out.serialize_field("uuid", &p.uuid)?;
+ out.serialize_field("path", &p.path)?;
+ out.serialize_field("image", &RowRef(&p.image))?;
+ out.serialize_field("settings", &p.settings)?;
+ out.serialize_field("xmp", &p.xmp)?;
+ out.serialize_field("keywords", &p.keywords)?;
+ out.serialize_field("history", &RowSeq(&p.history))?;
+ out.serialize_field("snapshots", &RowSeq(&p.snapshots))?;
+ out.end()
+ }
+}
+
+struct RowSeq<'a>(&'a [BTreeMap<String, Value>]);
+impl Serialize for RowSeq<'_> {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut out = serializer.serialize_seq(Some(self.0.len()))?;
+ for row in self.0 {
+ out.serialize_element(&RowRef(row))?;
+ }
+ out.end()
+ }
+}
+
+struct RowRef<'a>(&'a BTreeMap<String, Value>);
+impl Serialize for RowRef<'_> {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut out = serializer.serialize_map(None)?;
+ for (key, value) in self.0 {
+ out.serialize_entry(key, value)?;
+ }
+ out.end()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};
+ use std::sync::atomic::{AtomicU64, Ordering};
+
+ static NEXT: AtomicU64 = AtomicU64::new(0);
+
+ fn tempdir() -> PathBuf {
+ let path = std::env::temp_dir().join(format!("lightcraft-lr-archive-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
+ fs::create_dir_all(&path).unwrap();
+ path
+ }
+
+ fn data(source: String) -> CatalogImport {
+ CatalogImport {
+ source,
+ photos: Vec::new(),
+ collections: Vec::new(),
+ members: Vec::new(),
+ collection_content: Vec::new(),
+ warnings: Vec::new(),
+ }
+ }
+
+ fn rich_data() -> CatalogImport {
+ let photo = CatalogPhoto {
+ source_id: 7,
+ uuid: "uuid-7".into(),
+ path: "/photos/one.raw".into(),
+ image: BTreeMap::from([("rating".into(), serde_json::json!(5)), ("opaque_blob".into(), serde_json::json!([1, 2, 3, 4]))]),
+ settings: "s = { Exposure2012 = 0.75 }".into(),
+ xmp: "<xmp>decoded</xmp>".into(),
+ keywords: vec!["Trips|Goa".into()],
+ history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])],
+ snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])],
+ };
+ CatalogImport {
+ source: "source.lrcat".into(),
+ photos: vec![photo],
+ collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])],
+ members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])],
+ collection_content: vec![BTreeMap::from([("rule".into(), serde_json::json!("rating > 3"))])],
+ warnings: vec!["one warning".into()],
+ }
+ }
+
+ fn managed_count(path: &Path) -> usize {
+ fs::read_dir(path).unwrap().filter_map(Result::ok).filter(|entry| parse_managed_name(&entry.file_name()).is_some()).count()
+ }
+
+ #[test]
+ fn deduplicates_identical_deterministic_archives() {
+ let path = tempdir();
+ let input = data("catalog.lrcat".into());
+ let first = store(&path, &input).unwrap();
+ let second = store(&path, &input).unwrap();
+ assert_eq!(first, second);
+ assert_eq!(managed_count(&path), 1);
+ fs::remove_dir_all(path).unwrap();
+ }
+
+ #[test]
+ fn dedup_returns_archive_that_retention_keeps() {
+ let path = tempdir();
+ let input = data("catalog.lrcat".into());
+ let first = store(&path, &input).unwrap();
+ for index in 1..MAX_MANAGED_ARCHIVES {
+ let filler = path.join(format!("lightroom-import-{index}.lca"));
+ File::create(filler).unwrap().set_len(19 * 1024 * 1024).unwrap();
+ }
+ let reused = store(&path, &input).unwrap();
+ assert_eq!(reused, first);
+ assert!(reused.is_file());
+ assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);
+ fs::remove_dir_all(path).unwrap();
+ }
+
+ #[test]
+ fn caps_managed_archive_count_and_preserves_unrelated_files() {
+ let path = tempdir();
+ let human = path.join("notes.txt");
+ let legacy = path.join("lightroom-import-0.json");
+ fs::write(&human, b"keep").unwrap();
+ fs::write(&legacy, b"legacy").unwrap();
+ for index in 0..(MAX_MANAGED_ARCHIVES + 3) {
+ let input = data(format!("catalog-{index}.lrcat"));
+ store(&path, &input).unwrap();
+ }
+ assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);
+ assert_eq!(fs::read(&human).unwrap(), b"keep");
+ assert_eq!(fs::read(&legacy).unwrap(), b"legacy");
+ fs::remove_dir_all(path).unwrap();
+ }
+
+ #[test]
+ fn rejects_oversize_before_creating_archive() {
+ let path = tempdir();
+ let input = data("x".repeat((MAX_ARCHIVE_BYTES as usize) + 1));
+ assert!(store(&path, &input).is_err());
+ assert_eq!(managed_count(&path), 0);
+ fs::remove_dir_all(path).unwrap();
+ }
+
+ #[test]
+ fn archive_roundtrips_zlib_payload_and_preserves_numeric_arrays() {
+ let path = tempdir();
+ let archive = store(&path, &rich_data()).unwrap();
+ let bytes = fs::read(archive).unwrap();
+ assert!(bytes.starts_with(HEADER));
+ let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&bytes[HEADER.len()..], MAX_ARCHIVE_BYTES as usize).unwrap();
+ let decoded: Value = serde_json::from_slice(&json).unwrap();
+ assert_eq!(decoded["version"], serde_json::json!(FORMAT_VERSION));
+ assert_eq!(decoded["source"], serde_json::json!("source.lrcat"));
+ assert_eq!(decoded["photos"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["photos"][0]["source_id"], serde_json::json!(7));
+ assert_eq!(decoded["photos"][0]["uuid"], serde_json::json!("uuid-7"));
+ assert_eq!(decoded["photos"][0]["path"], serde_json::json!("/photos/one.raw"));
+ assert_eq!(decoded["photos"][0]["settings"], serde_json::json!("s = { Exposure2012 = 0.75 }"));
+ assert_eq!(decoded["photos"][0]["xmp"], serde_json::json!("<xmp>decoded</xmp>"));
+ assert_eq!(decoded["photos"][0]["keywords"], serde_json::json!(["Trips|Goa"]));
+ assert_eq!(decoded["photos"][0]["image"]["rating"], serde_json::json!(5));
+ assert_eq!(decoded["photos"][0]["image"]["opaque_blob"], serde_json::json!([1, 2, 3, 4]));
+ assert_eq!(decoded["photos"][0]["history"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["photos"][0]["history"][0]["curve"], serde_json::json!([0, 128, 255]));
+ assert_eq!(decoded["photos"][0]["snapshots"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["collections"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["collections"][0]["name"], serde_json::json!("Keepers"));
+ assert_eq!(decoded["members"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["members"][0]["image"], serde_json::json!(7));
+ assert_eq!(decoded["collection_content"].as_array().unwrap().len(), 1);
+ assert_eq!(decoded["warnings"], serde_json::json!(["one warning"]));
+ fs::remove_dir_all(path).unwrap();
+ }
+
+ #[test]
+ fn retention_enforces_aggregate_byte_cap_with_small_synthetic_entries() {
+ let mut entries: Vec<_> = (0..MAX_MANAGED_ARCHIVES as u64)
+ .map(|index| Managed { index, path: PathBuf::from(format!("lightroom-import-{index}.lca")), size: 20 * 1024 * 1024 })
+ .collect();
+ entries.push(Managed { index: 8, path: PathBuf::from("lightroom-import-8.lca"), size: 20 * 1024 * 1024 });
+ let retirees = retirement_set(&entries).unwrap();
+ assert_eq!(retirees.len(), 3);
+ assert_eq!(retirees.iter().map(|entry| entry.index).collect::<Vec<_>>(), vec![0, 1, 2]);
+ }
+
+ #[test]
+ fn retention_failure_preserves_old_archive() {
+ let path = tempdir();
+ let old = path.join("lightroom-import-0.lca");
+ let held = path.join(format!(".lightroom-retire-{}-0.tmp", std::process::id()));
+ fs::write(&old, b"old archive").unwrap();
+ fs::write(&held, b"occupied").unwrap();
+ let entry = Managed { index: 0, path: old.clone(), size: 11 };
+ assert!(quarantine(&[entry]).is_err());
+ assert_eq!(fs::read(&old).unwrap(), b"old archive");
+ fs::remove_dir_all(path).unwrap();
+ }
+}
diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs
index 5ce8d577a..47f222473 100644
--- a/crates/engine/src/lightroom_catalog.rs
+++ b/crates/engine/src/lightroom_catalog.rs
@@ -3,12 +3,15 @@
//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicUsize, Ordering},
+};
+use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};
use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
-use sqlite_core::{Database, Value as SqlValue};
type Row = BTreeMap<String, Value>;
const COMMAND: &str = "library.importLightroom";
@@ -296,6 +299,10 @@ fn text<'a>(r: &'a Row, k: &str) -> &'a str {
fn number(r: &Row, k: &str) -> i64 {
r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64
}
+
+pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 {
+ number(r, k)
+}
fn key(path: &str) -> String {
let p = path.replace('\\', "/");
if cfg!(windows) { p.to_lowercase() } else { p }
@@ -309,7 +316,7 @@ fn source_path(p: &Photo) -> Option<&str> {
struct Reader {
db: Database,
- tables: Vec<sqlite_core::attribution::LiveTable>,
+ tables: Vec<LiveTable>,
}
impl Reader {
fn table(&self, name: &str, required: bool) -> Result<Vec<Row>, String> {
@@ -328,17 +335,64 @@ impl Reader {
.iter()
.cloned()
.zip(r.values.into_iter().map(|v| match v {
- SqlValue::Null => Value::Null,
- SqlValue::Integer(n) => json!(n),
- SqlValue::Real(n) => json!(n),
- SqlValue::Text(s) => json!(s),
- // Opaque data is retained in the source catalog; the XMP packet is decoded below.
- SqlValue::Blob(b) => json!(b),
+ SqlValue::Null => Some(Value::Null),
+ SqlValue::Integer(n) => Some(json!(n)),
+ SqlValue::Real(n) => Some(json!(n)),
+ SqlValue::Text(s) => Some(json!(s)),
+ // Opaque Lightroom payloads never become JSON byte arrays.
+ SqlValue::Blob(_) => None,
}))
+ .filter_map(|(column, value)| value.map(|value| (column, value)))
.collect()
})
.collect())
}
+
+ fn xmp_table(&self) -> Result<(Vec<Row>, Vec<String>), String> {
+ let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) };
+ let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?;
+ let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?;
+ if rows.len() > 1_000_000 {
+ return Err("Adobe_AdditionalMetadata: more than one million records".into());
+ }
+ let mut warnings = Vec::new();
+ let mut out = Vec::with_capacity(rows.len());
+ for row in rows {
+ let image = row
+ .values
+ .iter()
+ .zip(columns)
+ .find_map(|(value, column)| {
+ (column == "image").then(|| match value {
+ SqlValue::Integer(n) => *n,
+ _ => 0,
+ })
+ })
+ .unwrap_or(0);
+ let mut mapped = Row::new();
+ for (column, value) in columns.iter().zip(row.values) {
+ let value = match value {
+ SqlValue::Null => Some(Value::Null),
+ SqlValue::Integer(n) => Some(json!(n)),
+ SqlValue::Real(n) => Some(json!(n)),
+ SqlValue::Text(s) => Some(json!(s)),
+ SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) {
+ Ok(packet) => Some(Value::String(packet)),
+ Err(error) => {
+ warnings.push(format!("image {image}: {error}"));
+ None
+ }
+ },
+ SqlValue::Blob(_) => None,
+ };
+ if let Some(value) = value {
+ mapped.insert(column.clone(), value);
+ }
+ }
+ out.push(mapped);
+ }
+ Ok((out, warnings))
+ }
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -364,10 +418,38 @@ pub struct CatalogImport {
pub warnings: Vec<String>,
}
-#[derive(Default, Serialize, Deserialize)]
-struct ImportIndex {
- photos: BTreeMap<String, u64>,
- collections: BTreeMap<String, u64>,
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub(crate) struct ImportIndex {
+ pub(crate) photos: BTreeMap<String, u64>,
+ pub(crate) collections: BTreeMap<String, u64>,
+}
+
+pub(crate) fn load_index(path: Option<&Path>) -> crate::Result<ImportIndex> {
+ match path {
+ Some(path) => {
+ let file = match std::fs::File::open(path) {
+ Ok(file) => file,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()),
+ Err(e) => return Err(error(e.to_string())),
+ };
+ if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 {
+ return Err(error("Lightroom import index exceeds 16 MiB"));
+ }
+ let mut bytes = Vec::new();
+ use std::io::Read;
+ file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?;
+ if bytes.len() > 16 * 1024 * 1024 {
+ return Err(error("Lightroom import index exceeds 16 MiB"));
+ }
+ serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}")))
+ }
+ None => Ok(ImportIndex::default()),
+ }
+}
+
+pub(crate) struct AppliedImport {
+ pub(crate) report: Value,
+ pub(crate) index: ImportIndex,
}
fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String {
@@ -406,6 +488,10 @@ fn validate(data: &CatalogImport) -> Result<(), String> {
Ok(())
}
+pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> {
+ validate(data)
+}
+
fn bytes(path: &Path) -> Result<Vec<u8>, String> {
use std::io::Read;
let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
@@ -440,6 +526,19 @@ fn xmp(v: Option<&Value>) -> Result<String, String> {
}
}
+fn decode_xmp_bytes(bytes: &[u8]) -> Result<String, String> {
+ let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?;
+ if size > 16 << 20 {
+ return Err("XMP packet exceeds 16 MiB".into());
+ }
+ let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20)
+ .map_err(|_| "invalid compressed XMP")?;
+ if decoded.len() != size as usize {
+ return Err("XMP packet length mismatch".into());
+ }
+ String::from_utf8(decoded).map_err(|e| e.to_string())
+}
+
fn be_word(bytes: &[u8], at: usize) -> Result<u32, String> {
let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?;
Ok(u32::from_be_bytes(word))
@@ -500,6 +599,17 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {
/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.
pub fn read(path: &Path) -> Result<CatalogImport, String> {
+ let cancel = AtomicBool::new(false);
+ let total = AtomicUsize::new(0);
+ let done = AtomicUsize::new(0);
+ read_with_progress(path, &cancel, &total, &done)
+}
+
+pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result<CatalogImport, String> {
+ // 13 catalog tables plus one validation stage; ImportJob adds one unit per original path.
+ total.store(14, Ordering::Relaxed);
+ done.store(0, Ordering::Relaxed);
+ ensure_read_active(cancel)?;
let before = std::fs::metadata(path).map_err(|e| e.to_string())?;
let wal_path = PathBuf::from(format!("{}-wal", path.display()));
let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
@@ -510,22 +620,33 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
Err(e) => return Err(e.to_string()),
};
let wal_len = committed_wal_len(&main, &wal)?;
+ ensure_read_active(cancel)?;
let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;
let after = std::fs::metadata(path).map_err(|e| e.to_string())?;
let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after {
return Err("catalog changed during reading; close Lightroom and retry".into());
}
- let reader = Reader { tables: db.live_tables(), db };
- let roots: HashMap<_, _> = reader.table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
- let folders: HashMap<_, _> = reader.table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
- let files: HashMap<_, _> = reader.table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
+ let reader = Reader { tables: db.live_tables()?, db };
+ let mut table = |name: &str, required: bool| -> Result<Vec<Row>, String> {
+ ensure_read_active(cancel)?;
+ let rows = reader.table(name, required)?;
+ done.fetch_add(1, Ordering::Relaxed);
+ Ok(rows)
+ };
+ let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
+ let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
+ let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let develops: HashMap<_, _> =
- reader.table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();
- let metadata: HashMap<_, _> = reader.table("Adobe_AdditionalMetadata", false)?.into_iter().map(|r| (number(&r, "image"), r)).collect();
- let keyword_rows: HashMap<_, _> = reader.table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
+ table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();
+ ensure_read_active(cancel)?;
+ let (metadata_rows, mut metadata_warnings) = reader.xmp_table()?;
+ done.fetch_add(1, Ordering::Relaxed);
+ let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect();
+ let keyword_rows: HashMap<_, _> = table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
let mut keywords: HashMap<i64, Vec<String>> = HashMap::new();
- for r in reader.table("AgLibraryKeywordImage", false)? {
+ for r in table("AgLibraryKeywordImage", false)? {
+ ensure_read_active(cancel)?;
let mut parts = Vec::new();
let mut at = number(&r, "tag");
let mut seen = HashSet::new();
@@ -552,17 +673,18 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
}
out
};
- let mut history = group(reader.table("Adobe_libraryImageDevelopHistoryStep", false)?);
- let mut snapshots = group(reader.table("Adobe_libraryImageDevelopSnapshot", false)?);
+ let mut history = group(table("Adobe_libraryImageDevelopHistoryStep", false)?);
+ let mut snapshots = group(table("Adobe_libraryImageDevelopSnapshot", false)?);
let mut out = CatalogImport {
source: path.to_string_lossy().into(),
photos: Vec::new(),
- collections: reader.table("AgLibraryCollection", false)?,
- members: reader.table("AgLibraryCollectionImage", false)?,
- collection_content: reader.table("AgLibraryCollectionContent", false)?,
- warnings: Vec::new(),
+ collections: table("AgLibraryCollection", false)?,
+ members: table("AgLibraryCollectionImage", false)?,
+ collection_content: table("AgLibraryCollectionContent", false)?,
+ warnings: std::mem::take(&mut metadata_warnings),
};
- for image in reader.table("Adobe_images", true)? {
+ for image in table("Adobe_images", true)? {
+ ensure_read_active(cancel)?;
let id = number(&image, "id_local");
let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?;
let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?;
@@ -593,9 +715,14 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
snapshots: snapshots.remove(&id).unwrap_or_default(),
});
}
+ ensure_read_active(cancel)?;
Ok(out)
}
+fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
+ if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
+}
+
fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec<String>), String> {
let mut root = crate::preset_import::parse_lua(text)?;
let deferred = strip_lua_sentinels(&mut root);
@@ -651,39 +778,60 @@ fn strip_xmp_sentinels(value: &mut Value) {
/// Import into the current library. Existing edited records are preserved unless explicitly
/// requested; missing originals remain catalogued for later relinking. The source catalog is read-only.
pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result<Value> {
- let data = read(path).map_err(error)?;
- apply(s, data, update_existing)
+ let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?;
+ let cancel = AtomicBool::new(false);
+ let prepared = job.prepare(&cancel)?;
+ let completion = crate::lightroom_job::commit_prepared(s, prepared)?;
+ if s.library.as_ref().is_some_and(|library| library.on_disk) {
+ s.persist()?;
+ }
+ completion.finalization.finish()?;
+ Ok(completion.report)
}
+#[cfg(test)]
fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result<Value> {
- validate(&data).map_err(error)?;
- let mut warnings = data.warnings.clone();
- let archive = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));
- let index_path = archive.as_ref().map(|p| p.join("lightroom-index.json"));
- let mut index: ImportIndex = match index_path.as_ref().map(std::fs::read) {
- Some(Ok(b)) => serde_json::from_slice(&b).map_err(|e| error(format!("invalid Lightroom import index: {e}")))?,
- Some(Err(e)) if e.kind() != std::io::ErrorKind::NotFound => return Err(error(e.to_string())),
- _ => ImportIndex::default(),
- };
- // Preserve source data before catalog changes. A unique archive is never overwritten.
- let archive_path = if let Some(dir) = archive {
- std::fs::create_dir_all(&dir).map_err(|e| error(e.to_string()))?;
- let bytes = serde_json::to_vec(&data).map_err(|e| error(e.to_string()))?;
- let mut names = (0..10000).map(|n| dir.join(format!("lightroom-import-{n}.json")));
- Some(lightcraft_catalog::safe_file::write_new_unique(&mut names, &bytes).map_err(|e| error(e.to_string()))?)
- } else {
- None
- };
+ let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));
+ let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json"));
+ let mut index = load_index(index_path.as_deref())?;
+ let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;
+ let undo0 = s.undo.len();
let existing: HashMap<_, _> =
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
- let undo0 = s.undo.len();
let paths: Vec<_> = data
.photos
.iter()
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file())
.map(|p| p.path.clone())
.collect();
+ let missing_sources: HashSet<String> =
+ data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect();
let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;
+ let applied =
+ apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;
+ if let Some(path) = index_path {
+ s.persist()?;
+ let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?;
+ lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;
+ }
+ Ok(applied.report)
+}
+
+/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation.
+/// This function does no filesystem work; callers finish its returned index separately.
+pub(crate) struct ApplyPrepared<'a> {
+ pub(crate) update_existing: bool,
+ pub(crate) files: crate::import::ImportReport,
+ pub(crate) index: &'a mut ImportIndex,
+ pub(crate) archive_path: Option<PathBuf>,
+ pub(crate) missing_sources: &'a HashSet<String>,
+ pub(crate) undo0: usize,
+}
+
+pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
+ let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;
+ validate(&data).map_err(error)?;
+ let mut warnings = data.warnings.clone();
let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect();
let mut by_path: HashMap<_, _> =
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
@@ -729,7 +877,7 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
} else {
let id = s.catalog.alloc_photo_id();
let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone());
- if !Path::new(&src.path).is_file() {
+ if missing_sources.contains(&key(&src.path)) {
missing.push(src.path.clone());
}
let mut p = Photo::new(
@@ -866,12 +1014,8 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
s.commit("Import Lightroom Catalog", Op::Batch { ops })?;
}
s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog");
- if let Some(path) = index_path {
- s.persist()?;
- let bytes = serde_json::to_vec(&index).map_err(|e| error(e.to_string()))?;
- lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;
- }
- Ok(
- json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),
- )
+ Ok(AppliedImport {
+ report: json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),
+ index: index.clone(),
+ })
}
diff --git a/crates/engine/src/lightroom_job.rs b/crates/engine/src/lightroom_job.rs
new file mode 100644
index 000000000..ab383aa11
--- /dev/null
+++ b/crates/engine/src/lightroom_job.rs
@@ -0,0 +1,386 @@
+//! Detached Lightroom catalog preparation and owner-thread commit.
+//!
+//! `LightroomJob::new` takes only an in-memory session snapshot. `prepare` owns all catalog
+//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations
+//! to the session that created the job. Archive index bytes are returned for a worker to finish
+//! after the owner has accepted the catalog operation.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+use std::sync::{
+ Arc,
+ atomic::{AtomicBool, AtomicUsize, Ordering},
+};
+
+use serde_json::Value;
+
+use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared};
+use crate::lightroom_catalog::{self, CatalogImport, ImportIndex};
+
+const COMMAND: &str = "library.importLightroom";
+const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024;
+
+fn error(message: impl Into<String>) -> crate::EngineError {
+ crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() }
+}
+
+/// Memory identity of the library which owns a prepared completion.
+///
+/// Each successful open/close refreshes session identity, including reopening the same path.
+/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation
+/// runs must survive and are merged by the owner commit.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct LightroomLibraryToken {
+ library_dir: Option<PathBuf>,
+ identity: Arc<()>,
+}
+
+impl LightroomLibraryToken {
+ fn capture(s: &crate::Session) -> Self {
+ Self {
+ library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()),
+ identity: Arc::clone(&s.library_identity),
+ }
+ }
+
+ /// Whether `s` still refers to the library which started preparation.
+ pub fn matches(&self, s: &crate::Session) -> bool {
+ Arc::ptr_eq(&self.identity, &s.library_identity)
+ && self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone())
+ }
+
+ pub fn library_dir(&self) -> Option<&Path> {
+ self.library_dir.as_deref()
+ }
+}
+
+/// Filesystem-free result of catalog preparation.
+pub struct PreparedLightroom {
+ pub(crate) data: CatalogImport,
+ pub(crate) normal: Prepared,
+ pub(crate) index: ImportIndex,
+ pub(crate) archive_path: Option<PathBuf>,
+ pub(crate) token: LightroomLibraryToken,
+ pub(crate) missing_sources: HashSet<String>,
+ pub(crate) update_existing: bool,
+ pub(crate) now: String,
+ cancelled: bool,
+}
+
+impl PreparedLightroom {
+ pub fn source(&self) -> &str {
+ &self.data.source
+ }
+
+ pub fn photo_count(&self) -> usize {
+ self.data.photos.len()
+ }
+
+ pub fn token(&self) -> &LightroomLibraryToken {
+ &self.token
+ }
+
+ pub fn is_cancelled(&self) -> bool {
+ self.cancelled
+ }
+
+ /// Roll back any normal-import filesystem placement held by this prepared value.
+ pub fn rollback(&self) {
+ self.normal.rollback();
+ }
+}
+
+/// Index write returned by owner commit. `finish` is intended for a worker after the owner has
+/// returned, and performs the only archive-index filesystem operation in this pipeline.
+pub struct LightroomArchiveFinalization {
+ path: Option<PathBuf>,
+ bytes: Option<Vec<u8>>,
+}
+
+impl LightroomArchiveFinalization {
+ pub fn finish(self) -> crate::Result<()> {
+ let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) };
+ if bytes.len() > MAX_INDEX_BYTES {
+ return Err(error("Lightroom import index exceeds 16 MiB"));
+ }
+ lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))
+ }
+
+ pub fn path(&self) -> Option<&Path> {
+ self.path.as_deref()
+ }
+}
+
+/// Owner-thread catalog result. The report is ready for the UI immediately; index finalization
+/// can remain on the worker while its task stays alive.
+pub struct LightroomCompletion {
+ pub report: Value,
+ pub finalization: LightroomArchiveFinalization,
+}
+
+/// Alias used by hosts that call the result a commit.
+pub type LightroomCommit = LightroomCompletion;
+
+/// A Lightroom import detached from the owner thread.
+pub struct LightroomJob {
+ path: PathBuf,
+ update_existing: bool,
+ import: ImportJob,
+ token: LightroomLibraryToken,
+ archive_dir: Option<PathBuf>,
+}
+
+impl LightroomJob {
+ /// Snapshot session state only. This constructor performs no filesystem work.
+ pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result<Self> {
+ let opts = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };
+ let import = ImportJob::new(s, opts)?;
+ let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop"));
+ Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir })
+ }
+
+ pub fn total(&self) -> usize {
+ self.import.total.load(Ordering::Relaxed)
+ }
+
+ pub fn done(&self) -> usize {
+ self.import.done.load(Ordering::Relaxed)
+ }
+
+ pub fn progress(&self) -> (usize, usize) {
+ (self.done(), self.total())
+ }
+
+ pub fn token(&self) -> &LightroomLibraryToken {
+ &self.token
+ }
+
+ pub fn total_atomic(&self) -> std::sync::Arc<AtomicUsize> {
+ std::sync::Arc::clone(&self.import.total)
+ }
+
+ pub fn done_atomic(&self) -> std::sync::Arc<AtomicUsize> {
+ std::sync::Arc::clone(&self.import.done)
+ }
+
+ /// Finish an owner commit's deferred archive-index write on a worker.
+ pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> {
+ finalization.finish()
+ }
+
+ /// Read/probe/stage all source data off the owner thread.
+ pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result<PreparedLightroom> {
+ if cancel.load(Ordering::Relaxed) {
+ return Err(error("Lightroom import cancelled"));
+ }
+ self.import.total.store(1, Ordering::Relaxed);
+ self.import.done.store(0, Ordering::Relaxed);
+ let data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?;
+ lightroom_catalog::validate_for_job(&data).map_err(error)?;
+ if cancel.load(Ordering::Relaxed) {
+ return Err(error("Lightroom import cancelled"));
+ }
+ self.import.done.fetch_add(1, Ordering::Relaxed);
+ let paths: Vec<String> = data
+ .photos
+ .iter()
+ .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)
+ .map(|photo| photo.path.clone())
+ .collect();
+ let missing_sources = data
+ .photos
+ .iter()
+ .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)
+ .filter(|photo| !Path::new(&photo.path).is_file())
+ .map(|photo| normalize(&photo.path))
+ .collect();
+ let normal = self.import.prepare(&paths, cancel);
+ if cancel.load(Ordering::Relaxed) {
+ normal.rollback();
+ return Err(error("Lightroom import cancelled"));
+ }
+ let archive_path = self.archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;
+ let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json"));
+ let index = lightroom_catalog::load_index(index_path.as_deref())?;
+ Ok(PreparedLightroom {
+ data,
+ normal,
+ index,
+ archive_path,
+ token: self.token.clone(),
+ missing_sources,
+ update_existing: self.update_existing,
+ now: self.import.now().to_string(),
+ cancelled: cancel.load(Ordering::Relaxed),
+ })
+ }
+}
+
+/// Apply prepared data on the owner thread. No source, archive or index filesystem work occurs.
+pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result<LightroomCompletion> {
+ if prepared.cancelled {
+ prepared.normal.rollback();
+ return Err(error("Lightroom import cancelled"));
+ }
+ if !prepared.token.matches(s) {
+ prepared.normal.rollback();
+ return Err(error("Lightroom import belongs to a different open library"));
+ }
+ // A command-scoped Lightroom import must never rewrite source XMP sidecars. Direct depth-0
+ // callers do not set this transient flag, so they cannot affect a later command.
+ if s.depth > 0 {
+ s.skip_auto_write = true;
+ }
+
+ let undo0 = s.undo.len();
+ prepared.normal.revalidate_add(s);
+ let options = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };
+ let now = prepared.now.clone();
+ let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?;
+ let applied = lightroom_catalog::apply_prepared(
+ s,
+ prepared.data,
+ lightroom_catalog::ApplyPrepared {
+ update_existing: prepared.update_existing,
+ files,
+ index: &mut prepared.index,
+ archive_path: prepared.archive_path,
+ missing_sources: &prepared.missing_sources,
+ undo0,
+ },
+ )?;
+ let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json"));
+ let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None };
+ Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } })
+}
+
+fn normalize(path: &str) -> String {
+ let path = path.replace('\\', "/");
+ if cfg!(windows) { path.to_lowercase() } else { path }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn empty_data() -> CatalogImport {
+ CatalogImport {
+ source: "fixture.lrcat".into(),
+ photos: Vec::new(),
+ collections: Vec::new(),
+ members: Vec::new(),
+ collection_content: Vec::new(),
+ warnings: Vec::new(),
+ }
+ }
+
+ fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom {
+ PreparedLightroom {
+ data: empty_data(),
+ normal: Prepared::default(),
+ index: ImportIndex::default(),
+ archive_path: None,
+ token: LightroomLibraryToken::capture(s),
+ missing_sources: HashSet::new(),
+ update_existing: false,
+ now: "2026-01-01T00:00:00".into(),
+ cancelled: true,
+ }
+ }
+
+ fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom {
+ let mut image = std::collections::BTreeMap::new();
+ image.insert("id_local".into(), serde_json::json!(1));
+ image.insert("rating".into(), serde_json::json!(rating));
+ image.insert("pick".into(), serde_json::json!(0));
+ image.insert("fileFormat".into(), serde_json::json!("JPG"));
+ image.insert("fileWidth".into(), serde_json::json!(1));
+ image.insert("fileHeight".into(), serde_json::json!(1));
+ PreparedLightroom {
+ data: CatalogImport {
+ source: "fixture.lrcat".into(),
+ photos: vec![lightroom_catalog::CatalogPhoto {
+ source_id: 1,
+ uuid: "fixture-photo".into(),
+ path: path.into(),
+ image,
+ settings: String::new(),
+ xmp: String::new(),
+ keywords: Vec::new(),
+ history: Vec::new(),
+ snapshots: Vec::new(),
+ }],
+ collections: Vec::new(),
+ members: Vec::new(),
+ collection_content: Vec::new(),
+ warnings: Vec::new(),
+ },
+ normal: Prepared::default(),
+ index: ImportIndex::default(),
+ archive_path: None,
+ token: LightroomLibraryToken::capture(s),
+ missing_sources: [normalize(path)].into_iter().collect(),
+ update_existing: false,
+ now: "2026-01-01T00:00:00".into(),
+ cancelled: false,
+ }
+ }
+
+ #[test]
+ fn cancellation_before_commit_leaves_owner_catalog_unchanged() {
+ let mut session = crate::Session::new();
+ let before = session.catalog.to_snapshot();
+ let prepared = cancelled_prepared(&session);
+ assert!(commit_prepared(&mut session, prepared).is_err());
+ assert_eq!(session.catalog.to_snapshot(), before);
+ assert!(session.undo.is_empty());
+ }
+
+ #[test]
+ fn library_identity_rejects_late_completion_after_reopen() {
+ let mut session = crate::Session::new();
+ let token = LightroomLibraryToken::capture(&session);
+ assert!(token.matches(&session));
+ session.library_identity = Arc::new(());
+ assert!(!token.matches(&session));
+ }
+
+ #[test]
+ fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() {
+ let mut session = crate::Session::new();
+ let path = "/missing/lightroom-job-edit.jpg";
+ let prepared = prepared_for(&session, path, 4);
+ let first = commit_prepared(&mut session, prepared).unwrap();
+ assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1));
+ assert_eq!(session.undo.len(), 1);
+ let id = lightcraft_catalog::PhotoId(1);
+ session.set_develop(id, lightcraft_develop::DevelopSettings::default(), "Personal").unwrap();
+ let before = session.catalog.photo(id).map(|photo| photo.develop.clone());
+ let prepared = prepared_for(&session, path, 5);
+ commit_prepared(&mut session, prepared).unwrap();
+ assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before);
+ assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5));
+ assert_eq!(session.undo.len(), 3);
+ }
+
+ #[test]
+ fn execute_fn_import_does_not_rewrite_source_sidecar() {
+ let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id()));
+ let path = dir.join("source.jpg");
+ let sidecar = dir.join("source.xmp");
+ std::fs::create_dir_all(&dir).unwrap();
+ std::fs::write(&path, b"fixture").unwrap();
+ std::fs::write(&sidecar, b"original sidecar").unwrap();
+ let original = std::fs::read(&sidecar).unwrap();
+ let mut session = crate::Session::new();
+ session.xmp.auto_write = true;
+ let result = session.execute_fn("library.importLightroom", |s| {
+ let prepared = prepared_for(s, &path.to_string_lossy(), 5);
+ let completion = commit_prepared(s, prepared)?;
+ Ok(completion.report)
+ });
+ assert!(result.is_ok());
+ assert_eq!(std::fs::read(&sidecar).unwrap(), original);
+ std::fs::remove_dir_all(dir).unwrap();
+ }
+}
diff --git a/crates/engine/src/lightroom_sqlite.rs b/crates/engine/src/lightroom_sqlite.rs
new file mode 100644
index 000000000..be7f4cb92
--- /dev/null
+++ b/crates/engine/src/lightroom_sqlite.rs
@@ -0,0 +1,1071 @@
+//! Small, read-only SQLite reader used by Lightroom catalog import.
+//!
+//! This is a clean-room implementation based on SQLite's public file-format
+//! description: <https://www.sqlite.org/fileformat.html> (sections 1.6, 1.7,
+//! 2.1, 2.2, 2.3, 2.4 and 2.5). It is deliberately a page reader, rather
+//! than a SQL engine. It never writes its input and has no SQLite runtime
+//! dependency.
+
+use std::collections::{HashMap, HashSet};
+
+const MAX_PAGE_ID: u32 = 16_777_216;
+const MAX_COLUMNS: usize = 16_384;
+const MAX_ROWS: usize = 1_000_000;
+const MAX_CELLS: usize = 4_000_000;
+const MAX_PAYLOAD: usize = 64 * 1024 * 1024;
+const MAX_TABLE_PAYLOAD: usize = 256 * 1024 * 1024;
+const MAX_OVERFLOW_PAGES: usize = 1_048_576;
+const MAX_BTREE_DEPTH: usize = 64;
+
+/// Values supported by SQLite table records.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Value {
+ Null,
+ Integer(i64),
+ Real(f64),
+ Text(String),
+ Blob(Vec<u8>),
+}
+
+/// A row returned by [`Database::read_table`].
+#[derive(Clone, Debug, PartialEq)]
+pub struct Record {
+ pub values: Vec<Value>,
+}
+
+/// A table discovered in `sqlite_schema`.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct LiveTable {
+ pub name: String,
+ pub rootpage: u32,
+ pub column_names: Option<Vec<String>>,
+}
+
+/// Read-only SQLite database image, with a committed WAL overlay if supplied.
+pub struct Database {
+ main: Vec<u8>,
+ overlay: HashMap<u32, Vec<u8>>,
+ page_size: usize,
+ reserved: usize,
+ encoding: TextEncoding,
+ page_count: u32,
+ tables: Vec<LiveTable>,
+ rowid_aliases: HashMap<u32, usize>,
+}
+
+#[derive(Clone, Copy)]
+enum TextEncoding {
+ Utf8,
+ Utf16Le,
+ Utf16Be,
+}
+
+struct WalkState {
+ seen: HashSet<u32>,
+ out: Vec<Record>,
+ payload_total: usize,
+}
+
+struct WalOverlay {
+ page_size: usize,
+ frame_count: usize,
+ pages: HashMap<u32, Vec<u8>>,
+ page_count: Option<u32>,
+}
+
+struct SchemaInfo {
+ tables: Vec<LiveTable>,
+ aliases: HashMap<u32, usize>,
+}
+
+impl Database {
+ /// Open a main database image and an already checksum-validated WAL prefix.
+ ///
+ /// The WAL argument must include its 32-byte header and only complete
+ /// frames through a commit. Frames are overlaid by page number, with the
+ /// final commit's database-size field defining the visible page count.
+ pub fn open_with_wal(main: Vec<u8>, committed_wal: &[u8]) -> Result<Self, String> {
+ let main_header = main.get(..100).ok_or("truncated SQLite header")?;
+ if main_header.get(..16) != Some(b"SQLite format 3\0") {
+ return Err("invalid SQLite header signature".into());
+ }
+ let page_size = sqlite_page_size(be_u16(main_header, 16)?)?;
+ validate_header_fields(main_header, page_size)?;
+ if main.len() < page_size || main.len() % page_size != 0 {
+ return Err("SQLite image is not a whole number of pages".into());
+ }
+ let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?;
+ if main_pages == 0 || main_pages > MAX_PAGE_ID {
+ return Err("invalid SQLite image page count".into());
+ }
+
+ let mut overlay = HashMap::new();
+ let mut wal_page_count = None;
+ if !committed_wal.is_empty() {
+ let parsed = parse_wal(committed_wal, page_size)?;
+ if parsed.page_size != page_size {
+ return Err("WAL page size mismatch".into());
+ }
+ if parsed.frame_count > MAX_CELLS {
+ return Err("WAL contains too many frames".into());
+ }
+ overlay = parsed.pages;
+ wal_page_count = parsed.page_count;
+ }
+ let effective_header = overlay.get(&1).map(Vec::as_slice).unwrap_or(main_header);
+ if effective_header.get(..16) != Some(b"SQLite format 3\0") {
+ return Err("invalid SQLite header page in WAL".into());
+ }
+ let effective_size = sqlite_page_size(be_u16(effective_header, 16)?)?;
+ if effective_size != page_size {
+ return Err("WAL page 1 changes SQLite page size".into());
+ }
+ let reserved = validate_header_fields(effective_header, page_size)?;
+ let header_pages = be_u32(effective_header, 28)?;
+ let header_valid = be_u32(effective_header, 24)? != 0 && be_u32(effective_header, 24)? == be_u32(effective_header, 92)?;
+ if header_valid && header_pages > main_pages && wal_page_count.is_none() {
+ return Err("SQLite header page count exceeds image".into());
+ }
+ let mut page_count = if header_valid && header_pages != 0 { header_pages } else { main_pages };
+ if let Some(count) = wal_page_count {
+ page_count = count;
+ }
+ if page_count == 0 || page_count > MAX_PAGE_ID {
+ return Err("invalid visible SQLite page count".into());
+ }
+ let encoding = match be_u32(effective_header, 56)? {
+ 0 | 1 => TextEncoding::Utf8,
+ 2 => TextEncoding::Utf16Le,
+ 3 => TextEncoding::Utf16Be,
+ _ => return Err("unsupported SQLite text encoding".into()),
+ };
+
+ let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() };
+ let schema = db.parse_schema()?;
+ db.tables = schema.tables;
+ db.rowid_aliases = schema.aliases;
+ Ok(db)
+ }
+
+ /// Return tables from `sqlite_schema` in schema order.
+ pub fn live_tables(&self) -> Result<Vec<LiveTable>, String> {
+ Ok(self.tables.clone())
+ }
+
+ /// Read a table B-tree, decoding up to `ncols` values per record.
+ pub fn read_table(&self, rootpage: u32, ncols: usize) -> Result<Vec<Record>, String> {
+ if !(1..=MAX_PAGE_ID).contains(&rootpage) || rootpage > self.page_count {
+ return Err("table root page is outside database".into());
+ }
+ if ncols > MAX_COLUMNS {
+ return Err("table has too many columns".into());
+ }
+ let alias = self.rowid_aliases.get(&rootpage).copied();
+ let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 };
+ self.visit_table(rootpage, ncols, alias, 0, &mut state)?;
+ Ok(state.out)
+ }
+
+ fn page(&self, page_id: u32) -> Result<&[u8], String> {
+ if page_id == 0 || page_id > self.page_count {
+ return Err(format!("page {page_id} is outside database"));
+ }
+ if let Some(page) = self.overlay.get(&page_id) {
+ return Ok(page);
+ }
+ let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?;
+ let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?;
+ let end = start.checked_add(self.page_size).ok_or("page offset overflow")?;
+ self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated"))
+ }
+
+ fn visit_table(&self, page_id: u32, ncols: usize, alias: Option<usize>, depth: usize, state: &mut WalkState) -> Result<(), String> {
+ if depth > MAX_BTREE_DEPTH {
+ return Err("SQLite B-tree is too deep".into());
+ }
+ if !state.seen.insert(page_id) {
+ return Err("SQLite B-tree contains a cycle or shared page".into());
+ }
+ let page = self.page(page_id)?;
+ let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?;
+ let base = if page_id == 1 { 100 } else { 0 };
+ let content = page.get(..usable).ok_or("truncated usable SQLite page")?;
+ let kind = *content.get(base).ok_or("truncated B-tree header")?;
+ let cells = usize::from(be_u16_at(content, base + 3)?);
+ if cells > MAX_CELLS || state.out.len().saturating_add(cells) > MAX_ROWS.saturating_add(MAX_CELLS) {
+ return Err("SQLite B-tree contains too many cells".into());
+ }
+ match kind {
+ 0x0d => {
+ let ptr_start = base.checked_add(8).ok_or("B-tree header overflow")?;
+ let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?;
+ let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?;
+ if ptr_end > usable {
+ return Err("truncated B-tree pointer array".into());
+ }
+ for i in 0..cells {
+ let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?;
+ let cell = usize::from(be_u16_at(content, p)?);
+ if cell < ptr_end || cell >= usable {
+ return Err("invalid B-tree cell pointer".into());
+ }
+ let mut at = cell;
+ let payload_len = usize_from_u64(read_varint(content, &mut at)?, "payload length")?;
+ if payload_len > MAX_PAYLOAD {
+ return Err("SQLite record payload is too large".into());
+ }
+ let rowid = i64::from_be_bytes(read_varint(content, &mut at)?.to_be_bytes());
+ state.payload_total = state.payload_total.checked_add(payload_len).ok_or("SQLite table payload total overflow")?;
+ if state.payload_total > MAX_TABLE_PAYLOAD {
+ return Err("SQLite table payload exceeds bounded import size".into());
+ }
+ let payload = self.payload(page, at, payload_len)?;
+ let mut values = decode_record(&payload, ncols, self.encoding)?;
+ if let Some(index) = alias {
+ if index >= values.len() {
+ return Err("INTEGER PRIMARY KEY column is missing from record".into());
+ }
+ if matches!(values.get(index), Some(Value::Null)) {
+ if let Some(value) = values.get_mut(index) {
+ *value = Value::Integer(rowid);
+ }
+ }
+ }
+ if state.out.len() >= MAX_ROWS {
+ return Err("SQLite table contains more than one million records".into());
+ }
+ state.out.push(Record { values });
+ }
+ Ok(())
+ }
+ 0x05 => {
+ let ptr_start = base.checked_add(12).ok_or("B-tree header overflow")?;
+ let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?;
+ let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?;
+ if ptr_end > usable {
+ return Err("truncated interior B-tree pointer array".into());
+ }
+ for i in 0..cells {
+ let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?;
+ let cell = usize::from(be_u16_at(content, p)?);
+ if cell < ptr_end || cell >= usable {
+ return Err("invalid interior B-tree cell pointer".into());
+ }
+ let child = be_u32_at(content, cell)?;
+ if child == 0 {
+ return Err("interior B-tree cell has null child".into());
+ }
+ let mut key_at = cell.checked_add(4).ok_or("interior cell overflow")?;
+ let _ = read_varint(content, &mut key_at)?;
+ self.visit_table(child, ncols, alias, depth + 1, state)?;
+ }
+ let right = be_u32_at(content, base + 8)?;
+ if right == 0 {
+ return Err("interior B-tree has null right child".into());
+ }
+ self.visit_table(right, ncols, alias, depth + 1, state)
+ }
+ 0x02 | 0x0a => Err("SQLite index B-trees are unsupported".into()),
+ other => Err(format!("unsupported SQLite B-tree page type 0x{other:02x}")),
+ }
+ }
+
+ fn payload(&self, page: &[u8], cell_at: usize, payload_len: usize) -> Result<Vec<u8>, String> {
+ let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?;
+ if usable < 8 {
+ return Err("SQLite usable page size is too small".into());
+ }
+ let max_local = usable.saturating_sub(35);
+ let min_local = ((usable - 12).saturating_mul(32) / 255).saturating_sub(23);
+ let local = if payload_len <= max_local {
+ payload_len
+ } else {
+ let span = usable - 4;
+ let candidate = min_local.saturating_add((payload_len - min_local) % span);
+ if candidate > max_local { min_local } else { candidate }
+ };
+ let local_end = cell_at.checked_add(local).ok_or("SQLite cell payload overflow")?;
+ if local_end > usable {
+ return Err("truncated SQLite cell payload".into());
+ }
+ let mut out = Vec::with_capacity(payload_len);
+ out.extend_from_slice(page.get(cell_at..local_end).ok_or("truncated SQLite cell payload")?);
+ let mut remaining = payload_len.saturating_sub(local);
+ if remaining == 0 {
+ return Ok(out);
+ }
+ let overflow_at = local_end;
+ match overflow_at.checked_add(4) {
+ Some(end) if end <= usable => {}
+ _ => return Err("SQLite overflow pointer is in reserved page bytes".into()),
+ }
+ let first = be_u32_at(page, overflow_at)?;
+ let mut next = first;
+ let mut seen = HashSet::new();
+ while remaining > 0 {
+ if next == 0 || !seen.insert(next) || seen.len() > MAX_OVERFLOW_PAGES {
+ return Err("invalid or cyclic SQLite overflow chain".into());
+ }
+ let overflow = self.page(next)?;
+ let take = remaining.min(usable - 4);
+ let end = 4usize.checked_add(take).ok_or("overflow payload offset")?;
+ out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?);
+ remaining -= take;
+ if remaining > 0 {
+ next = be_u32_at(overflow, 0)?;
+ }
+ }
+ Ok(out)
+ }
+
+ fn parse_schema(&self) -> Result<SchemaInfo, String> {
+ let mut tables = Vec::new();
+ let mut aliases = HashMap::new();
+ let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 };
+ self.visit_table(1, 5, None, 0, &mut state)?;
+ for record in state.out {
+ if record.values.len() < 5 {
+ return Err("sqlite_schema row has too few columns".into());
+ }
+ let kind = match record.values.first() {
+ Some(Value::Text(s)) => s.as_str(),
+ _ => return Err("sqlite_schema row has invalid type".into()),
+ };
+ let name = match record.values.get(1) {
+ Some(Value::Text(s)) => s.clone(),
+ _ => return Err("sqlite_schema row has invalid name".into()),
+ };
+ if kind.eq_ignore_ascii_case("virtual") {
+ return Err(format!("virtual table {name} is unsupported"));
+ }
+ if !kind.eq_ignore_ascii_case("table") {
+ continue;
+ }
+ let rootpage = match record.values.get(3) {
+ Some(Value::Integer(n)) if *n > 0 => u32::try_from(*n).map_err(|_| "sqlite_schema root page is invalid")?,
+ _ => return Err(format!("table {name} has invalid root page")),
+ };
+ let sql = match record.values.get(4) {
+ Some(Value::Text(s)) => s,
+ _ => return Err(format!("table {name} has no CREATE TABLE SQL")),
+ };
+ let (column_names, alias) = parse_create_table(sql)?;
+ if column_names.len() > MAX_COLUMNS {
+ return Err(format!("table {name} has too many columns"));
+ }
+ if tables.iter().any(|table: &LiveTable| table.rootpage == rootpage) {
+ return Err("duplicate SQLite table root page".into());
+ }
+ if let Some(index) = alias {
+ aliases.insert(rootpage, index);
+ }
+ tables.push(LiveTable { name, rootpage, column_names: Some(column_names) });
+ }
+ Ok(SchemaInfo { tables, aliases })
+ }
+}
+
+fn validate_header_fields(header: &[u8], page_size: usize) -> Result<usize, String> {
+ let write_version = *header.get(18).ok_or("truncated SQLite header")?;
+ let read_version = *header.get(19).ok_or("truncated SQLite header")?;
+ if !matches!(write_version, 1 | 2) || !matches!(read_version, 1 | 2) {
+ return Err("unsupported SQLite file format version".into());
+ }
+ if *header.get(21).ok_or("truncated SQLite header")? != 64
+ || *header.get(22).ok_or("truncated SQLite header")? != 32
+ || *header.get(23).ok_or("truncated SQLite header")? != 32
+ {
+ return Err("invalid SQLite payload fractions".into());
+ }
+ let reserved = usize::from(*header.get(20).ok_or("truncated SQLite header")?);
+ let usable = page_size.checked_sub(reserved).ok_or("invalid SQLite reserved-byte count")?;
+ if usable < 480 {
+ return Err("SQLite usable page size is below 480 bytes".into());
+ }
+ let schema_format = be_u32(header, 44)?;
+ if !(1..=4).contains(&schema_format) {
+ return Err("unsupported SQLite schema format".into());
+ }
+ Ok(reserved)
+}
+
+fn parse_wal(wal: &[u8], page_size: usize) -> Result<WalOverlay, String> {
+ let header = wal.get(..32).ok_or("truncated WAL header")?;
+ let magic = be_u32(header, 0)?;
+ if magic != 0x377f0682 && magic != 0x377f0683 {
+ return Err("invalid WAL magic".into());
+ }
+ if be_u32(header, 4)? != 3_007_000 || be_u32(header, 8)? != u32::try_from(page_size).map_err(|_| "page size overflow")? {
+ return Err("invalid WAL page size or format version".into());
+ }
+ let stride = page_size.checked_add(24).ok_or("WAL frame size overflow")?;
+ let body = wal.get(32..).ok_or("truncated WAL body")?;
+ if body.len() % stride != 0 {
+ return Err("WAL ends with a partial frame".into());
+ }
+ let salt = (be_u32(header, 16)?, be_u32(header, 20)?);
+ let frame_count = body.len() / stride;
+ if frame_count > MAX_CELLS {
+ return Err("WAL has too many frames".into());
+ }
+ let mut overlay = HashMap::new();
+ let mut page_count = None;
+ for frame_no in 0..frame_count {
+ let start = frame_no.checked_mul(stride).ok_or("WAL frame offset overflow")?;
+ let end = start.checked_add(stride).ok_or("WAL frame offset overflow")?;
+ let frame = body.get(start..end).ok_or("truncated WAL frame")?;
+ let page_id = be_u32(frame, 0)?;
+ if page_id == 0 || page_id > MAX_PAGE_ID || salt != (be_u32(frame, 8)?, be_u32(frame, 12)?) {
+ return Err("invalid WAL frame page or salt".into());
+ }
+ let commit = be_u32(frame, 4)?;
+ if commit != 0 {
+ if commit > MAX_PAGE_ID {
+ return Err("WAL commit database size is too large".into());
+ }
+ page_count = Some(commit);
+ }
+ overlay.insert(page_id, frame.get(24..).ok_or("truncated WAL page")?.to_vec());
+ }
+ Ok(WalOverlay { page_size, frame_count, pages: overlay, page_count })
+}
+
+fn decode_record(payload: &[u8], ncols: usize, encoding: TextEncoding) -> Result<Vec<Value>, String> {
+ let mut at = 0;
+ let header_len = usize_from_u64(read_varint(payload, &mut at)?, "record header length")?;
+ if header_len < at || header_len > payload.len() {
+ return Err("invalid SQLite record header length".into());
+ }
+ let mut serials = Vec::new();
+ while at < header_len {
+ if serials.len() >= ncols {
+ return Err("SQLite record has more columns than table schema".into());
+ }
+ serials.push(read_varint(payload, &mut at)?);
+ }
+ if at != header_len {
+ return Err("SQLite record header is malformed".into());
+ }
+ let mut body = header_len;
+ let mut values = Vec::with_capacity(ncols);
+ for serial in serials {
+ let (value, size) = serial_value(payload, &mut body, serial, encoding)?;
+ values.push(value);
+ if values.len() > MAX_COLUMNS {
+ return Err("SQLite record has too many columns".into());
+ }
+ let _ = size;
+ }
+ if body > payload.len() {
+ return Err("SQLite record body is truncated".into());
+ }
+ values.resize(ncols, Value::Null);
+ Ok(values)
+}
+
+fn serial_value(payload: &[u8], body: &mut usize, serial: u64, encoding: TextEncoding) -> Result<(Value, usize), String> {
+ let (kind, len) = match serial {
+ 0 => (0, 0),
+ 1 => (1, 1),
+ 2 => (2, 2),
+ 3 => (3, 3),
+ 4 => (4, 4),
+ 5 => (5, 6),
+ 6 => (6, 8),
+ 7 => (7, 8),
+ 8 => (8, 0),
+ 9 => (9, 0),
+ 10 | 11 => return Err("reserved SQLite serial type".into()),
+ n if n >= 12 => {
+ let len = usize_from_u64((n - if n % 2 == 0 { 12 } else { 13 }) / 2, "SQLite field length")?;
+ (if n % 2 == 0 { 12 } else { 13 }, len)
+ }
+ _ => return Err("invalid SQLite serial type".into()),
+ };
+ let end = body.checked_add(len).ok_or("SQLite record body offset overflow")?;
+ let bytes = payload.get(*body..end).ok_or("SQLite record body is truncated")?;
+ let value = match kind {
+ 0 => Value::Null,
+ 1 => Value::Integer(i64::from(i8::from_be_bytes([*bytes.first().ok_or("truncated integer")?]))),
+ 2 => Value::Integer(i64::from(i16::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))),
+ 3 => {
+ let first = *bytes.first().ok_or("truncated integer")?;
+ let second = *bytes.get(1).ok_or("truncated integer")?;
+ let third = *bytes.get(2).ok_or("truncated integer")?;
+ let n = (u32::from(first) << 16) | (u32::from(second) << 8) | u32::from(third);
+ let signed = if n & 0x80_0000 != 0 { n | 0xff00_0000 } else { n };
+ Value::Integer(i64::from(i32::from_be_bytes(signed.to_be_bytes())))
+ }
+ 4 => Value::Integer(i64::from(i32::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))),
+ 5 => {
+ let n = bytes.iter().fold(0u64, |acc, b| (acc << 8) | u64::from(*b));
+ let signed = if n & (1 << 47) != 0 { n | 0xffff_0000_0000_0000 } else { n };
+ Value::Integer(i64::from_be_bytes(signed.to_be_bytes()))
+ }
+ 6 => Value::Integer(i64::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?)),
+ 7 => {
+ let n = f64::from_be_bytes(bytes.try_into().map_err(|_| "truncated real")?);
+ if !n.is_finite() {
+ return Err("non-finite SQLite real value".into());
+ }
+ Value::Real(n)
+ }
+ 8 => Value::Integer(0),
+ 9 => Value::Integer(1),
+ 12 => Value::Blob(bytes.to_vec()),
+ 13 => Value::Text(decode_text(bytes, encoding)?),
+ _ => return Err("invalid SQLite field type".into()),
+ };
+ *body = end;
+ Ok((value, len))
+}
+
+fn decode_text(bytes: &[u8], encoding: TextEncoding) -> Result<String, String> {
+ match encoding {
+ TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| "invalid UTF-8 SQLite text".into()),
+ TextEncoding::Utf16Le | TextEncoding::Utf16Be => {
+ if bytes.len() % 2 != 0 {
+ return Err("odd-length UTF-16 SQLite text".into());
+ }
+ let units: Result<Vec<u16>, String> = bytes
+ .as_chunks::<2>()
+ .0
+ .iter()
+ .map(|pair| {
+ let first = *pair.first().ok_or("truncated UTF-16 SQLite text")?;
+ let second = *pair.get(1).ok_or("truncated UTF-16 SQLite text")?;
+ Ok(match encoding {
+ TextEncoding::Utf16Le => u16::from_le_bytes([first, second]),
+ TextEncoding::Utf16Be => u16::from_be_bytes([first, second]),
+ TextEncoding::Utf8 => return Err("invalid UTF-16 encoding state".into()),
+ })
+ })
+ .collect();
+ String::from_utf16(&units?).map_err(|_| "invalid UTF-16 SQLite text".into())
+ }
+ }
+}
+
+fn parse_create_table(sql: &str) -> Result<(Vec<String>, Option<usize>), String> {
+ let tokens = tokenize_sql(sql)?;
+ if !tokens.first().is_some_and(|t| t.word_is("CREATE")) {
+ return Err("malformed CREATE TABLE SQL".into());
+ }
+ let mut kind_at = 1usize;
+ if tokens.get(kind_at).is_some_and(|t| t.word_is("TEMPORARY") || t.word_is("TEMP")) {
+ kind_at += 1;
+ }
+ if tokens.get(kind_at).is_some_and(|t| t.word_is("VIRTUAL")) {
+ return Err("virtual tables are unsupported".into());
+ }
+ if !tokens.get(kind_at).is_some_and(|t| t.word_is("TABLE")) {
+ return Err("malformed CREATE TABLE SQL".into());
+ }
+ let open = tokens.iter().position(|t| t.is_symbol(b'(')).ok_or("CREATE TABLE has no column list")?;
+ let mut depth = 0usize;
+ let mut close = None;
+ for (i, token) in tokens.iter().enumerate().skip(open) {
+ if token.is_symbol(b'(') {
+ depth += 1;
+ } else if token.is_symbol(b')') {
+ depth = depth.checked_sub(1).ok_or("unbalanced CREATE TABLE parentheses")?;
+ if depth == 0 {
+ close = Some(i);
+ break;
+ }
+ }
+ }
+ let close = close.ok_or("unterminated CREATE TABLE column list")?;
+ if tokens
+ .get(close + 1..)
+ .ok_or("invalid CREATE TABLE suffix")?
+ .windows(2)
+ .any(|w| w.first().is_some_and(|t| t.word_is("WITHOUT")) && w.get(1).is_some_and(|t| t.word_is("ROWID")))
+ {
+ return Err("WITHOUT ROWID tables are unsupported".into());
+ }
+ let body = tokens.get(open + 1..close).ok_or("invalid CREATE TABLE column list")?;
+ let mut columns = Vec::new();
+ let mut alias = None;
+ let mut integer_columns = HashSet::new();
+ let mut table_primary = None;
+ let mut start = 0usize;
+ let mut nested = 0usize;
+ for i in 0..=body.len() {
+ let comma = i == body.len() || (nested == 0 && body.get(i).is_some_and(|t| t.is_symbol(b',')));
+ if comma {
+ let part = body.get(start..i).ok_or("invalid CREATE TABLE column range")?;
+ if part.is_empty() {
+ return Err("empty CREATE TABLE column definition".into());
+ }
+ if is_table_constraint(part) {
+ if table_primary.is_none() {
+ table_primary = table_primary_column(part);
+ }
+ } else {
+ let name = part.first().and_then(Token::identifier).ok_or("column name is not an identifier")?;
+ if columns.iter().any(|old| old == name) {
+ return Err("duplicate SQLite column name".into());
+ }
+ let index = columns.len();
+ if exact_integer_decl(part) {
+ integer_columns.insert(name.to_ascii_lowercase());
+ }
+ if integer_primary_key_alias(part) {
+ alias = Some(index);
+ }
+ columns.push(name.to_string());
+ }
+ start = i + 1;
+ } else if body.get(i).is_some_and(|token| token.is_symbol(b'(')) {
+ nested += 1;
+ } else if body.get(i).is_some_and(|token| token.is_symbol(b')')) {
+ nested = nested.checked_sub(1).ok_or("unbalanced CREATE TABLE column definition")?;
+ }
+ }
+ if columns.is_empty() {
+ return Err("CREATE TABLE has no columns".into());
+ }
+ if alias.is_none() {
+ if let Some(primary) = table_primary {
+ let key = primary.to_ascii_lowercase();
+ if integer_columns.contains(&key) {
+ alias = columns.iter().position(|column| column.eq_ignore_ascii_case(&primary));
+ }
+ }
+ }
+ Ok((columns, alias))
+}
+
+fn is_table_constraint(part: &[Token]) -> bool {
+ part.first().is_some_and(|token| ["CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN"].iter().any(|word| token.word_is(word)))
+}
+
+fn exact_integer_decl(part: &[Token]) -> bool {
+ if !part.get(1).is_some_and(|token| token.word_is("INTEGER")) {
+ return false;
+ }
+ match part.get(2) {
+ None => true,
+ Some(token) if token.is_symbol(b'(') => false,
+ Some(token) => ["PRIMARY", "NOT", "NULL", "UNIQUE", "CHECK", "DEFAULT", "COLLATE", "REFERENCES", "CONSTRAINT", "GENERATED"]
+ .iter()
+ .any(|word| token.word_is(word)),
+ }
+}
+
+fn table_primary_column(part: &[Token]) -> Option<String> {
+ let primary = part.iter().position(|token| token.word_is("PRIMARY"))?;
+ if !part.get(primary + 1).is_some_and(|token| token.word_is("KEY")) {
+ return None;
+ }
+ let open = primary + 2;
+ if !part.get(open).is_some_and(|token| token.is_symbol(b'(')) {
+ return None;
+ }
+ let close = (open + 1..part.len()).find(|index| part.get(*index).is_some_and(|token| token.is_symbol(b')')))?;
+ let terms = part.get(open + 1..close)?;
+ if terms.len() == 1 || (terms.len() == 2 && terms.get(1).is_some_and(|token| token.word_is("ASC") || token.word_is("DESC"))) {
+ return terms.first().and_then(Token::identifier).map(str::to_owned);
+ }
+ None
+}
+
+fn integer_primary_key_alias(part: &[Token]) -> bool {
+ if !exact_integer_decl(part) {
+ return false;
+ }
+ let mut primary = None;
+ let mut depth = 0usize;
+ for i in 2..part.len() {
+ if part.get(i).is_some_and(|token| token.is_symbol(b'(')) {
+ depth += 1;
+ } else if part.get(i).is_some_and(|token| token.is_symbol(b')')) {
+ depth = depth.saturating_sub(1);
+ } else if depth == 0 && part.get(i).is_some_and(|token| token.word_is("PRIMARY")) && part.get(i + 1).is_some_and(|t| t.word_is("KEY")) {
+ primary = Some(i);
+ break;
+ }
+ }
+ let Some(i) = primary else { return false };
+ !part.get(i + 2).is_some_and(|token| token.word_is("DESC")) && !part.get(i + 2).is_some_and(|token| token.is_symbol(b'('))
+}
+
+#[derive(Clone, Debug)]
+enum Token {
+ Word(String),
+ Quoted(String),
+ StringLiteral,
+ Symbol(u8),
+}
+
+impl Token {
+ fn is_symbol(&self, symbol: u8) -> bool {
+ matches!(self, Token::Symbol(value) if *value == symbol)
+ }
+
+ fn word_is(&self, wanted: &str) -> bool {
+ matches!(self, Token::Word(word) if word.eq_ignore_ascii_case(wanted))
+ }
+
+ fn identifier(&self) -> Option<&str> {
+ match self {
+ Token::Word(word) | Token::Quoted(word) => Some(word),
+ _ => None,
+ }
+ }
+}
+
+fn tokenize_sql(sql: &str) -> Result<Vec<Token>, String> {
+ let bytes = sql.as_bytes();
+ let mut out = Vec::new();
+ let mut at = 0usize;
+ while at < bytes.len() {
+ if bytes.get(at).is_some_and(u8::is_ascii_whitespace) {
+ at += 1;
+ } else if bytes.get(at) == Some(&b'-') && bytes.get(at + 1) == Some(&b'-') {
+ at += 2;
+ while at < bytes.len() && bytes.get(at) != Some(&b'\n') {
+ at += 1;
+ }
+ } else if bytes.get(at) == Some(&b'/') && bytes.get(at + 1) == Some(&b'*') {
+ at += 2;
+ let mut closed = false;
+ while at + 1 < bytes.len() {
+ if bytes.get(at) == Some(&b'*') && bytes.get(at + 1) == Some(&b'/') {
+ at += 2;
+ closed = true;
+ break;
+ }
+ at += 1;
+ }
+ if !closed {
+ return Err("unterminated SQL comment".into());
+ }
+ } else if matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';')) {
+ out.push(Token::Symbol(*bytes.get(at).ok_or("truncated SQL punctuation")?));
+ at += 1;
+ } else if matches!(bytes.get(at), Some(b'"' | b'`' | b'[')) {
+ let quote = *bytes.get(at).ok_or("truncated SQL quote")?;
+ let end_quote = if quote == b'[' { b']' } else { quote };
+ at += 1;
+ let mut value = Vec::new();
+ let mut closed = false;
+ while at < bytes.len() {
+ if bytes.get(at) == Some(&end_quote) {
+ if bytes.get(at + 1) == Some(&end_quote) {
+ value.push(end_quote);
+ at += 2;
+ } else {
+ at += 1;
+ closed = true;
+ break;
+ }
+ } else {
+ value.push(*bytes.get(at).ok_or("truncated SQL identifier")?);
+ at += 1;
+ }
+ }
+ if !closed {
+ return Err("unterminated quoted SQL identifier".into());
+ }
+ out.push(Token::Quoted(String::from_utf8(value).map_err(|_| "invalid SQL identifier encoding")?));
+ } else if bytes.get(at) == Some(&b'\'') {
+ at += 1;
+ let mut closed = false;
+ while at < bytes.len() {
+ if bytes.get(at) == Some(&b'\'') {
+ if bytes.get(at + 1) == Some(&b'\'') {
+ at += 2;
+ } else {
+ at += 1;
+ closed = true;
+ break;
+ }
+ } else {
+ at += 1;
+ }
+ }
+ if !closed {
+ return Err("unterminated SQL string".into());
+ }
+ out.push(Token::StringLiteral);
+ } else {
+ let start = at;
+ while at < bytes.len()
+ && !bytes.get(at).is_some_and(u8::is_ascii_whitespace)
+ && !matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';' | b'\'' | b'"' | b'`' | b'['))
+ {
+ at += 1;
+ }
+ if start == at {
+ return Err("unsupported SQL punctuation".into());
+ }
+ out.push(Token::Word(sql.get(start..at).ok_or("invalid SQL token boundary")?.to_string()));
+ }
+ }
+ Ok(out)
+}
+
+fn sqlite_page_size(value: u16) -> Result<usize, String> {
+ let size = if value == 1 { 65_536 } else { usize::from(value) };
+ if !(512..=65_536).contains(&size) || !size.is_power_of_two() {
+ return Err("invalid SQLite page size".into());
+ }
+ Ok(size)
+}
+
+fn read_varint(bytes: &[u8], at: &mut usize) -> Result<u64, String> {
+ let mut value = 0u64;
+ for i in 0..9 {
+ let byte = *bytes.get(*at).ok_or("truncated SQLite varint")?;
+ *at = (*at).checked_add(1).ok_or("SQLite varint offset overflow")?;
+ if i == 8 {
+ value = (value << 8) | u64::from(byte);
+ return Ok(value);
+ }
+ value = (value << 7) | u64::from(byte & 0x7f);
+ if byte & 0x80 == 0 {
+ return Ok(value);
+ }
+ }
+ Err("invalid SQLite varint".into())
+}
+
+fn usize_from_u64(value: u64, label: &str) -> Result<usize, String> {
+ usize::try_from(value).map_err(|_| format!("{label} exceeds host size"))
+}
+
+fn be_u16(bytes: &[u8], at: usize) -> Result<u16, String> {
+ let end = at.checked_add(2).ok_or("SQLite integer offset overflow")?;
+ bytes.get(at..end).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite integer".into())
+}
+
+fn be_u16_at(bytes: &[u8], at: usize) -> Result<u16, String> {
+ be_u16(bytes, at)
+}
+
+fn be_u32(bytes: &[u8], at: usize) -> Result<u32, String> {
+ let end = at.checked_add(4).ok_or("SQLite integer offset overflow")?;
+ bytes.get(at..end).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated SQLite integer".into())
+}
+
+fn be_u32_at(bytes: &[u8], at: usize) -> Result<u32, String> {
+ be_u32(bytes, at)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn varint(mut value: u64) -> Vec<u8> {
+ let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)];
+ value >>= 7;
+ while value != 0 {
+ out.push(u8::try_from(value & 0x7f).unwrap_or(0) | 0x80);
+ value >>= 7;
+ }
+ out.reverse();
+ out
+ }
+
+ fn leaf(page: &mut [u8], base: usize, rowid: u64, payload: &[u8]) {
+ page[base] = 0x0d;
+ page[base + 3..base + 5].copy_from_slice(&1u16.to_be_bytes());
+ let mut cell = varint(payload.len() as u64);
+ cell.extend(varint(rowid));
+ cell.extend(payload);
+ let start = page.len() - cell.len();
+ page[start..].copy_from_slice(&cell);
+ page[base + 8..base + 10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());
+ page[base + 5..base + 7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());
+ }
+
+ #[test]
+ fn schema_names_and_integer_primary_key_alias_are_decoded() {
+ let sql = r#"CREATE TABLE "t" ("id_local" INTEGER PRIMARY KEY, "odd,name" TEXT, PRIMARY KEY ("id_local"))"#;
+ let (names, alias) = parse_create_table(sql).unwrap();
+ assert_eq!(names, vec!["id_local", "odd,name"]);
+ assert_eq!(alias, Some(0));
+ let (_, no_alias) = parse_create_table("CREATE TABLE t (id INTEGER PRIMARY KEY DESC, value TEXT)").unwrap();
+ assert_eq!(no_alias, None);
+ let (table_names, table_alias) = parse_create_table("CREATE TABLE t (id INTEGER, value TEXT, PRIMARY KEY(id))").unwrap();
+ assert_eq!(table_names, vec!["id", "value"]);
+ assert_eq!(table_alias, Some(0));
+ }
+
+ #[test]
+ fn synthetic_btree_substitutes_rowid_and_reads_schema() {
+ let mut main = vec![0u8; 2 * 512];
+ main[..16].copy_from_slice(b"SQLite format 3\0");
+ main[16..18].copy_from_slice(&512u16.to_be_bytes());
+ main[18] = 1;
+ main[19] = 1;
+ main[21] = 64;
+ main[22] = 32;
+ main[23] = 32;
+ main[28..32].copy_from_slice(&2u32.to_be_bytes());
+ main[44..48].copy_from_slice(&4u32.to_be_bytes());
+ main[56..60].copy_from_slice(&1u32.to_be_bytes());
+ let sql = "CREATE TABLE \"t\" (\"id_local\" INTEGER PRIMARY KEY, \"name\" TEXT)";
+ // rootpage is an integer field, so replace fourth text value with a compact record.
+ let mut header = vec![23, 15, 15, 1, u8::try_from(13 + 2 * sql.len()).unwrap_or(0)];
+ let mut body = b"tablett".to_vec();
+ body.push(2);
+ body.extend(sql.as_bytes());
+ let mut record = varint((1 + header.len()) as u64);
+ record.append(&mut header);
+ record.extend(body);
+ let schema = record;
+ leaf(&mut main[512..], 0, 1, &[3, 0, 15, b'x']);
+ leaf(&mut main[..512], 100, 1, &schema);
+ let db = Database::open_with_wal(main, &[]).unwrap();
+ let tables = db.live_tables().unwrap();
+ assert!(tables.iter().any(|table| table.name == "t"));
+ let rows = db.read_table(2, 2).unwrap();
+ assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1)));
+ }
+
+ #[test]
+ fn interior_table_pointer_array_uses_cell_offsets() {
+ let mut main = vec![0u8; 4 * 512];
+ let mut interior = vec![0u8; 512];
+ interior[0] = 0x05;
+ interior[3..5].copy_from_slice(&1u16.to_be_bytes());
+ interior[8..12].copy_from_slice(&4u32.to_be_bytes());
+ interior[12..14].copy_from_slice(&500u16.to_be_bytes());
+ interior[500..504].copy_from_slice(&3u32.to_be_bytes());
+ interior[504] = 1;
+ main[512..1024].copy_from_slice(&interior);
+ leaf(&mut main[1024..1536], 0, 1, &[2, 0]);
+ leaf(&mut main[1536..], 0, 2, &[2, 0]);
+ let db = Database {
+ main,
+ overlay: HashMap::new(),
+ page_size: 512,
+ reserved: 0,
+ encoding: TextEncoding::Utf8,
+ page_count: 4,
+ tables: Vec::new(),
+ rowid_aliases: HashMap::new(),
+ };
+ assert_eq!(db.read_table(2, 1).unwrap().len(), 2);
+ }
+
+ #[test]
+ fn negative_rowid_varint_is_signed() {
+ let mut main = vec![0u8; 512];
+ let page = &mut main[..];
+ page[100] = 0x0d;
+ page[103..105].copy_from_slice(&1u16.to_be_bytes());
+ let start = 500usize;
+ page[start] = 2;
+ for offset in 1..10 {
+ if let Some(byte) = page.get_mut(start + offset) {
+ *byte = 0xff;
+ }
+ }
+ page[start + 10..start + 12].copy_from_slice(&[2, 0]);
+ page[108..110].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());
+ page[105..107].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());
+ let mut aliases = HashMap::new();
+ aliases.insert(1, 0);
+ let db = Database {
+ main,
+ overlay: HashMap::new(),
+ page_size: 512,
+ reserved: 0,
+ encoding: TextEncoding::Utf8,
+ page_count: 1,
+ tables: Vec::new(),
+ rowid_aliases: aliases,
+ };
+ let rows = db.read_table(1, 1).unwrap();
+ assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(-1)));
+ }
+
+ #[test]
+ fn reserved_page_trailer_cannot_supply_cells() {
+ let mut main = vec![0u8; 512];
+ main[100] = 0x0d;
+ main[103..105].copy_from_slice(&1u16.to_be_bytes());
+ main[108..110].copy_from_slice(&490u16.to_be_bytes());
+ let db = Database {
+ main,
+ overlay: HashMap::new(),
+ page_size: 512,
+ reserved: 32,
+ encoding: TextEncoding::Utf8,
+ page_count: 1,
+ tables: Vec::new(),
+ rowid_aliases: HashMap::new(),
+ };
+ assert!(db.read_table(1, 1).is_err());
+ }
+
+ #[test]
+ fn utf16_and_overflow_payloads_are_supported() {
+ let mut payload = vec![2, 17, b'A', 0];
+ assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Le).unwrap(), vec![Value::Text("A".into())]);
+ payload.clear();
+ payload.extend([2, 17, 0, b'A']);
+ assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Be).unwrap(), vec![Value::Text("A".into())]);
+
+ let mut main = vec![0u8; 2 * 512];
+ let db = Database {
+ main: std::mem::take(&mut main),
+ overlay: HashMap::new(),
+ page_size: 512,
+ reserved: 0,
+ encoding: TextEncoding::Utf8,
+ page_count: 2,
+ tables: Vec::new(),
+ rowid_aliases: HashMap::new(),
+ };
+ let mut cell_page = vec![b'x'; 512];
+ cell_page[100..104].copy_from_slice(&2u32.to_be_bytes());
+ let inline_page = vec![b'i'; 512];
+ assert_eq!(db.payload(&inline_page, 0, 110).unwrap(), vec![b'i'; 110]);
+ let overflow_page = vec![b'y'; 512];
+ let mut db = db;
+ db.main[512..].copy_from_slice(&overflow_page);
+ let mut with_overflow = db.overlay;
+ with_overflow.insert(2, overflow_page);
+ db.overlay = with_overflow;
+ let bytes = db.payload(&cell_page, 0, 600).unwrap();
+ assert_eq!(bytes.len(), 600);
+ }
+
+ #[test]
+ fn malformed_varint_and_wal_cycle_fail_cleanly() {
+ let mut at = 0;
+ assert!(read_varint(&[0x80; 8], &mut at).is_err());
+ let mut wal = vec![0u8; 32 + 24 + 512];
+ wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes());
+ wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes());
+ wal[8..12].copy_from_slice(&512u32.to_be_bytes());
+ wal[16..20].copy_from_slice(&7u32.to_be_bytes());
+ wal[20..24].copy_from_slice(&8u32.to_be_bytes());
+ wal[32..36].copy_from_slice(&1u32.to_be_bytes());
+ wal[36..40].copy_from_slice(&1u32.to_be_bytes());
+ wal[40..44].copy_from_slice(&7u32.to_be_bytes());
+ wal[44..48].copy_from_slice(&8u32.to_be_bytes());
+ let parsed = parse_wal(&wal, 512).unwrap();
+ assert_eq!(parsed.frame_count, 1);
+ assert_eq!(parsed.page_count, Some(1));
+ assert_eq!(parsed.pages.get(&1).map(Vec::len), Some(512));
+ let mut page = vec![0u8; 512];
+ page[16..18].copy_from_slice(&512u16.to_be_bytes());
+ let db = Database {
+ main: vec![0; 512],
+ overlay: HashMap::new(),
+ page_size: 512,
+ reserved: 0,
+ encoding: TextEncoding::Utf8,
+ page_count: 1,
+ tables: Vec::new(),
+ rowid_aliases: HashMap::new(),
+ };
+ assert!(db.payload(&page, 0, 600).is_err());
+ }
+}
diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs
index 1b7b99a44..1252fdb35 100644
--- a/crates/ui-egui/src/lib.rs
+++ b/crates/ui-egui/src/lib.rs
@@ -13,6 +13,7 @@ pub mod headless;
pub mod i18n;
pub mod icons;
pub mod import;
+pub mod lightroom_import;
pub mod links;
pub mod menubar;
pub mod menus;
@@ -187,6 +188,10 @@ pub struct LightcraftApp {
pub import: Option<import::ImportTask>,
/// A folder scan in progress (feeds the import review).
pub scan: Option<import::ScanTask>,
+ /// A Lightroom catalog inspect/import in progress.
+ pub lightroom: Option<lightroom_import::LightroomTask>,
+ /// Last terminal Lightroom result, exposed by the command's status/wait response.
+ pub lightroom_last: Option<Value>,
/// A background export in progress.
pub export: Option<export_task::ExportTask>,
/// Background file-system work of other commands (Find Missing Photos, auto import…).
@@ -246,6 +251,8 @@ impl LightcraftApp {
merge: merge::MergeState::default(),
import: None,
scan: None,
+ lightroom: None,
+ lightroom_last: None,
export: None,
tasks: Default::default(),
last_export_result: None,
@@ -621,6 +628,7 @@ impl LightcraftApp {
merge::poll(self, ctx);
import::poll_scan(self, ctx);
import::tick(self, ctx);
+ lightroom_import::tick(self, ctx);
tasks::poll(self, ctx);
self.preview_build_status(ctx);
self.save_status(ctx);
@@ -642,7 +650,7 @@ impl LightcraftApp {
if let Some(folder) = self.session.import_defaults.auto_folder.clone() {
const LABEL: &str = "Auto Import";
let now = ctx.input(|i| i.time);
- if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) {
+ if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && self.lightroom.is_none() && !self.tasks.is_running(LABEL) {
self.ui.auto_import_at = now;
let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder);
let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result<Vec<(String, u64)>, String>| {
@@ -683,10 +691,12 @@ impl LightcraftApp {
let _ = self.run("file.importPresets", serde_json::json!({"paths": presets}));
}
// read and added on a worker thread (dropped folders can be large, or on a slow drive)
- if !photos.is_empty()
- && let Err(e) = import::start_paths(self, photos)
- {
- self.toast(ctx, e);
+ if !photos.is_empty() {
+ if self.lightroom.is_some() {
+ self.toast(ctx, "Finish Lightroom catalog import before adding photos");
+ } else if let Err(e) = import::start_paths(self, photos) {
+ self.toast(ctx, e);
+ }
}
}
}
@@ -823,6 +833,7 @@ impl LightcraftApp {
panels::library_problem::show(self, &ctx);
import::progress(self, &ctx);
import::scan_progress(self, &ctx);
+ lightroom_import::progress(self, &ctx);
export_task::poll(self, &ctx);
panels::grid::drag_feedback(self, &ctx);
panels::toast(self, &ctx);
diff --git a/crates/ui-egui/src/lightroom_import.rs b/crates/ui-egui/src/lightroom_import.rs
new file mode 100644
index 000000000..6b2e0a2d7
--- /dev/null
+++ b/crates/ui-egui/src/lightroom_import.rs
@@ -0,0 +1,456 @@
+//! Background Lightroom catalog inspection and import.
+//!
+//! Reading a catalog, probing its originals and staging its archive can take minutes on a
+//! removable or network drive. This task keeps that work off the owner thread, commits prepared
+//! data only on the owner thread, then finishes the archive index on a worker.
+
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Arc, mpsc};
+
+use egui::Align2;
+use serde_json::{Value, json};
+
+use crate::LightcraftApp;
+use crate::theme::Tokens;
+use crate::widgets::register;
+
+#[derive(Clone, Copy)]
+enum Kind {
+ Inspect,
+ Import,
+}
+
+enum Message {
+ Inspected(Result<Value, String>),
+ Prepared(Result<lightcraft_engine::lightroom_job::PreparedLightroom, String>),
+ Finalized(Result<(), String>),
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum Phase {
+ Reading,
+ Committing,
+ Finalizing,
+}
+
+/// A Lightroom task in flight. The receiver is polled from the egui owner thread.
+pub struct LightroomTask {
+ kind: Kind,
+ path: PathBuf,
+ cancel: Arc<AtomicBool>,
+ total: Arc<AtomicUsize>,
+ done: Arc<AtomicUsize>,
+ rx: mpsc::Receiver<Message>,
+ phase: Phase,
+ cancelled: bool,
+}
+
+impl LightroomTask {
+ #[cfg(not(target_arch = "wasm32"))]
+ fn status(&self) -> Value {
+ let kind = match self.kind {
+ Kind::Inspect => "inspect",
+ Kind::Import => "import",
+ };
+ let phase = match self.phase {
+ Phase::Reading => "reading",
+ Phase::Committing => "committing",
+ Phase::Finalizing => "finalizing",
+ };
+ json!({
+ "running": true,
+ "kind": kind,
+ "path": self.path,
+ "phase": phase,
+ "done": self.done.load(Ordering::Relaxed),
+ "total": self.total.load(Ordering::Relaxed),
+ "cancelled": self.cancelled || self.cancel.load(Ordering::Relaxed),
+ })
+ }
+
+ fn cancel(&mut self) {
+ self.cancelled = true;
+ self.cancel.store(true, Ordering::Relaxed);
+ }
+}
+
+impl Drop for LightroomTask {
+ fn drop(&mut self) {
+ self.cancel.store(true, Ordering::Relaxed);
+ }
+}
+
+#[cfg(target_arch = "wasm32")]
+fn unsupported_wasm() -> Result<Value, String> {
+ Err("Lightroom catalog import is unavailable in browser builds; open the catalog in the native LightCraft app".into())
+}
+
+/// Whether a Lightroom task currently owns the catalog transition.
+pub fn is_running(app: &LightcraftApp) -> bool {
+ app.lightroom.is_some()
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn busy(app: &LightcraftApp) -> Result<(), String> {
+ if app.lightroom.is_some() {
+ return Err("a Lightroom catalog task is already running".into());
+ }
+ if app.import.is_some() || app.scan.is_some() {
+ return Err("an import is running".into());
+ }
+ Ok(())
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn spawn_inspect(
+ path: PathBuf,
+ cancel: Arc<AtomicBool>,
+ total: Arc<AtomicUsize>,
+ done: Arc<AtomicUsize>,
+ tx: mpsc::Sender<Message>,
+ ctx: egui::Context,
+) -> Result<(), String> {
+ std::thread::Builder::new()
+ .name("lc-lightroom-inspect".into())
+ .spawn(move || {
+ let result = lightcraft_engine::guard::catch("Lightroom inspect", || {
+ lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &total, &done)
+ })
+ .and_then(|r| r.map(inspect_report));
+ let _ = tx.send(Message::Inspected(result));
+ ctx.request_repaint();
+ })
+ .map(|_| ())
+ .map_err(|e| format!("could not start Lightroom inspect: {e}"))
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn spawn_prepare(
+ mut job: lightcraft_engine::lightroom_job::LightroomJob,
+ cancel: Arc<AtomicBool>,
+ tx: mpsc::Sender<Message>,
+ ctx: egui::Context,
+) -> Result<(), String> {
+ std::thread::Builder::new()
+ .name("lc-lightroom-import".into())
+ .spawn(move || {
+ let result = lightcraft_engine::guard::catch("Lightroom import", || job.prepare(&cancel)).and_then(|r| r.map_err(|e| e.to_string()));
+ if let Err(mpsc::SendError(Message::Prepared(Ok(prepared)))) = tx.send(Message::Prepared(result)) {
+ // The owner dropped the task (for example while closing): staged copies/moves
+ // must be put back even though no UI receiver remains to accept the result.
+ prepared.rollback();
+ }
+ ctx.request_repaint();
+ })
+ .map(|_| ())
+ .map_err(|e| format!("could not start Lightroom import: {e}"))
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn spawn_finalize(
+ finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,
+ tx: mpsc::Sender<Message>,
+ ctx: egui::Context,
+) -> Result<(), String> {
+ std::thread::Builder::new()
+ .name("lc-lightroom-finalize".into())
+ .spawn(move || {
+ let result = lightcraft_engine::guard::catch("Lightroom archive finalization", || finalization.finish())
+ .and_then(|r| r.map_err(|e| e.to_string()));
+ let _ = tx.send(Message::Finalized(result));
+ ctx.request_repaint();
+ })
+ .map(|_| ())
+ .map_err(|e| format!("could not start Lightroom archive finalization: {e}"))
+}
+
+#[cfg(target_arch = "wasm32")]
+fn spawn_finalize(
+ _finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,
+ _tx: mpsc::Sender<Message>,
+ _ctx: egui::Context,
+) -> Result<(), String> {
+ unsupported_wasm().map(|_| ())
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn inspect_report(data: lightcraft_engine::lightroom_catalog::CatalogImport) -> Value {
+ json!({
+ "photos": data.photos.len(),
+ "collections": data.collections.iter().filter(|r| r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0) == 0.0).count(),
+ "missing": data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p| &p.path).collect::<Vec<_>>(),
+ "warnings": data.warnings,
+ })
+}
+
+fn terminal(app: &mut LightcraftApp, ctx: &egui::Context, kind: Kind, value: Value) {
+ let message = if let Some(e) = value.get("error").and_then(Value::as_str) {
+ let warning = value.get("indexWarning").and_then(Value::as_str).map_or(String::new(), |w| format!("; index warning: {w}"));
+ format!("Lightroom {} failed: {e}{warning}", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })
+ } else if value.get("cancelled").and_then(Value::as_bool).unwrap_or(false) {
+ format!("Lightroom {} cancelled", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })
+ } else if matches!(kind, Kind::Inspect) {
+ let photos = value["photos"].as_u64().unwrap_or(0);
+ let collections = value["collections"].as_u64().unwrap_or(0);
+ let missing = value["missing"].as_array().map_or(0, Vec::len);
+ format!("Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing")
+ } else {
+ let report = value.get("report").unwrap_or(&value);
+ let imported = report["imported"].as_u64().unwrap_or(0);
+ let failed = report["failed"].as_array().map_or(0, Vec::len);
+ let warning = value.get("indexWarning").and_then(Value::as_str);
+ if let Some(warning) = warning {
+ format!("Lightroom import complete: {imported} photos added; index warning: {warning}")
+ } else if failed == 0 {
+ format!("Lightroom import complete: {imported} photos added")
+ } else {
+ format!("Lightroom import complete: {imported} photos added, {failed} failed")
+ }
+ };
+ app.lightroom_last = Some(value);
+ app.lightroom = None;
+ app.toast_for(ctx, message, 6.0);
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> Result<Value, String> {
+ busy(app)?;
+ let cancel = Arc::new(AtomicBool::new(false));
+ let total = Arc::new(AtomicUsize::new(0));
+ let done = Arc::new(AtomicUsize::new(0));
+ let (tx, rx) = mpsc::channel();
+ spawn_inspect(path.clone(), cancel.clone(), total.clone(), done.clone(), tx, ctx.clone())?;
+ app.lightroom_last = None;
+ app.lightroom = Some(LightroomTask { kind: Kind::Inspect, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });
+ Ok(json!({"running": true, "kind": "inspect", "path": path}))
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn start_import(app: &mut LightcraftApp, path: PathBuf, update_existing: bool, ctx: &egui::Context) -> Result<Value, String> {
+ busy(app)?;
+ let job = lightcraft_engine::lightroom_job::LightroomJob::new(&mut app.session, path.clone(), update_existing).map_err(|e| e.to_string())?;
+ let cancel = Arc::new(AtomicBool::new(false));
+ let total = job.total_atomic();
+ let done = job.done_atomic();
+ let (tx, rx) = mpsc::channel();
+ spawn_prepare(job, cancel.clone(), tx, ctx.clone())?;
+ app.lightroom_last = None;
+ app.lightroom = Some(LightroomTask { kind: Kind::Import, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });
+ Ok(json!({"running": true, "kind": "import", "path": path}))
+}
+
+/// Run Lightroom UI commands before the generic engine dispatcher.
+pub fn command(app: &mut LightcraftApp, id: &str, p: &Value, ctx: &egui::Context) -> Result<Value, String> {
+ #[cfg(target_arch = "wasm32")]
+ {
+ let _ = (app, id, p, ctx);
+ return unsupported_wasm();
+ }
+ #[cfg(not(target_arch = "wasm32"))]
+ {
+ let path = p.get("path").and_then(Value::as_str).map(PathBuf::from);
+ let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
+ let status = p.get("status").and_then(Value::as_bool).unwrap_or(false) || path.is_none();
+ if status {
+ if p.get("cancel").and_then(Value::as_bool).unwrap_or(false) {
+ if let Some(task) = app.lightroom.as_mut() {
+ task.cancel();
+ }
+ }
+ if wait && app.lightroom.is_some() {
+ return wait_for(app, ctx, std::time::Duration::from_secs(600));
+ }
+ return Ok(app.lightroom.as_ref().map(LightroomTask::status).or_else(|| app.lightroom_last.clone()).unwrap_or(Value::Null));
+ }
+ let Some(path) = path else { return Err("missing `path`".into()) };
+ let result = if id == "library.inspectLightroom" {
+ start_inspect(app, path, ctx)
+ } else {
+ start_import(app, path, p.get("updateExisting").and_then(Value::as_bool).unwrap_or(false), ctx)
+ }?;
+ if wait {
+ return wait_for(app, ctx, std::time::Duration::from_secs(600));
+ }
+ Ok(result)
+ }
+}
+
+/// Poll owner-thread work, commit a prepared import, and finish its archive index.
+pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
+ let Some(mut task) = app.lightroom.take() else { return };
+ let message = match task.rx.try_recv() {
+ Ok(m) => m,
+ Err(mpsc::TryRecvError::Empty) => {
+ app.lightroom = Some(task);
+ return;
+ }
+ Err(mpsc::TryRecvError::Disconnected) => {
+ terminal(app, ctx, task.kind, json!({"error": "Lightroom worker stopped unexpectedly"}));
+ return;
+ }
+ };
+ match message {
+ Message::Inspected(result) => match result {
+ Ok(report) if !task.cancel.load(Ordering::Relaxed) => terminal(app, ctx, task.kind, report),
+ Ok(_) => terminal(app, ctx, task.kind, json!({"cancelled": true})),
+ Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => terminal(app, ctx, task.kind, json!({"cancelled": true})),
+ Err(e) => terminal(app, ctx, task.kind, json!({"error": e})),
+ },
+ Message::Prepared(result) => {
+ let prepared = match result {
+ Ok(p) => p,
+ Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => {
+ terminal(app, ctx, task.kind, json!({"cancelled": true}));
+ return;
+ }
+ Err(e) => {
+ terminal(app, ctx, task.kind, json!({"error": e}));
+ return;
+ }
+ };
+ if task.cancel.load(Ordering::Relaxed) || prepared.is_cancelled() || !prepared.token().matches(&app.session) {
+ prepared.rollback();
+ terminal(
+ app,
+ ctx,
+ task.kind,
+ if task.cancel.load(Ordering::Relaxed) {
+ json!({"cancelled": true})
+ } else {
+ json!({"error": "Lightroom import belongs to a different open library"})
+ },
+ );
+ return;
+ }
+ task.phase = Phase::Committing;
+ let mut finalization = None;
+ let mut report = None;
+ let committed = app.session.execute_fn("library.importLightroom", |s| {
+ let completion = lightcraft_engine::lightroom_job::commit_prepared(s, prepared)?;
+ report = Some(completion.report.clone());
+ finalization = Some(completion.finalization);
+ Ok(completion.report)
+ });
+ let report = report.unwrap_or(Value::Null);
+ task.phase = Phase::Finalizing;
+ let (tx, rx) = mpsc::channel();
+ let commit_error = committed.as_ref().err().map(ToString::to_string);
+ let Some(finalization) = finalization else {
+ terminal(app, ctx, task.kind, json!({"error": commit_error.clone().unwrap_or_else(|| "Lightroom import did not commit".into())}));
+ return;
+ };
+ if let Err(e) = spawn_finalize(finalization, tx, ctx.clone()) {
+ let mut result = json!({"report": report, "indexWarning": e});
+ if let Some(error) = commit_error {
+ result["error"] = json!(error);
+ }
+ terminal(app, ctx, task.kind, result);
+ return;
+ }
+ task.rx = rx;
+ task.cancelled = false;
+ let mut pending = json!({"running": true, "kind": "import", "report": report});
+ if let Some(error) = commit_error {
+ pending["error"] = json!(error);
+ }
+ app.lightroom_last = Some(pending);
+ app.lightroom = Some(task);
+ }
+ Message::Finalized(result) => {
+ let pending = app.lightroom_last.take().unwrap_or(Value::Null);
+ let report = pending.get("report").cloned().unwrap_or(Value::Null);
+ match result {
+ Ok(()) if pending.get("error").is_none() => terminal(app, ctx, task.kind, report),
+ Ok(()) => terminal(app, ctx, task.kind, json!({"report": report, "error": pending["error"].clone()})),
+ Err(e) => {
+ let mut value = json!({"report": report, "indexWarning": e});
+ if let Some(error) = pending.get("error") {
+ value["error"] = error.clone();
+ }
+ terminal(app, ctx, task.kind, value);
+ }
+ }
+ }
+ }
+ ctx.request_repaint();
+}
+
+/// Wait for a task in control/headless mode while still applying owner-thread completion.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn wait_for(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> Result<Value, String> {
+ let start = std::time::Instant::now();
+ while app.lightroom.is_some() {
+ tick(app, ctx);
+ if start.elapsed() >= timeout {
+ return Err("timed out waiting for Lightroom task".into());
+ }
+ std::thread::sleep(std::time::Duration::from_millis(2));
+ }
+ Ok(app.lightroom_last.clone().unwrap_or(Value::Null))
+}
+
+#[cfg(target_arch = "wasm32")]
+pub fn wait_for(_app: &mut LightcraftApp, _ctx: &egui::Context, _timeout: std::time::Duration) -> Result<Value, String> {
+ unsupported_wasm()
+}
+
+/// The progress window for inspect/import, with cancellation.
+pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) {
+ let Some(task) = app.lightroom.as_ref() else { return };
+ let total = task.total.load(Ordering::Relaxed);
+ let done = task.done.load(Ordering::Relaxed);
+ let title = match task.kind {
+ Kind::Inspect => "Inspecting Lightroom Catalog",
+ Kind::Import => "Importing Lightroom Catalog",
+ };
+ let text = match task.phase {
+ Phase::Reading if total == 0 => "Reading Lightroom catalog…".to_string(),
+ Phase::Reading => format!("Reading Lightroom catalog… {done} of {total}"),
+ Phase::Committing => "Adding Lightroom photos…".to_string(),
+ Phase::Finalizing => "Saving Lightroom import index…".to_string(),
+ };
+ let mut cancel = false;
+ let t = Tokens::get(ctx);
+ egui::Window::new(title).title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([360.0, 92.0]).show(
+ ctx,
+ |ui| {
+ ui.label(egui::RichText::new(text).color(t.text));
+ ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(340.0));
+ let r = ui.add_enabled(task.phase == Phase::Reading && !task.cancel.load(Ordering::Relaxed), egui::Button::new("Cancel"));
+ register(ui.ctx(), "button:lightroomCancel", r.rect);
+ cancel = r.clicked();
+ },
+ );
+ if cancel {
+ if let Some(task) = app.lightroom.as_mut() {
+ task.cancel();
+ }
+ ctx.request_repaint();
+ }
+ if app.lightroom.is_some() {
+ ctx.request_repaint_after(std::time::Duration::from_millis(100));
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::inspect_report;
+ use lightcraft_engine::lightroom_catalog::CatalogImport;
+
+ #[test]
+ fn inspect_report_keeps_empty_catalog_shape() {
+ let report = inspect_report(CatalogImport {
+ source: "synthetic.lrcat".into(),
+ photos: Vec::new(),
+ collections: Vec::new(),
+ members: Vec::new(),
+ collection_content: Vec::new(),
+ warnings: vec!["warning".into()],
+ });
+ assert_eq!(report["photos"], 0);
+ assert_eq!(report["collections"], 0);
+ assert_eq!(report["missing"], serde_json::json!([]));
+ assert_eq!(report["warnings"], serde_json::json!(["warning"]));
+ }
+}
diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs
index 5c841d9e3..b04ba2bd3 100644
--- a/crates/ui-egui/src/menus.rs
+++ b/crates/ui-egui/src/menus.rs
@@ -236,6 +236,10 @@ pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> {
/// Handle UI commands; `None` means "not a UI command — send it to the engine".
pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Result<Value, String>> {
+ if matches!(id, "library.inspectLightroom" | "library.importLightroom") {
+ let ctx = egui::Context::default();
+ return Some(crate::lightroom_import::command(app, id, p, &ctx));
+ }
if let Some(language) = language_from_command(id) {
app.ui.language = language;
// Immediately, not on the next frame: the reply and anything else run this frame
@@ -948,11 +952,17 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re
Ok(Value::Null)
}
"library.browse" if !cfg!(target_arch = "wasm32") => {
+ if crate::lightroom_import::is_running(app) {
+ return Some(Err("finish Lightroom catalog import before browsing folders".into()));
+ }
// listed and read in the background (see `import::browse`)
let path = p.get("path").and_then(Value::as_str)?;
crate::import::browse(app, path, p.get("subfolders").and_then(Value::as_bool))
}
"file.addPhotos" => {
+ if crate::lightroom_import::is_running(app) {
+ return Some(Err("finish Lightroom catalog import before adding photos".into()));
+ }
let paths = match p.get("paths").and_then(Value::as_array) {
Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(),
None => app.services.pick_files.as_mut().map(|f| f()).unwrap_or_default(),
@@ -1278,7 +1288,7 @@ pub fn ui_enabled(app: &LightcraftApp, id: &str) -> bool {
"file.exportCurvePresets" => !app.session.curve_presets.is_empty(),
"view.compare" => app.session.catalog.len() > 1,
"view.fullScreenPreview" | "view.infoOverlay" | "view.navigator" => app.session.active().is_some() || app.ui.fullscreen,
- "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some(),
+ "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some() && !crate::lightroom_import::is_running(app),
"file.backupLibrary" => app.services.backup_library.is_some(),
"file.restoreLibrary" => app.services.restore_library.is_some(),
"compare.swap" | "compare.makeSelect" => app.ui.view == ViewMode::Compare,
diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs
index e191668d9..745c06ad0 100644
--- a/crates/ui-egui/src/panels/settings.rs
+++ b/crates/ui-egui/src/panels/settings.rs
@@ -530,6 +530,9 @@ fn interface_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) {
/// `app.openLibrary {path?}`: close the current library and open (or create) the one at `path`,
/// or a folder chosen in a dialog. Remembered as the library to open at launch.
pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String> {
+ if crate::lightroom_import::is_running(app) {
+ return Err("wait for Lightroom catalog import to finish before switching libraries".into());
+ }
let path = match p.get("path").and_then(Value::as_str) {
Some(x) => x.to_string(),
None => match app.services.pick_folder.as_mut() {
@@ -542,6 +545,7 @@ pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String>
};
app.session.close_library().map_err(|e| e.to_string())?;
app.session.open_library(&path, false).map_err(|e| e.to_string())?;
+ app.lightroom_last = None;
app.renderer.forget_all();
app.ui.compare = None;
app.ui.settings.library_path = path.clone();
diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md
index a5412da19..694441aa1 100644
--- a/docs/lightroom-catalog-import.md
+++ b/docs/lightroom-catalog-import.md
@@ -19,10 +19,15 @@ identity prevents reimport from duplicating virtual copies and collections. One
catalog changes; recovery archives remain available.
Before changing records, source image records, decoded XMP, verbatim develop settings, history,
-snapshots and collection content are saved to `Interop/lightroom-import-N.json` in the LightCraft
-library. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals
+snapshots and collection content are saved to compressed `Interop/lightroom-import-N.lca` archives
+in the LightCraft library. Unchanged source data reuses its existing archive. Opaque SQL BLOBs are
+skipped; decoded XMP and structured recovery data remain. Each archive is capped at 32 MiB
+(uncompressed and on disk), with at most eight archives and 128 MiB total retained. Oldest managed
+archives are retired; legacy `.json` archives remain untouched. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals
are never overwritten. Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and
-one million rows per table. The SQLite reader is Apache-2.0 `sqlite-core`, written in Rust.
+one million rows per table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format specification;
+it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. Unsupported
+WITHOUT ROWID and virtual tables produce explicit errors.
Develop settings reuse the existing XMP/preset mapper. Supported sliders, curves and supported
mask structures remain editable, but this is approximate rendering: camera profiles, Adobe AI
@@ -34,5 +39,15 @@ Lightroom's `-999999` deferred-adjustment sentinel is omitted from both catalog
it is reported and archived, never clamped into a real slider value. Deferred Adobe Auto Tone
is not evaluated by the importer; LightCraft's Auto control remains available after migration.
+Native desktop inspection/import runs on a cancellable background worker with progress. Only
+prepared catalog operations commit on the owner thread; archive index writing stays on a worker.
+Late results from a closed or switched library are rejected, including reopening the same path.
+Ordinary edits during preparation remain intact. CLI commands complete synchronously using the
+same preparation/commit pipeline; browser builds reject native catalog import.
+
+Archive version 1 begins with the 12 bytes `LC-LRARCH\0\x01Z`, followed by zlib-compressed JSON.
+Recovery readers must cap decompression at 32 MiB. JSON retains source identities, photo paths,
+settings, decoded XMP, keywords, history, snapshots, collection definitions and membership.
+
The importer uses shared Rust code on macOS, Windows and Linux. If originals move between
computers or volumes, use LightCraft's missing-photo relinking tools to update their paths.
diff --git a/docs/parity.md b/docs/parity.md
index 3aee3d8e0..79ae9f5e1 100644
--- a/docs/parity.md
+++ b/docs/parity.md
@@ -716,7 +716,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06-
| LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N |
| LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo |
| LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) |
-| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-process Rust SQLite reader; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Verbatim settings/history/snapshots/collection rules archived. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs |
+| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Settings/history/snapshots/collection rules retained in deduplicated compressed archives (32 MiB each, eight archives/128 MiB aggregate); opaque SQL BLOBs skipped. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs |
| LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning; no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) |
| LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) |
| LRC-LIB-COLLECTIONS | Collections & sets | P1 | ✅ | `cmd:album.create`, `cmd:album.createSmart`, `cmd:album.toggleTarget`, `cmd:album.setTarget`, `cmd:album.clearQuick` | albums + folders (sets), smart albums (rule editor; rules incl. File Path — the whole string, either separator, case-insensitive), Quick Collection and target album (B in the grids adds / removes the selection; album menu ▸ Set as Target Album; the target is marked +) |