LightCraft: An open-source non-destructive photo editor written by scratch in pure Rust
LRC-LIB-CATALOG-IMPORT: address SQLite, worker & archive review
Adrian D'souza866d02e577cb
diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.tomlindex f59523f02..c2d4e8a80 100644--- a/crates/engine/Cargo.toml+++ b/crates/engine/Cargo.toml@@ -29,7 +29,6 @@ serde = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }log = { workspace = true }-sqlite-core = "=0.12.0"ab_glyph = { workspace = true }lightcraft-segment = { workspace = true, optional = true }diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rsindex 4895eee73..1e94226e3 100644--- a/crates/engine/src/import.rs+++ b/crates/engine/src/import.rs@@ -610,6 +610,78 @@ impl Prepared {pub fn is_empty(&self) -> bool {self.items.is_empty()}++ /// Recheck Add-only entries against the owner catalog immediately before commit.+ /// Preparation may have run while another import added the same path or content.+ pub fn revalidate_add(&mut self, s: &Session) {+ let mut paths = HashMap::new();+ let mut hashes = HashMap::new();+ for photo in s.catalog.photos() {+ if let Source::File { path } = &photo.source {+ paths.insert(normalize_import_path(path), photo.id);+ }+ if let Some(hash) = &photo.content_hash {+ hashes.insert(hash.clone(), photo.id);+ }+ }+ let mut items = Vec::with_capacity(self.items.len());+ for item in std::mem::take(&mut self.items) {+ let PreparedItem::Ready(ready) = item else {+ items.push(item);+ continue;+ };+ let ReadyFile { path, stored, info, sidecar, placed } = *ready;+ let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied());+ let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied());+ if let Some(existing) = existing_path.or(existing_hash) {+ if let Some(placed) = &placed {+ crate::import_move::rollback(placed);+ }+ items.push(PreparedItem::Duplicate {+ path,+ existing: Some(existing),+ reason: if existing_path.is_some() { "path" } else { "content" },+ hash: info.content_hash,+ });+ } else {+ items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })));+ }+ }+ self.items = items;+ }+}++fn normalize_import_path(path: &str) -> String {+ let path = path.replace('\\', "/");+ if cfg!(windows) { path.to_lowercase() } else { path }+}++#[cfg(test)]+mod prepared_tests {+ use super::*;++ #[test]+ fn revalidate_add_only_converts_matching_ready_item() {+ let mut s = Session::new();+ let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg");+ let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg");+ let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now");+ s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap();+ let ready = |path: &Path| ReadyFile {+ path: path.to_string_lossy().into(),+ stored: path.to_string_lossy().into(),+ info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() },+ sidecar: None,+ placed: None,+ };+ let mut prepared = Prepared {+ scanned: 2,+ items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))],+ };+ prepared.revalidate_add(&s);+ assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. })));+ assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_))));+ }}impl ImportJob {diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rsindex 4ca8b5bc6..122bde8fa 100644--- a/crates/engine/src/lib.rs+++ b/crates/engine/src/lib.rs@@ -25,7 +25,10 @@ pub mod guard;pub mod import;mod import_move;pub mod library;+mod lightroom_archive;pub mod lightroom_catalog;+pub mod lightroom_job;+mod lightroom_sqlite;pub mod media;pub mod memory;pub mod merge;@@ -162,6 +165,8 @@ pub struct Session {pub active_spot: Option<usize>,/// The persistent library this session writes to (`None` = in-memory only).pub library: Option<library::Library>,+ /// Identity of current library opening; refreshed only after a successful open.+ pub(crate) library_identity: Arc<()>,/// XMP sidecar preferences (persisted with the library).pub xmp: sidecar::XmpPrefs,/// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json.@@ -249,6 +254,7 @@ impl Session {segmenter: segment::Segmenter::default(),active_spot: None,library: None,+ library_identity: Arc::new(()),xmp: sidecar::XmpPrefs::default(),last_export: None,export_presets: Vec::new(),diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rsindex 6a63e378e..a3f5be336 100644--- a/crates/engine/src/library.rs+++ b/crates/engine/src/library.rs@@ -399,6 +399,7 @@ impl Session {lib.forgot_local = Some(plan);}}+ self.library_identity = std::sync::Arc::new(());Ok(())}@@ -491,12 +492,16 @@ impl Session {lib.retry_at = None;lib.last_error = None;}- match (persisted, snapshot) {+ let result = match (persisted, snapshot) {(_, Ok(())) => Ok(()),// neither the log nor the snapshot took the queued ops(Err(EngineError::NotSaved(e)), Err(s)) if lib.journal.seq() == before => Err(EngineError::NotSaved(format!("{e}; snapshot: {s}"))),(_, Err(s)) => Err(s.into()),+ };+ if result.is_ok() {+ self.library_identity = std::sync::Arc::new(());}+ result}fn view_json(&self) -> Vec<u8> {diff --git a/crates/engine/src/lightroom_archive.rs b/crates/engine/src/lightroom_archive.rsnew file mode 100644index 000000000..ecb53e25c--- /dev/null+++ b/crates/engine/src/lightroom_archive.rs@@ -0,0 +1,654 @@+//! Bounded, deduplicated persistence for Lightroom import source data.+//!+//! Archives use a deterministic JSON representation wrapped in zlib. The JSON+//! writer streams through a bounded compressor, so an import cannot first build+//! an unbounded archive buffer. Blob columns are omitted by the catalog reader;+//! decoded XMP,+//! settings, history, snapshots, identities, paths, collections & membership+//! remain available for recovery.++use std::collections::BTreeMap;+use std::fs::{self, File};+use std::io::{self, Read, Write};+use std::path::{Path, PathBuf};++use miniz_oxide::deflate::{+ core::{CompressorOxide, create_comp_flags_from_zip_params},+ stream::deflate,+};+use miniz_oxide::{MZFlush, MZStatus};+use serde::ser::{Serialize, SerializeMap, SerializeSeq, SerializeStruct, Serializer};+use serde_json::Value;++use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};++/// Maximum size of one complete archive on disk, including format header.+pub const MAX_ARCHIVE_BYTES: u64 = 32 * 1024 * 1024;+/// Maximum total size retained in managed archives.+pub const MAX_TOTAL_ARCHIVE_BYTES: u64 = 128 * 1024 * 1024;+/// Maximum number of managed archives retained.+pub const MAX_MANAGED_ARCHIVES: usize = 8;++const FORMAT_VERSION: u8 = 1;+const HEADER: &[u8] = b"LC-LRARCH\0\x01Z";+const NAME_PREFIX: &str = "lightroom-import-";+const NAME_SUFFIX: &str = ".lca";+const MAX_SCAN: usize = 1024;+const IO_BUFFER: usize = 16 * 1024;++/// Store compact source data below `directory` and return its managed archive.+///+/// The operation is safe to call before catalog mutation. It rejects a+/// symlink directory, writes only `lightroom-import-<number>.lca`, compares the+/// complete bounded byte stream with recent managed archives, and retains at+/// most [`MAX_MANAGED_ARCHIVES`] and [`MAX_TOTAL_ARCHIVE_BYTES`]. Legacy+/// `lightroom-import-*.json` files are deliberately ignored and preserved for+/// an explicit caller warning.+pub fn store(directory: &Path, data: &CatalogImport) -> Result<PathBuf, String> {+ prepare_directory(directory)?;+ let mut managed = managed_files(directory)?;+ managed.sort_by_key(|entry| entry.index);++ let temporary = temporary_path(directory)?;+ write_archive(&temporary, data).map_err(|error| {+ let _ = remove_managed_file(&temporary);+ error+ })?;++ let recent = managed.iter().rev().take(MAX_MANAGED_ARCHIVES);+ for entry in recent {+ let same = match same_bounded(&temporary, &entry.path) {+ Ok(same) => same,+ Err(error) => {+ let _ = remove_managed_file(&temporary);+ return Err(error.to_string());+ }+ };+ if same {+ remove_managed_file(&temporary).map_err(|e| e.to_string())?;+ let matched = entry.path.clone();+ let mut all = managed.clone();+ prune_existing(directory, &mut all, Some(&matched))?;+ return Ok(matched);+ }+ }++ let index = managed.last().map_or(0, |entry| entry.index.saturating_add(1));+ if managed.last().is_some_and(|entry| entry.index == u64::MAX) {+ let _ = remove_managed_file(&temporary);+ return Err("managed Lightroom archive index is exhausted".into());+ }+ let destination = match next_destination(directory, index) {+ Ok(path) => path,+ Err(error) => {+ let _ = remove_managed_file(&temporary);+ return Err(error);+ }+ };+ let size = match fs::metadata(&temporary) {+ Ok(meta) => meta.len(),+ Err(error) => {+ let _ = remove_managed_file(&temporary);+ return Err(error.to_string());+ }+ };+ let new_entry = Managed { index, path: destination.clone(), size };+ let mut after = managed;+ after.push(new_entry);+ after.sort_by_key(|entry| entry.index);+ let retirees = match retirement_set(&after) {+ Ok(retirees) => retirees,+ Err(error) => {+ let _ = remove_managed_file(&temporary);+ return Err(error);+ }+ };+ let quarantined = match quarantine(&retirees) {+ Ok(quarantined) => quarantined,+ Err(error) => {+ let _ = remove_managed_file(&temporary);+ return Err(error);+ }+ };++ if let Err(error) = publish_new(&temporary, &destination) {+ rollback_quarantine(&quarantined);+ let _ = remove_managed_file(&temporary);+ return Err(error.to_string());+ }+ if let Err(error) = remove_managed_file(&temporary) {+ return Err(format!("archive published but temporary cleanup failed: {error}"));+ }+ for (old, held) in quarantined {+ if let Err(error) = remove_managed_file(&held) {+ return Err(format!("archive published but could not retire {}: {error}", old.display()));+ }+ }+ Ok(destination)+}++#[derive(Clone, Debug)]+struct Managed {+ index: u64,+ path: PathBuf,+ size: u64,+}++fn prepare_directory(directory: &Path) -> Result<(), String> {+ match fs::symlink_metadata(directory) {+ Ok(meta) => {+ if meta.file_type().is_symlink() || !meta.is_dir() {+ return Err(format!("archive directory is not a real directory: {}", directory.display()));+ }+ }+ Err(error) if error.kind() == io::ErrorKind::NotFound => {+ fs::create_dir_all(directory).map_err(|e| e.to_string())?;+ let meta = fs::symlink_metadata(directory).map_err(|e| e.to_string())?;+ if meta.file_type().is_symlink() || !meta.is_dir() {+ return Err(format!("archive directory is not a real directory: {}", directory.display()));+ }+ }+ Err(error) => return Err(error.to_string()),+ }+ Ok(())+}++fn managed_files(directory: &Path) -> Result<Vec<Managed>, String> {+ let mut files = Vec::new();+ for (seen, result) in fs::read_dir(directory).map_err(|e| e.to_string())?.enumerate() {+ if seen >= MAX_SCAN {+ return Err(format!("too many entries in archive directory (limit {MAX_SCAN})"));+ }+ let entry = result.map_err(|e| e.to_string())?;+ let Some(index) = parse_managed_name(&entry.file_name()) else { continue };+ let meta = fs::symlink_metadata(entry.path()).map_err(|e| e.to_string())?;+ if meta.file_type().is_symlink() || !meta.is_file() {+ continue;+ }+ files.push(Managed { index, path: entry.path(), size: meta.len() });+ }+ Ok(files)+}++fn parse_managed_name(name: &std::ffi::OsStr) -> Option<u64> {+ let name = name.to_str()?;+ let digits = name.strip_prefix(NAME_PREFIX)?.strip_suffix(NAME_SUFFIX)?;+ if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {+ return None;+ }+ digits.parse().ok()+}++fn temporary_path(directory: &Path) -> Result<PathBuf, String> {+ let pid = std::process::id();+ for slot in 0..MAX_SCAN {+ let path = directory.join(format!(".lightroom-import-{pid}-{slot}.tmp"));+ match fs::symlink_metadata(&path) {+ Ok(_) => continue,+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),+ Err(error) => return Err(error.to_string()),+ }+ }+ Err("no safe temporary archive name available".into())+}++fn next_destination(directory: &Path, mut index: u64) -> Result<PathBuf, String> {+ for _ in 0..MAX_SCAN {+ let path = directory.join(format!("{NAME_PREFIX}{index}{NAME_SUFFIX}"));+ match fs::symlink_metadata(&path) {+ Ok(_) => index = index.checked_add(1).ok_or("managed archive index is exhausted")?,+ Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),+ Err(error) => return Err(error.to_string()),+ }+ }+ Err("no safe managed archive name available".into())+}++fn write_archive(path: &Path, data: &CatalogImport) -> Result<(), String> {+ lightcraft_catalog::safe_file::write_atomic_with(path, &mut |raw| {+ let mut bounded = BoundedWriter { inner: raw, total: 0 };+ bounded.write_all(HEADER)?;+ let mut compressor = ZlibWriter::new(&mut bounded);+ serde_json::to_writer(&mut compressor, &ArchiveRef(data)).map_err(json_io)?;+ compressor.finish()?;+ Ok(())+ })+ .map(|_| ())+ .map_err(|error| format!("could not write Lightroom archive: {error}"))+}++fn json_io(error: serde_json::Error) -> io::Error {+ io::Error::new(io::ErrorKind::InvalidData, error.to_string())+}++struct BoundedWriter<'a> {+ inner: &'a mut dyn Write,+ total: u64,+}++impl Write for BoundedWriter<'_> {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ let next = self.total.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;+ if next > MAX_ARCHIVE_BYTES {+ return Err(limit_error());+ }+ self.inner.write_all(bytes)?;+ self.total = next;+ Ok(bytes.len())+ }++ fn flush(&mut self) -> io::Result<()> {+ self.inner.flush()+ }+}++fn limit_error() -> io::Error {+ io::Error::new(io::ErrorKind::InvalidData, "Lightroom archive exceeds 32 MiB")+}++struct ZlibWriter<'a, 'b> {+ sink: &'a mut BoundedWriter<'b>,+ compressor: CompressorOxide,+ plain: u64,+}++impl<'a, 'b> ZlibWriter<'a, 'b> {+ fn new(sink: &'a mut BoundedWriter<'b>) -> ZlibWriter<'a, 'b> {+ let flags = create_comp_flags_from_zip_params(6, 15, 0);+ ZlibWriter { sink, compressor: CompressorOxide::new(flags), plain: 0 }+ }++ fn run(&mut self, input: &[u8], flush: MZFlush) -> io::Result<()> {+ let mut rest = input;+ loop {+ let mut output = [0u8; IO_BUFFER];+ let result = deflate(&mut self.compressor, rest, &mut output, flush);+ self.sink.write_all(&output[..result.bytes_written])?;+ match result.status {+ Ok(MZStatus::StreamEnd) => return Ok(()),+ Ok(MZStatus::Ok) => {+ let consumed = result.bytes_consumed;+ if consumed > rest.len() {+ return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressor progress"));+ }+ rest = &rest[consumed..];+ if rest.is_empty() && flush == MZFlush::None {+ return Ok(());+ }+ if consumed == 0 && result.bytes_written == 0 {+ return Err(io::Error::new(io::ErrorKind::InvalidData, "compressor made no progress"));+ }+ }+ Ok(status) => {+ return Err(io::Error::new(io::ErrorKind::InvalidData, format!("unsupported compressor status: {status:?}")));+ }+ Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("compression failed: {error:?}"))),+ }+ }+ }++ fn finish(&mut self) -> io::Result<()> {+ self.run(&[], MZFlush::Finish)+ }+}++impl Write for ZlibWriter<'_, '_> {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ if bytes.is_empty() {+ return Ok(0);+ }+ let next = self.plain.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;+ if next > MAX_ARCHIVE_BYTES {+ return Err(limit_error());+ }+ self.run(bytes, MZFlush::None)?;+ self.plain = next;+ Ok(bytes.len())+ }++ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}++fn same_bounded(left: &Path, right: &Path) -> io::Result<bool> {+ let left_meta = fs::symlink_metadata(left)?;+ let right_meta = fs::symlink_metadata(right)?;+ if left_meta.file_type().is_symlink() || right_meta.file_type().is_symlink() || !left_meta.is_file() || !right_meta.is_file() {+ return Ok(false);+ }+ let left_len = left_meta.len();+ let right_len = right_meta.len();+ if left_len != right_len || left_len > MAX_ARCHIVE_BYTES {+ return Ok(false);+ }+ let mut left = File::open(left)?;+ let mut right = File::open(right)?;+ let mut left_buf = [0u8; IO_BUFFER];+ let mut right_buf = [0u8; IO_BUFFER];+ loop {+ let left_n = left.read(&mut left_buf)?;+ let right_n = right.read(&mut right_buf)?;+ if left_n != right_n {+ return Ok(false);+ }+ if left_n == 0 {+ return Ok(true);+ }+ if left_buf[..left_n] != right_buf[..right_n] {+ return Ok(false);+ }+ }+}++fn retirement_set(entries: &[Managed]) -> Result<Vec<Managed>, String> {+ retirement_set_protected(entries, None)+}++fn retirement_set_protected(entries: &[Managed], protected: Option<&Path>) -> Result<Vec<Managed>, String> {+ let mut ordered = entries.to_vec();+ ordered.sort_by_key(|entry| entry.index);+ let mut total = ordered.iter().try_fold(0u64, |sum, entry| sum.checked_add(entry.size)).ok_or("archive size overflow")?;+ let mut retirees = Vec::new();+ while ordered.len() > MAX_MANAGED_ARCHIVES || total > MAX_TOTAL_ARCHIVE_BYTES {+ let Some(position) = ordered.iter().position(|entry| protected.map_or(true, |path| entry.path != path)) else {+ return Err("cannot satisfy archive retention limit".into());+ };+ let oldest = ordered.remove(position);+ total = total.saturating_sub(oldest.size);+ retirees.push(oldest);+ }+ Ok(retirees)+}++fn prune_existing(directory: &Path, entries: &mut [Managed], protected: Option<&Path>) -> Result<(), String> {+ let retirees = retirement_set_protected(entries, protected)?;+ let quarantined = quarantine(&retirees)?;+ for (old, held) in quarantined {+ if let Err(error) = remove_managed_file(&held) {+ return Err(format!("could not retire {} from {}: {error}", old.display(), directory.display()));+ }+ }+ Ok(())+}++fn quarantine(entries: &[Managed]) -> Result<Vec<(PathBuf, PathBuf)>, String> {+ let mut moved = Vec::new();+ let pid = std::process::id();+ for (slot, entry) in entries.iter().enumerate() {+ let held = entry.path.with_file_name(format!(".lightroom-retire-{pid}-{slot}.tmp"));+ match fs::symlink_metadata(&held) {+ Ok(_) => {+ rollback_quarantine(&moved);+ return Err(format!("retention temporary path already exists: {}", held.display()));+ }+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}+ Err(error) => {+ rollback_quarantine(&moved);+ return Err(error.to_string());+ }+ }+ let meta = fs::symlink_metadata(&entry.path).map_err(|error| {+ rollback_quarantine(&moved);+ error.to_string()+ })?;+ if meta.file_type().is_symlink() || !meta.is_file() {+ rollback_quarantine(&moved);+ return Err(format!("managed archive changed into non-file: {}", entry.path.display()));+ }+ if let Err(error) = fs::rename(&entry.path, &held) {+ rollback_quarantine(&moved);+ return Err(error.to_string());+ }+ moved.push((entry.path.clone(), held));+ }+ Ok(moved)+}++fn rollback_quarantine(moved: &[(PathBuf, PathBuf)]) {+ for (original, held) in moved.iter().rev() {+ let _ = fs::rename(held, original);+ }+}++fn publish_new(temporary: &Path, destination: &Path) -> io::Result<()> {+ match fs::hard_link(temporary, destination) {+ Ok(()) => Ok(()),+ Err(error) => Err(io::Error::new(error.kind(), format!("cannot atomically publish archive: {error}"))),+ }+}++fn remove_managed_file(path: &Path) -> io::Result<()> {+ let meta = fs::symlink_metadata(path)?;+ if meta.file_type().is_symlink() || !meta.is_file() {+ return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to remove non-regular archive path"));+ }+ fs::remove_file(path)+}++struct ArchiveRef<'a>(&'a CatalogImport);++impl Serialize for ArchiveRef<'_> {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut out = serializer.serialize_struct("CatalogImportArchive", 7)?;+ out.serialize_field("version", &FORMAT_VERSION)?;+ out.serialize_field("source", &self.0.source)?;+ out.serialize_field("photos", &PhotoSeq(&self.0.photos))?;+ out.serialize_field("collections", &RowSeq(&self.0.collections))?;+ out.serialize_field("members", &RowSeq(&self.0.members))?;+ out.serialize_field("collection_content", &RowSeq(&self.0.collection_content))?;+ out.serialize_field("warnings", &self.0.warnings)?;+ out.end()+ }+}++struct PhotoSeq<'a>(&'a [CatalogPhoto]);+impl Serialize for PhotoSeq<'_> {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut out = serializer.serialize_seq(Some(self.0.len()))?;+ for photo in self.0 {+ out.serialize_element(&PhotoRef(photo))?;+ }+ out.end()+ }+}++struct PhotoRef<'a>(&'a CatalogPhoto);+impl Serialize for PhotoRef<'_> {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let p = self.0;+ let mut out = serializer.serialize_struct("CatalogPhotoArchive", 8)?;+ out.serialize_field("source_id", &p.source_id)?;+ out.serialize_field("uuid", &p.uuid)?;+ out.serialize_field("path", &p.path)?;+ out.serialize_field("image", &RowRef(&p.image))?;+ out.serialize_field("settings", &p.settings)?;+ out.serialize_field("xmp", &p.xmp)?;+ out.serialize_field("keywords", &p.keywords)?;+ out.serialize_field("history", &RowSeq(&p.history))?;+ out.serialize_field("snapshots", &RowSeq(&p.snapshots))?;+ out.end()+ }+}++struct RowSeq<'a>(&'a [BTreeMap<String, Value>]);+impl Serialize for RowSeq<'_> {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut out = serializer.serialize_seq(Some(self.0.len()))?;+ for row in self.0 {+ out.serialize_element(&RowRef(row))?;+ }+ out.end()+ }+}++struct RowRef<'a>(&'a BTreeMap<String, Value>);+impl Serialize for RowRef<'_> {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut out = serializer.serialize_map(None)?;+ for (key, value) in self.0 {+ out.serialize_entry(key, value)?;+ }+ out.end()+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};+ use std::sync::atomic::{AtomicU64, Ordering};++ static NEXT: AtomicU64 = AtomicU64::new(0);++ fn tempdir() -> PathBuf {+ let path = std::env::temp_dir().join(format!("lightcraft-lr-archive-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));+ fs::create_dir_all(&path).unwrap();+ path+ }++ fn data(source: String) -> CatalogImport {+ CatalogImport {+ source,+ photos: Vec::new(),+ collections: Vec::new(),+ members: Vec::new(),+ collection_content: Vec::new(),+ warnings: Vec::new(),+ }+ }++ fn rich_data() -> CatalogImport {+ let photo = CatalogPhoto {+ source_id: 7,+ uuid: "uuid-7".into(),+ path: "/photos/one.raw".into(),+ image: BTreeMap::from([("rating".into(), serde_json::json!(5)), ("opaque_blob".into(), serde_json::json!([1, 2, 3, 4]))]),+ settings: "s = { Exposure2012 = 0.75 }".into(),+ xmp: "<xmp>decoded</xmp>".into(),+ keywords: vec!["Trips|Goa".into()],+ history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])],+ snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])],+ };+ CatalogImport {+ source: "source.lrcat".into(),+ photos: vec![photo],+ collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])],+ members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])],+ collection_content: vec![BTreeMap::from([("rule".into(), serde_json::json!("rating > 3"))])],+ warnings: vec!["one warning".into()],+ }+ }++ fn managed_count(path: &Path) -> usize {+ fs::read_dir(path).unwrap().filter_map(Result::ok).filter(|entry| parse_managed_name(&entry.file_name()).is_some()).count()+ }++ #[test]+ fn deduplicates_identical_deterministic_archives() {+ let path = tempdir();+ let input = data("catalog.lrcat".into());+ let first = store(&path, &input).unwrap();+ let second = store(&path, &input).unwrap();+ assert_eq!(first, second);+ assert_eq!(managed_count(&path), 1);+ fs::remove_dir_all(path).unwrap();+ }++ #[test]+ fn dedup_returns_archive_that_retention_keeps() {+ let path = tempdir();+ let input = data("catalog.lrcat".into());+ let first = store(&path, &input).unwrap();+ for index in 1..MAX_MANAGED_ARCHIVES {+ let filler = path.join(format!("lightroom-import-{index}.lca"));+ File::create(filler).unwrap().set_len(19 * 1024 * 1024).unwrap();+ }+ let reused = store(&path, &input).unwrap();+ assert_eq!(reused, first);+ assert!(reused.is_file());+ assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);+ fs::remove_dir_all(path).unwrap();+ }++ #[test]+ fn caps_managed_archive_count_and_preserves_unrelated_files() {+ let path = tempdir();+ let human = path.join("notes.txt");+ let legacy = path.join("lightroom-import-0.json");+ fs::write(&human, b"keep").unwrap();+ fs::write(&legacy, b"legacy").unwrap();+ for index in 0..(MAX_MANAGED_ARCHIVES + 3) {+ let input = data(format!("catalog-{index}.lrcat"));+ store(&path, &input).unwrap();+ }+ assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);+ assert_eq!(fs::read(&human).unwrap(), b"keep");+ assert_eq!(fs::read(&legacy).unwrap(), b"legacy");+ fs::remove_dir_all(path).unwrap();+ }++ #[test]+ fn rejects_oversize_before_creating_archive() {+ let path = tempdir();+ let input = data("x".repeat((MAX_ARCHIVE_BYTES as usize) + 1));+ assert!(store(&path, &input).is_err());+ assert_eq!(managed_count(&path), 0);+ fs::remove_dir_all(path).unwrap();+ }++ #[test]+ fn archive_roundtrips_zlib_payload_and_preserves_numeric_arrays() {+ let path = tempdir();+ let archive = store(&path, &rich_data()).unwrap();+ let bytes = fs::read(archive).unwrap();+ assert!(bytes.starts_with(HEADER));+ let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&bytes[HEADER.len()..], MAX_ARCHIVE_BYTES as usize).unwrap();+ let decoded: Value = serde_json::from_slice(&json).unwrap();+ assert_eq!(decoded["version"], serde_json::json!(FORMAT_VERSION));+ assert_eq!(decoded["source"], serde_json::json!("source.lrcat"));+ assert_eq!(decoded["photos"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["photos"][0]["source_id"], serde_json::json!(7));+ assert_eq!(decoded["photos"][0]["uuid"], serde_json::json!("uuid-7"));+ assert_eq!(decoded["photos"][0]["path"], serde_json::json!("/photos/one.raw"));+ assert_eq!(decoded["photos"][0]["settings"], serde_json::json!("s = { Exposure2012 = 0.75 }"));+ assert_eq!(decoded["photos"][0]["xmp"], serde_json::json!("<xmp>decoded</xmp>"));+ assert_eq!(decoded["photos"][0]["keywords"], serde_json::json!(["Trips|Goa"]));+ assert_eq!(decoded["photos"][0]["image"]["rating"], serde_json::json!(5));+ assert_eq!(decoded["photos"][0]["image"]["opaque_blob"], serde_json::json!([1, 2, 3, 4]));+ assert_eq!(decoded["photos"][0]["history"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["photos"][0]["history"][0]["curve"], serde_json::json!([0, 128, 255]));+ assert_eq!(decoded["photos"][0]["snapshots"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["collections"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["collections"][0]["name"], serde_json::json!("Keepers"));+ assert_eq!(decoded["members"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["members"][0]["image"], serde_json::json!(7));+ assert_eq!(decoded["collection_content"].as_array().unwrap().len(), 1);+ assert_eq!(decoded["warnings"], serde_json::json!(["one warning"]));+ fs::remove_dir_all(path).unwrap();+ }++ #[test]+ fn retention_enforces_aggregate_byte_cap_with_small_synthetic_entries() {+ let mut entries: Vec<_> = (0..MAX_MANAGED_ARCHIVES as u64)+ .map(|index| Managed { index, path: PathBuf::from(format!("lightroom-import-{index}.lca")), size: 20 * 1024 * 1024 })+ .collect();+ entries.push(Managed { index: 8, path: PathBuf::from("lightroom-import-8.lca"), size: 20 * 1024 * 1024 });+ let retirees = retirement_set(&entries).unwrap();+ assert_eq!(retirees.len(), 3);+ assert_eq!(retirees.iter().map(|entry| entry.index).collect::<Vec<_>>(), vec![0, 1, 2]);+ }++ #[test]+ fn retention_failure_preserves_old_archive() {+ let path = tempdir();+ let old = path.join("lightroom-import-0.lca");+ let held = path.join(format!(".lightroom-retire-{}-0.tmp", std::process::id()));+ fs::write(&old, b"old archive").unwrap();+ fs::write(&held, b"occupied").unwrap();+ let entry = Managed { index: 0, path: old.clone(), size: 11 };+ assert!(quarantine(&[entry]).is_err());+ assert_eq!(fs::read(&old).unwrap(), b"old archive");+ fs::remove_dir_all(path).unwrap();+ }+}diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rsindex 5ce8d577a..47f222473 100644--- a/crates/engine/src/lightroom_catalog.rs+++ b/crates/engine/src/lightroom_catalog.rs@@ -3,12 +3,15 @@//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.use std::collections::{BTreeMap, HashMap, HashSet};use std::path::{Path, PathBuf};-use std::sync::Arc;+use std::sync::{+ Arc,+ atomic::{AtomicBool, AtomicUsize, Ordering},+};+use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};use serde::{Deserialize, Serialize};use serde_json::{Value, json};-use sqlite_core::{Database, Value as SqlValue};type Row = BTreeMap<String, Value>;const COMMAND: &str = "library.importLightroom";@@ -296,6 +299,10 @@ fn text<'a>(r: &'a Row, k: &str) -> &'a str {fn number(r: &Row, k: &str) -> i64 {r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64}++pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 {+ number(r, k)+}fn key(path: &str) -> String {let p = path.replace('\\', "/");if cfg!(windows) { p.to_lowercase() } else { p }@@ -309,7 +316,7 @@ fn source_path(p: &Photo) -> Option<&str> {struct Reader {db: Database,- tables: Vec<sqlite_core::attribution::LiveTable>,+ tables: Vec<LiveTable>,}impl Reader {fn table(&self, name: &str, required: bool) -> Result<Vec<Row>, String> {@@ -328,17 +335,64 @@ impl Reader {.iter().cloned().zip(r.values.into_iter().map(|v| match v {- SqlValue::Null => Value::Null,- SqlValue::Integer(n) => json!(n),- SqlValue::Real(n) => json!(n),- SqlValue::Text(s) => json!(s),- // Opaque data is retained in the source catalog; the XMP packet is decoded below.- SqlValue::Blob(b) => json!(b),+ SqlValue::Null => Some(Value::Null),+ SqlValue::Integer(n) => Some(json!(n)),+ SqlValue::Real(n) => Some(json!(n)),+ SqlValue::Text(s) => Some(json!(s)),+ // Opaque Lightroom payloads never become JSON byte arrays.+ SqlValue::Blob(_) => None,}))+ .filter_map(|(column, value)| value.map(|value| (column, value))).collect()}).collect())}++ fn xmp_table(&self) -> Result<(Vec<Row>, Vec<String>), String> {+ let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) };+ let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?;+ let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?;+ if rows.len() > 1_000_000 {+ return Err("Adobe_AdditionalMetadata: more than one million records".into());+ }+ let mut warnings = Vec::new();+ let mut out = Vec::with_capacity(rows.len());+ for row in rows {+ let image = row+ .values+ .iter()+ .zip(columns)+ .find_map(|(value, column)| {+ (column == "image").then(|| match value {+ SqlValue::Integer(n) => *n,+ _ => 0,+ })+ })+ .unwrap_or(0);+ let mut mapped = Row::new();+ for (column, value) in columns.iter().zip(row.values) {+ let value = match value {+ SqlValue::Null => Some(Value::Null),+ SqlValue::Integer(n) => Some(json!(n)),+ SqlValue::Real(n) => Some(json!(n)),+ SqlValue::Text(s) => Some(json!(s)),+ SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) {+ Ok(packet) => Some(Value::String(packet)),+ Err(error) => {+ warnings.push(format!("image {image}: {error}"));+ None+ }+ },+ SqlValue::Blob(_) => None,+ };+ if let Some(value) = value {+ mapped.insert(column.clone(), value);+ }+ }+ out.push(mapped);+ }+ Ok((out, warnings))+ }}#[derive(Clone, Debug, Serialize, Deserialize)]@@ -364,10 +418,38 @@ pub struct CatalogImport {pub warnings: Vec<String>,}-#[derive(Default, Serialize, Deserialize)]-struct ImportIndex {- photos: BTreeMap<String, u64>,- collections: BTreeMap<String, u64>,+#[derive(Clone, Default, Serialize, Deserialize)]+pub(crate) struct ImportIndex {+ pub(crate) photos: BTreeMap<String, u64>,+ pub(crate) collections: BTreeMap<String, u64>,+}++pub(crate) fn load_index(path: Option<&Path>) -> crate::Result<ImportIndex> {+ match path {+ Some(path) => {+ let file = match std::fs::File::open(path) {+ Ok(file) => file,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()),+ Err(e) => return Err(error(e.to_string())),+ };+ if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 {+ return Err(error("Lightroom import index exceeds 16 MiB"));+ }+ let mut bytes = Vec::new();+ use std::io::Read;+ file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?;+ if bytes.len() > 16 * 1024 * 1024 {+ return Err(error("Lightroom import index exceeds 16 MiB"));+ }+ serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}")))+ }+ None => Ok(ImportIndex::default()),+ }+}++pub(crate) struct AppliedImport {+ pub(crate) report: Value,+ pub(crate) index: ImportIndex,}fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String {@@ -406,6 +488,10 @@ fn validate(data: &CatalogImport) -> Result<(), String> {Ok(())}+pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> {+ validate(data)+}+fn bytes(path: &Path) -> Result<Vec<u8>, String> {use std::io::Read;let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;@@ -440,6 +526,19 @@ fn xmp(v: Option<&Value>) -> Result<String, String> {}}+fn decode_xmp_bytes(bytes: &[u8]) -> Result<String, String> {+ let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?;+ if size > 16 << 20 {+ return Err("XMP packet exceeds 16 MiB".into());+ }+ let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20)+ .map_err(|_| "invalid compressed XMP")?;+ if decoded.len() != size as usize {+ return Err("XMP packet length mismatch".into());+ }+ String::from_utf8(decoded).map_err(|e| e.to_string())+}+fn be_word(bytes: &[u8], at: usize) -> Result<u32, String> {let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?;Ok(u32::from_be_bytes(word))@@ -500,6 +599,17 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.pub fn read(path: &Path) -> Result<CatalogImport, String> {+ let cancel = AtomicBool::new(false);+ let total = AtomicUsize::new(0);+ let done = AtomicUsize::new(0);+ read_with_progress(path, &cancel, &total, &done)+}++pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result<CatalogImport, String> {+ // 13 catalog tables plus one validation stage; ImportJob adds one unit per original path.+ total.store(14, Ordering::Relaxed);+ done.store(0, Ordering::Relaxed);+ ensure_read_active(cancel)?;let before = std::fs::metadata(path).map_err(|e| e.to_string())?;let wal_path = PathBuf::from(format!("{}-wal", path.display()));let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));@@ -510,22 +620,33 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {Err(e) => return Err(e.to_string()),};let wal_len = committed_wal_len(&main, &wal)?;+ ensure_read_active(cancel)?;let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;let after = std::fs::metadata(path).map_err(|e| e.to_string())?;let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after {return Err("catalog changed during reading; close Lightroom and retry".into());}- let reader = Reader { tables: db.live_tables(), db };- let roots: HashMap<_, _> = reader.table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();- let folders: HashMap<_, _> = reader.table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();- let files: HashMap<_, _> = reader.table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();+ let reader = Reader { tables: db.live_tables()?, db };+ let mut table = |name: &str, required: bool| -> Result<Vec<Row>, String> {+ ensure_read_active(cancel)?;+ let rows = reader.table(name, required)?;+ done.fetch_add(1, Ordering::Relaxed);+ Ok(rows)+ };+ let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();+ let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();+ let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();let develops: HashMap<_, _> =- reader.table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();- let metadata: HashMap<_, _> = reader.table("Adobe_AdditionalMetadata", false)?.into_iter().map(|r| (number(&r, "image"), r)).collect();- let keyword_rows: HashMap<_, _> = reader.table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();+ table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();+ ensure_read_active(cancel)?;+ let (metadata_rows, mut metadata_warnings) = reader.xmp_table()?;+ done.fetch_add(1, Ordering::Relaxed);+ let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect();+ let keyword_rows: HashMap<_, _> = table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();let mut keywords: HashMap<i64, Vec<String>> = HashMap::new();- for r in reader.table("AgLibraryKeywordImage", false)? {+ for r in table("AgLibraryKeywordImage", false)? {+ ensure_read_active(cancel)?;let mut parts = Vec::new();let mut at = number(&r, "tag");let mut seen = HashSet::new();@@ -552,17 +673,18 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {}out};- let mut history = group(reader.table("Adobe_libraryImageDevelopHistoryStep", false)?);- let mut snapshots = group(reader.table("Adobe_libraryImageDevelopSnapshot", false)?);+ let mut history = group(table("Adobe_libraryImageDevelopHistoryStep", false)?);+ let mut snapshots = group(table("Adobe_libraryImageDevelopSnapshot", false)?);let mut out = CatalogImport {source: path.to_string_lossy().into(),photos: Vec::new(),- collections: reader.table("AgLibraryCollection", false)?,- members: reader.table("AgLibraryCollectionImage", false)?,- collection_content: reader.table("AgLibraryCollectionContent", false)?,- warnings: Vec::new(),+ collections: table("AgLibraryCollection", false)?,+ members: table("AgLibraryCollectionImage", false)?,+ collection_content: table("AgLibraryCollectionContent", false)?,+ warnings: std::mem::take(&mut metadata_warnings),};- for image in reader.table("Adobe_images", true)? {+ for image in table("Adobe_images", true)? {+ ensure_read_active(cancel)?;let id = number(&image, "id_local");let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?;let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?;@@ -593,9 +715,14 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {snapshots: snapshots.remove(&id).unwrap_or_default(),});}+ ensure_read_active(cancel)?;Ok(out)}+fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {+ if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }+}+fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec<String>), String> {let mut root = crate::preset_import::parse_lua(text)?;let deferred = strip_lua_sentinels(&mut root);@@ -651,39 +778,60 @@ fn strip_xmp_sentinels(value: &mut Value) {/// Import into the current library. Existing edited records are preserved unless explicitly/// requested; missing originals remain catalogued for later relinking. The source catalog is read-only.pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result<Value> {- let data = read(path).map_err(error)?;- apply(s, data, update_existing)+ let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?;+ let cancel = AtomicBool::new(false);+ let prepared = job.prepare(&cancel)?;+ let completion = crate::lightroom_job::commit_prepared(s, prepared)?;+ if s.library.as_ref().is_some_and(|library| library.on_disk) {+ s.persist()?;+ }+ completion.finalization.finish()?;+ Ok(completion.report)}+#[cfg(test)]fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result<Value> {- validate(&data).map_err(error)?;- let mut warnings = data.warnings.clone();- let archive = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));- let index_path = archive.as_ref().map(|p| p.join("lightroom-index.json"));- let mut index: ImportIndex = match index_path.as_ref().map(std::fs::read) {- Some(Ok(b)) => serde_json::from_slice(&b).map_err(|e| error(format!("invalid Lightroom import index: {e}")))?,- Some(Err(e)) if e.kind() != std::io::ErrorKind::NotFound => return Err(error(e.to_string())),- _ => ImportIndex::default(),- };- // Preserve source data before catalog changes. A unique archive is never overwritten.- let archive_path = if let Some(dir) = archive {- std::fs::create_dir_all(&dir).map_err(|e| error(e.to_string()))?;- let bytes = serde_json::to_vec(&data).map_err(|e| error(e.to_string()))?;- let mut names = (0..10000).map(|n| dir.join(format!("lightroom-import-{n}.json")));- Some(lightcraft_catalog::safe_file::write_new_unique(&mut names, &bytes).map_err(|e| error(e.to_string()))?)- } else {- None- };+ let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));+ let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json"));+ let mut index = load_index(index_path.as_deref())?;+ let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;+ let undo0 = s.undo.len();let existing: HashMap<_, _> =s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();- let undo0 = s.undo.len();let paths: Vec<_> = data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file()).map(|p| p.path.clone()).collect();+ let missing_sources: HashSet<String> =+ data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect();let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;+ let applied =+ apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;+ if let Some(path) = index_path {+ s.persist()?;+ let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?;+ lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;+ }+ Ok(applied.report)+}++/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation.+/// This function does no filesystem work; callers finish its returned index separately.+pub(crate) struct ApplyPrepared<'a> {+ pub(crate) update_existing: bool,+ pub(crate) files: crate::import::ImportReport,+ pub(crate) index: &'a mut ImportIndex,+ pub(crate) archive_path: Option<PathBuf>,+ pub(crate) missing_sources: &'a HashSet<String>,+ pub(crate) undo0: usize,+}++pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {+ let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;+ validate(&data).map_err(error)?;+ let mut warnings = data.warnings.clone();let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect();let mut by_path: HashMap<_, _> =s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();@@ -729,7 +877,7 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->} else {let id = s.catalog.alloc_photo_id();let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone());- if !Path::new(&src.path).is_file() {+ if missing_sources.contains(&key(&src.path)) {missing.push(src.path.clone());}let mut p = Photo::new(@@ -866,12 +1014,8 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->s.commit("Import Lightroom Catalog", Op::Batch { ops })?;}s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog");- if let Some(path) = index_path {- s.persist()?;- let bytes = serde_json::to_vec(&index).map_err(|e| error(e.to_string()))?;- lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;- }- Ok(- json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),- )+ Ok(AppliedImport {+ report: json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),+ index: index.clone(),+ })}diff --git a/crates/engine/src/lightroom_job.rs b/crates/engine/src/lightroom_job.rsnew file mode 100644index 000000000..ab383aa11--- /dev/null+++ b/crates/engine/src/lightroom_job.rs@@ -0,0 +1,386 @@+//! Detached Lightroom catalog preparation and owner-thread commit.+//!+//! `LightroomJob::new` takes only an in-memory session snapshot. `prepare` owns all catalog+//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations+//! to the session that created the job. Archive index bytes are returned for a worker to finish+//! after the owner has accepted the catalog operation.++use std::collections::HashSet;+use std::path::{Path, PathBuf};+use std::sync::{+ Arc,+ atomic::{AtomicBool, AtomicUsize, Ordering},+};++use serde_json::Value;++use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared};+use crate::lightroom_catalog::{self, CatalogImport, ImportIndex};++const COMMAND: &str = "library.importLightroom";+const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024;++fn error(message: impl Into<String>) -> crate::EngineError {+ crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() }+}++/// Memory identity of the library which owns a prepared completion.+///+/// Each successful open/close refreshes session identity, including reopening the same path.+/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation+/// runs must survive and are merged by the owner commit.+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct LightroomLibraryToken {+ library_dir: Option<PathBuf>,+ identity: Arc<()>,+}++impl LightroomLibraryToken {+ fn capture(s: &crate::Session) -> Self {+ Self {+ library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()),+ identity: Arc::clone(&s.library_identity),+ }+ }++ /// Whether `s` still refers to the library which started preparation.+ pub fn matches(&self, s: &crate::Session) -> bool {+ Arc::ptr_eq(&self.identity, &s.library_identity)+ && self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone())+ }++ pub fn library_dir(&self) -> Option<&Path> {+ self.library_dir.as_deref()+ }+}++/// Filesystem-free result of catalog preparation.+pub struct PreparedLightroom {+ pub(crate) data: CatalogImport,+ pub(crate) normal: Prepared,+ pub(crate) index: ImportIndex,+ pub(crate) archive_path: Option<PathBuf>,+ pub(crate) token: LightroomLibraryToken,+ pub(crate) missing_sources: HashSet<String>,+ pub(crate) update_existing: bool,+ pub(crate) now: String,+ cancelled: bool,+}++impl PreparedLightroom {+ pub fn source(&self) -> &str {+ &self.data.source+ }++ pub fn photo_count(&self) -> usize {+ self.data.photos.len()+ }++ pub fn token(&self) -> &LightroomLibraryToken {+ &self.token+ }++ pub fn is_cancelled(&self) -> bool {+ self.cancelled+ }++ /// Roll back any normal-import filesystem placement held by this prepared value.+ pub fn rollback(&self) {+ self.normal.rollback();+ }+}++/// Index write returned by owner commit. `finish` is intended for a worker after the owner has+/// returned, and performs the only archive-index filesystem operation in this pipeline.+pub struct LightroomArchiveFinalization {+ path: Option<PathBuf>,+ bytes: Option<Vec<u8>>,+}++impl LightroomArchiveFinalization {+ pub fn finish(self) -> crate::Result<()> {+ let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) };+ if bytes.len() > MAX_INDEX_BYTES {+ return Err(error("Lightroom import index exceeds 16 MiB"));+ }+ lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))+ }++ pub fn path(&self) -> Option<&Path> {+ self.path.as_deref()+ }+}++/// Owner-thread catalog result. The report is ready for the UI immediately; index finalization+/// can remain on the worker while its task stays alive.+pub struct LightroomCompletion {+ pub report: Value,+ pub finalization: LightroomArchiveFinalization,+}++/// Alias used by hosts that call the result a commit.+pub type LightroomCommit = LightroomCompletion;++/// A Lightroom import detached from the owner thread.+pub struct LightroomJob {+ path: PathBuf,+ update_existing: bool,+ import: ImportJob,+ token: LightroomLibraryToken,+ archive_dir: Option<PathBuf>,+}++impl LightroomJob {+ /// Snapshot session state only. This constructor performs no filesystem work.+ pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result<Self> {+ let opts = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };+ let import = ImportJob::new(s, opts)?;+ let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop"));+ Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir })+ }++ pub fn total(&self) -> usize {+ self.import.total.load(Ordering::Relaxed)+ }++ pub fn done(&self) -> usize {+ self.import.done.load(Ordering::Relaxed)+ }++ pub fn progress(&self) -> (usize, usize) {+ (self.done(), self.total())+ }++ pub fn token(&self) -> &LightroomLibraryToken {+ &self.token+ }++ pub fn total_atomic(&self) -> std::sync::Arc<AtomicUsize> {+ std::sync::Arc::clone(&self.import.total)+ }++ pub fn done_atomic(&self) -> std::sync::Arc<AtomicUsize> {+ std::sync::Arc::clone(&self.import.done)+ }++ /// Finish an owner commit's deferred archive-index write on a worker.+ pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> {+ finalization.finish()+ }++ /// Read/probe/stage all source data off the owner thread.+ pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result<PreparedLightroom> {+ if cancel.load(Ordering::Relaxed) {+ return Err(error("Lightroom import cancelled"));+ }+ self.import.total.store(1, Ordering::Relaxed);+ self.import.done.store(0, Ordering::Relaxed);+ let data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?;+ lightroom_catalog::validate_for_job(&data).map_err(error)?;+ if cancel.load(Ordering::Relaxed) {+ return Err(error("Lightroom import cancelled"));+ }+ self.import.done.fetch_add(1, Ordering::Relaxed);+ let paths: Vec<String> = data+ .photos+ .iter()+ .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)+ .map(|photo| photo.path.clone())+ .collect();+ let missing_sources = data+ .photos+ .iter()+ .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)+ .filter(|photo| !Path::new(&photo.path).is_file())+ .map(|photo| normalize(&photo.path))+ .collect();+ let normal = self.import.prepare(&paths, cancel);+ if cancel.load(Ordering::Relaxed) {+ normal.rollback();+ return Err(error("Lightroom import cancelled"));+ }+ let archive_path = self.archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;+ let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json"));+ let index = lightroom_catalog::load_index(index_path.as_deref())?;+ Ok(PreparedLightroom {+ data,+ normal,+ index,+ archive_path,+ token: self.token.clone(),+ missing_sources,+ update_existing: self.update_existing,+ now: self.import.now().to_string(),+ cancelled: cancel.load(Ordering::Relaxed),+ })+ }+}++/// Apply prepared data on the owner thread. No source, archive or index filesystem work occurs.+pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result<LightroomCompletion> {+ if prepared.cancelled {+ prepared.normal.rollback();+ return Err(error("Lightroom import cancelled"));+ }+ if !prepared.token.matches(s) {+ prepared.normal.rollback();+ return Err(error("Lightroom import belongs to a different open library"));+ }+ // A command-scoped Lightroom import must never rewrite source XMP sidecars. Direct depth-0+ // callers do not set this transient flag, so they cannot affect a later command.+ if s.depth > 0 {+ s.skip_auto_write = true;+ }++ let undo0 = s.undo.len();+ prepared.normal.revalidate_add(s);+ let options = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };+ let now = prepared.now.clone();+ let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?;+ let applied = lightroom_catalog::apply_prepared(+ s,+ prepared.data,+ lightroom_catalog::ApplyPrepared {+ update_existing: prepared.update_existing,+ files,+ index: &mut prepared.index,+ archive_path: prepared.archive_path,+ missing_sources: &prepared.missing_sources,+ undo0,+ },+ )?;+ let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json"));+ let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None };+ Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } })+}++fn normalize(path: &str) -> String {+ let path = path.replace('\\', "/");+ if cfg!(windows) { path.to_lowercase() } else { path }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn empty_data() -> CatalogImport {+ CatalogImport {+ source: "fixture.lrcat".into(),+ photos: Vec::new(),+ collections: Vec::new(),+ members: Vec::new(),+ collection_content: Vec::new(),+ warnings: Vec::new(),+ }+ }++ fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom {+ PreparedLightroom {+ data: empty_data(),+ normal: Prepared::default(),+ index: ImportIndex::default(),+ archive_path: None,+ token: LightroomLibraryToken::capture(s),+ missing_sources: HashSet::new(),+ update_existing: false,+ now: "2026-01-01T00:00:00".into(),+ cancelled: true,+ }+ }++ fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom {+ let mut image = std::collections::BTreeMap::new();+ image.insert("id_local".into(), serde_json::json!(1));+ image.insert("rating".into(), serde_json::json!(rating));+ image.insert("pick".into(), serde_json::json!(0));+ image.insert("fileFormat".into(), serde_json::json!("JPG"));+ image.insert("fileWidth".into(), serde_json::json!(1));+ image.insert("fileHeight".into(), serde_json::json!(1));+ PreparedLightroom {+ data: CatalogImport {+ source: "fixture.lrcat".into(),+ photos: vec![lightroom_catalog::CatalogPhoto {+ source_id: 1,+ uuid: "fixture-photo".into(),+ path: path.into(),+ image,+ settings: String::new(),+ xmp: String::new(),+ keywords: Vec::new(),+ history: Vec::new(),+ snapshots: Vec::new(),+ }],+ collections: Vec::new(),+ members: Vec::new(),+ collection_content: Vec::new(),+ warnings: Vec::new(),+ },+ normal: Prepared::default(),+ index: ImportIndex::default(),+ archive_path: None,+ token: LightroomLibraryToken::capture(s),+ missing_sources: [normalize(path)].into_iter().collect(),+ update_existing: false,+ now: "2026-01-01T00:00:00".into(),+ cancelled: false,+ }+ }++ #[test]+ fn cancellation_before_commit_leaves_owner_catalog_unchanged() {+ let mut session = crate::Session::new();+ let before = session.catalog.to_snapshot();+ let prepared = cancelled_prepared(&session);+ assert!(commit_prepared(&mut session, prepared).is_err());+ assert_eq!(session.catalog.to_snapshot(), before);+ assert!(session.undo.is_empty());+ }++ #[test]+ fn library_identity_rejects_late_completion_after_reopen() {+ let mut session = crate::Session::new();+ let token = LightroomLibraryToken::capture(&session);+ assert!(token.matches(&session));+ session.library_identity = Arc::new(());+ assert!(!token.matches(&session));+ }++ #[test]+ fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() {+ let mut session = crate::Session::new();+ let path = "/missing/lightroom-job-edit.jpg";+ let prepared = prepared_for(&session, path, 4);+ let first = commit_prepared(&mut session, prepared).unwrap();+ assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1));+ assert_eq!(session.undo.len(), 1);+ let id = lightcraft_catalog::PhotoId(1);+ session.set_develop(id, lightcraft_develop::DevelopSettings::default(), "Personal").unwrap();+ let before = session.catalog.photo(id).map(|photo| photo.develop.clone());+ let prepared = prepared_for(&session, path, 5);+ commit_prepared(&mut session, prepared).unwrap();+ assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before);+ assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5));+ assert_eq!(session.undo.len(), 3);+ }++ #[test]+ fn execute_fn_import_does_not_rewrite_source_sidecar() {+ let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id()));+ let path = dir.join("source.jpg");+ let sidecar = dir.join("source.xmp");+ std::fs::create_dir_all(&dir).unwrap();+ std::fs::write(&path, b"fixture").unwrap();+ std::fs::write(&sidecar, b"original sidecar").unwrap();+ let original = std::fs::read(&sidecar).unwrap();+ let mut session = crate::Session::new();+ session.xmp.auto_write = true;+ let result = session.execute_fn("library.importLightroom", |s| {+ let prepared = prepared_for(s, &path.to_string_lossy(), 5);+ let completion = commit_prepared(s, prepared)?;+ Ok(completion.report)+ });+ assert!(result.is_ok());+ assert_eq!(std::fs::read(&sidecar).unwrap(), original);+ std::fs::remove_dir_all(dir).unwrap();+ }+}diff --git a/crates/engine/src/lightroom_sqlite.rs b/crates/engine/src/lightroom_sqlite.rsnew file mode 100644index 000000000..be7f4cb92--- /dev/null+++ b/crates/engine/src/lightroom_sqlite.rs@@ -0,0 +1,1071 @@+//! Small, read-only SQLite reader used by Lightroom catalog import.+//!+//! This is a clean-room implementation based on SQLite's public file-format+//! description: <https://www.sqlite.org/fileformat.html> (sections 1.6, 1.7,+//! 2.1, 2.2, 2.3, 2.4 and 2.5). It is deliberately a page reader, rather+//! than a SQL engine. It never writes its input and has no SQLite runtime+//! dependency.++use std::collections::{HashMap, HashSet};++const MAX_PAGE_ID: u32 = 16_777_216;+const MAX_COLUMNS: usize = 16_384;+const MAX_ROWS: usize = 1_000_000;+const MAX_CELLS: usize = 4_000_000;+const MAX_PAYLOAD: usize = 64 * 1024 * 1024;+const MAX_TABLE_PAYLOAD: usize = 256 * 1024 * 1024;+const MAX_OVERFLOW_PAGES: usize = 1_048_576;+const MAX_BTREE_DEPTH: usize = 64;++/// Values supported by SQLite table records.+#[derive(Clone, Debug, PartialEq)]+pub enum Value {+ Null,+ Integer(i64),+ Real(f64),+ Text(String),+ Blob(Vec<u8>),+}++/// A row returned by [`Database::read_table`].+#[derive(Clone, Debug, PartialEq)]+pub struct Record {+ pub values: Vec<Value>,+}++/// A table discovered in `sqlite_schema`.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct LiveTable {+ pub name: String,+ pub rootpage: u32,+ pub column_names: Option<Vec<String>>,+}++/// Read-only SQLite database image, with a committed WAL overlay if supplied.+pub struct Database {+ main: Vec<u8>,+ overlay: HashMap<u32, Vec<u8>>,+ page_size: usize,+ reserved: usize,+ encoding: TextEncoding,+ page_count: u32,+ tables: Vec<LiveTable>,+ rowid_aliases: HashMap<u32, usize>,+}++#[derive(Clone, Copy)]+enum TextEncoding {+ Utf8,+ Utf16Le,+ Utf16Be,+}++struct WalkState {+ seen: HashSet<u32>,+ out: Vec<Record>,+ payload_total: usize,+}++struct WalOverlay {+ page_size: usize,+ frame_count: usize,+ pages: HashMap<u32, Vec<u8>>,+ page_count: Option<u32>,+}++struct SchemaInfo {+ tables: Vec<LiveTable>,+ aliases: HashMap<u32, usize>,+}++impl Database {+ /// Open a main database image and an already checksum-validated WAL prefix.+ ///+ /// The WAL argument must include its 32-byte header and only complete+ /// frames through a commit. Frames are overlaid by page number, with the+ /// final commit's database-size field defining the visible page count.+ pub fn open_with_wal(main: Vec<u8>, committed_wal: &[u8]) -> Result<Self, String> {+ let main_header = main.get(..100).ok_or("truncated SQLite header")?;+ if main_header.get(..16) != Some(b"SQLite format 3\0") {+ return Err("invalid SQLite header signature".into());+ }+ let page_size = sqlite_page_size(be_u16(main_header, 16)?)?;+ validate_header_fields(main_header, page_size)?;+ if main.len() < page_size || main.len() % page_size != 0 {+ return Err("SQLite image is not a whole number of pages".into());+ }+ let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?;+ if main_pages == 0 || main_pages > MAX_PAGE_ID {+ return Err("invalid SQLite image page count".into());+ }++ let mut overlay = HashMap::new();+ let mut wal_page_count = None;+ if !committed_wal.is_empty() {+ let parsed = parse_wal(committed_wal, page_size)?;+ if parsed.page_size != page_size {+ return Err("WAL page size mismatch".into());+ }+ if parsed.frame_count > MAX_CELLS {+ return Err("WAL contains too many frames".into());+ }+ overlay = parsed.pages;+ wal_page_count = parsed.page_count;+ }+ let effective_header = overlay.get(&1).map(Vec::as_slice).unwrap_or(main_header);+ if effective_header.get(..16) != Some(b"SQLite format 3\0") {+ return Err("invalid SQLite header page in WAL".into());+ }+ let effective_size = sqlite_page_size(be_u16(effective_header, 16)?)?;+ if effective_size != page_size {+ return Err("WAL page 1 changes SQLite page size".into());+ }+ let reserved = validate_header_fields(effective_header, page_size)?;+ let header_pages = be_u32(effective_header, 28)?;+ let header_valid = be_u32(effective_header, 24)? != 0 && be_u32(effective_header, 24)? == be_u32(effective_header, 92)?;+ if header_valid && header_pages > main_pages && wal_page_count.is_none() {+ return Err("SQLite header page count exceeds image".into());+ }+ let mut page_count = if header_valid && header_pages != 0 { header_pages } else { main_pages };+ if let Some(count) = wal_page_count {+ page_count = count;+ }+ if page_count == 0 || page_count > MAX_PAGE_ID {+ return Err("invalid visible SQLite page count".into());+ }+ let encoding = match be_u32(effective_header, 56)? {+ 0 | 1 => TextEncoding::Utf8,+ 2 => TextEncoding::Utf16Le,+ 3 => TextEncoding::Utf16Be,+ _ => return Err("unsupported SQLite text encoding".into()),+ };++ let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() };+ let schema = db.parse_schema()?;+ db.tables = schema.tables;+ db.rowid_aliases = schema.aliases;+ Ok(db)+ }++ /// Return tables from `sqlite_schema` in schema order.+ pub fn live_tables(&self) -> Result<Vec<LiveTable>, String> {+ Ok(self.tables.clone())+ }++ /// Read a table B-tree, decoding up to `ncols` values per record.+ pub fn read_table(&self, rootpage: u32, ncols: usize) -> Result<Vec<Record>, String> {+ if !(1..=MAX_PAGE_ID).contains(&rootpage) || rootpage > self.page_count {+ return Err("table root page is outside database".into());+ }+ if ncols > MAX_COLUMNS {+ return Err("table has too many columns".into());+ }+ let alias = self.rowid_aliases.get(&rootpage).copied();+ let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 };+ self.visit_table(rootpage, ncols, alias, 0, &mut state)?;+ Ok(state.out)+ }++ fn page(&self, page_id: u32) -> Result<&[u8], String> {+ if page_id == 0 || page_id > self.page_count {+ return Err(format!("page {page_id} is outside database"));+ }+ if let Some(page) = self.overlay.get(&page_id) {+ return Ok(page);+ }+ let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?;+ let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?;+ let end = start.checked_add(self.page_size).ok_or("page offset overflow")?;+ self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated"))+ }++ fn visit_table(&self, page_id: u32, ncols: usize, alias: Option<usize>, depth: usize, state: &mut WalkState) -> Result<(), String> {+ if depth > MAX_BTREE_DEPTH {+ return Err("SQLite B-tree is too deep".into());+ }+ if !state.seen.insert(page_id) {+ return Err("SQLite B-tree contains a cycle or shared page".into());+ }+ let page = self.page(page_id)?;+ let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?;+ let base = if page_id == 1 { 100 } else { 0 };+ let content = page.get(..usable).ok_or("truncated usable SQLite page")?;+ let kind = *content.get(base).ok_or("truncated B-tree header")?;+ let cells = usize::from(be_u16_at(content, base + 3)?);+ if cells > MAX_CELLS || state.out.len().saturating_add(cells) > MAX_ROWS.saturating_add(MAX_CELLS) {+ return Err("SQLite B-tree contains too many cells".into());+ }+ match kind {+ 0x0d => {+ let ptr_start = base.checked_add(8).ok_or("B-tree header overflow")?;+ let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?;+ let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?;+ if ptr_end > usable {+ return Err("truncated B-tree pointer array".into());+ }+ for i in 0..cells {+ let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?;+ let cell = usize::from(be_u16_at(content, p)?);+ if cell < ptr_end || cell >= usable {+ return Err("invalid B-tree cell pointer".into());+ }+ let mut at = cell;+ let payload_len = usize_from_u64(read_varint(content, &mut at)?, "payload length")?;+ if payload_len > MAX_PAYLOAD {+ return Err("SQLite record payload is too large".into());+ }+ let rowid = i64::from_be_bytes(read_varint(content, &mut at)?.to_be_bytes());+ state.payload_total = state.payload_total.checked_add(payload_len).ok_or("SQLite table payload total overflow")?;+ if state.payload_total > MAX_TABLE_PAYLOAD {+ return Err("SQLite table payload exceeds bounded import size".into());+ }+ let payload = self.payload(page, at, payload_len)?;+ let mut values = decode_record(&payload, ncols, self.encoding)?;+ if let Some(index) = alias {+ if index >= values.len() {+ return Err("INTEGER PRIMARY KEY column is missing from record".into());+ }+ if matches!(values.get(index), Some(Value::Null)) {+ if let Some(value) = values.get_mut(index) {+ *value = Value::Integer(rowid);+ }+ }+ }+ if state.out.len() >= MAX_ROWS {+ return Err("SQLite table contains more than one million records".into());+ }+ state.out.push(Record { values });+ }+ Ok(())+ }+ 0x05 => {+ let ptr_start = base.checked_add(12).ok_or("B-tree header overflow")?;+ let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?;+ let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?;+ if ptr_end > usable {+ return Err("truncated interior B-tree pointer array".into());+ }+ for i in 0..cells {+ let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?;+ let cell = usize::from(be_u16_at(content, p)?);+ if cell < ptr_end || cell >= usable {+ return Err("invalid interior B-tree cell pointer".into());+ }+ let child = be_u32_at(content, cell)?;+ if child == 0 {+ return Err("interior B-tree cell has null child".into());+ }+ let mut key_at = cell.checked_add(4).ok_or("interior cell overflow")?;+ let _ = read_varint(content, &mut key_at)?;+ self.visit_table(child, ncols, alias, depth + 1, state)?;+ }+ let right = be_u32_at(content, base + 8)?;+ if right == 0 {+ return Err("interior B-tree has null right child".into());+ }+ self.visit_table(right, ncols, alias, depth + 1, state)+ }+ 0x02 | 0x0a => Err("SQLite index B-trees are unsupported".into()),+ other => Err(format!("unsupported SQLite B-tree page type 0x{other:02x}")),+ }+ }++ fn payload(&self, page: &[u8], cell_at: usize, payload_len: usize) -> Result<Vec<u8>, String> {+ let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?;+ if usable < 8 {+ return Err("SQLite usable page size is too small".into());+ }+ let max_local = usable.saturating_sub(35);+ let min_local = ((usable - 12).saturating_mul(32) / 255).saturating_sub(23);+ let local = if payload_len <= max_local {+ payload_len+ } else {+ let span = usable - 4;+ let candidate = min_local.saturating_add((payload_len - min_local) % span);+ if candidate > max_local { min_local } else { candidate }+ };+ let local_end = cell_at.checked_add(local).ok_or("SQLite cell payload overflow")?;+ if local_end > usable {+ return Err("truncated SQLite cell payload".into());+ }+ let mut out = Vec::with_capacity(payload_len);+ out.extend_from_slice(page.get(cell_at..local_end).ok_or("truncated SQLite cell payload")?);+ let mut remaining = payload_len.saturating_sub(local);+ if remaining == 0 {+ return Ok(out);+ }+ let overflow_at = local_end;+ match overflow_at.checked_add(4) {+ Some(end) if end <= usable => {}+ _ => return Err("SQLite overflow pointer is in reserved page bytes".into()),+ }+ let first = be_u32_at(page, overflow_at)?;+ let mut next = first;+ let mut seen = HashSet::new();+ while remaining > 0 {+ if next == 0 || !seen.insert(next) || seen.len() > MAX_OVERFLOW_PAGES {+ return Err("invalid or cyclic SQLite overflow chain".into());+ }+ let overflow = self.page(next)?;+ let take = remaining.min(usable - 4);+ let end = 4usize.checked_add(take).ok_or("overflow payload offset")?;+ out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?);+ remaining -= take;+ if remaining > 0 {+ next = be_u32_at(overflow, 0)?;+ }+ }+ Ok(out)+ }++ fn parse_schema(&self) -> Result<SchemaInfo, String> {+ let mut tables = Vec::new();+ let mut aliases = HashMap::new();+ let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 };+ self.visit_table(1, 5, None, 0, &mut state)?;+ for record in state.out {+ if record.values.len() < 5 {+ return Err("sqlite_schema row has too few columns".into());+ }+ let kind = match record.values.first() {+ Some(Value::Text(s)) => s.as_str(),+ _ => return Err("sqlite_schema row has invalid type".into()),+ };+ let name = match record.values.get(1) {+ Some(Value::Text(s)) => s.clone(),+ _ => return Err("sqlite_schema row has invalid name".into()),+ };+ if kind.eq_ignore_ascii_case("virtual") {+ return Err(format!("virtual table {name} is unsupported"));+ }+ if !kind.eq_ignore_ascii_case("table") {+ continue;+ }+ let rootpage = match record.values.get(3) {+ Some(Value::Integer(n)) if *n > 0 => u32::try_from(*n).map_err(|_| "sqlite_schema root page is invalid")?,+ _ => return Err(format!("table {name} has invalid root page")),+ };+ let sql = match record.values.get(4) {+ Some(Value::Text(s)) => s,+ _ => return Err(format!("table {name} has no CREATE TABLE SQL")),+ };+ let (column_names, alias) = parse_create_table(sql)?;+ if column_names.len() > MAX_COLUMNS {+ return Err(format!("table {name} has too many columns"));+ }+ if tables.iter().any(|table: &LiveTable| table.rootpage == rootpage) {+ return Err("duplicate SQLite table root page".into());+ }+ if let Some(index) = alias {+ aliases.insert(rootpage, index);+ }+ tables.push(LiveTable { name, rootpage, column_names: Some(column_names) });+ }+ Ok(SchemaInfo { tables, aliases })+ }+}++fn validate_header_fields(header: &[u8], page_size: usize) -> Result<usize, String> {+ let write_version = *header.get(18).ok_or("truncated SQLite header")?;+ let read_version = *header.get(19).ok_or("truncated SQLite header")?;+ if !matches!(write_version, 1 | 2) || !matches!(read_version, 1 | 2) {+ return Err("unsupported SQLite file format version".into());+ }+ if *header.get(21).ok_or("truncated SQLite header")? != 64+ || *header.get(22).ok_or("truncated SQLite header")? != 32+ || *header.get(23).ok_or("truncated SQLite header")? != 32+ {+ return Err("invalid SQLite payload fractions".into());+ }+ let reserved = usize::from(*header.get(20).ok_or("truncated SQLite header")?);+ let usable = page_size.checked_sub(reserved).ok_or("invalid SQLite reserved-byte count")?;+ if usable < 480 {+ return Err("SQLite usable page size is below 480 bytes".into());+ }+ let schema_format = be_u32(header, 44)?;+ if !(1..=4).contains(&schema_format) {+ return Err("unsupported SQLite schema format".into());+ }+ Ok(reserved)+}++fn parse_wal(wal: &[u8], page_size: usize) -> Result<WalOverlay, String> {+ let header = wal.get(..32).ok_or("truncated WAL header")?;+ let magic = be_u32(header, 0)?;+ if magic != 0x377f0682 && magic != 0x377f0683 {+ return Err("invalid WAL magic".into());+ }+ if be_u32(header, 4)? != 3_007_000 || be_u32(header, 8)? != u32::try_from(page_size).map_err(|_| "page size overflow")? {+ return Err("invalid WAL page size or format version".into());+ }+ let stride = page_size.checked_add(24).ok_or("WAL frame size overflow")?;+ let body = wal.get(32..).ok_or("truncated WAL body")?;+ if body.len() % stride != 0 {+ return Err("WAL ends with a partial frame".into());+ }+ let salt = (be_u32(header, 16)?, be_u32(header, 20)?);+ let frame_count = body.len() / stride;+ if frame_count > MAX_CELLS {+ return Err("WAL has too many frames".into());+ }+ let mut overlay = HashMap::new();+ let mut page_count = None;+ for frame_no in 0..frame_count {+ let start = frame_no.checked_mul(stride).ok_or("WAL frame offset overflow")?;+ let end = start.checked_add(stride).ok_or("WAL frame offset overflow")?;+ let frame = body.get(start..end).ok_or("truncated WAL frame")?;+ let page_id = be_u32(frame, 0)?;+ if page_id == 0 || page_id > MAX_PAGE_ID || salt != (be_u32(frame, 8)?, be_u32(frame, 12)?) {+ return Err("invalid WAL frame page or salt".into());+ }+ let commit = be_u32(frame, 4)?;+ if commit != 0 {+ if commit > MAX_PAGE_ID {+ return Err("WAL commit database size is too large".into());+ }+ page_count = Some(commit);+ }+ overlay.insert(page_id, frame.get(24..).ok_or("truncated WAL page")?.to_vec());+ }+ Ok(WalOverlay { page_size, frame_count, pages: overlay, page_count })+}++fn decode_record(payload: &[u8], ncols: usize, encoding: TextEncoding) -> Result<Vec<Value>, String> {+ let mut at = 0;+ let header_len = usize_from_u64(read_varint(payload, &mut at)?, "record header length")?;+ if header_len < at || header_len > payload.len() {+ return Err("invalid SQLite record header length".into());+ }+ let mut serials = Vec::new();+ while at < header_len {+ if serials.len() >= ncols {+ return Err("SQLite record has more columns than table schema".into());+ }+ serials.push(read_varint(payload, &mut at)?);+ }+ if at != header_len {+ return Err("SQLite record header is malformed".into());+ }+ let mut body = header_len;+ let mut values = Vec::with_capacity(ncols);+ for serial in serials {+ let (value, size) = serial_value(payload, &mut body, serial, encoding)?;+ values.push(value);+ if values.len() > MAX_COLUMNS {+ return Err("SQLite record has too many columns".into());+ }+ let _ = size;+ }+ if body > payload.len() {+ return Err("SQLite record body is truncated".into());+ }+ values.resize(ncols, Value::Null);+ Ok(values)+}++fn serial_value(payload: &[u8], body: &mut usize, serial: u64, encoding: TextEncoding) -> Result<(Value, usize), String> {+ let (kind, len) = match serial {+ 0 => (0, 0),+ 1 => (1, 1),+ 2 => (2, 2),+ 3 => (3, 3),+ 4 => (4, 4),+ 5 => (5, 6),+ 6 => (6, 8),+ 7 => (7, 8),+ 8 => (8, 0),+ 9 => (9, 0),+ 10 | 11 => return Err("reserved SQLite serial type".into()),+ n if n >= 12 => {+ let len = usize_from_u64((n - if n % 2 == 0 { 12 } else { 13 }) / 2, "SQLite field length")?;+ (if n % 2 == 0 { 12 } else { 13 }, len)+ }+ _ => return Err("invalid SQLite serial type".into()),+ };+ let end = body.checked_add(len).ok_or("SQLite record body offset overflow")?;+ let bytes = payload.get(*body..end).ok_or("SQLite record body is truncated")?;+ let value = match kind {+ 0 => Value::Null,+ 1 => Value::Integer(i64::from(i8::from_be_bytes([*bytes.first().ok_or("truncated integer")?]))),+ 2 => Value::Integer(i64::from(i16::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))),+ 3 => {+ let first = *bytes.first().ok_or("truncated integer")?;+ let second = *bytes.get(1).ok_or("truncated integer")?;+ let third = *bytes.get(2).ok_or("truncated integer")?;+ let n = (u32::from(first) << 16) | (u32::from(second) << 8) | u32::from(third);+ let signed = if n & 0x80_0000 != 0 { n | 0xff00_0000 } else { n };+ Value::Integer(i64::from(i32::from_be_bytes(signed.to_be_bytes())))+ }+ 4 => Value::Integer(i64::from(i32::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))),+ 5 => {+ let n = bytes.iter().fold(0u64, |acc, b| (acc << 8) | u64::from(*b));+ let signed = if n & (1 << 47) != 0 { n | 0xffff_0000_0000_0000 } else { n };+ Value::Integer(i64::from_be_bytes(signed.to_be_bytes()))+ }+ 6 => Value::Integer(i64::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?)),+ 7 => {+ let n = f64::from_be_bytes(bytes.try_into().map_err(|_| "truncated real")?);+ if !n.is_finite() {+ return Err("non-finite SQLite real value".into());+ }+ Value::Real(n)+ }+ 8 => Value::Integer(0),+ 9 => Value::Integer(1),+ 12 => Value::Blob(bytes.to_vec()),+ 13 => Value::Text(decode_text(bytes, encoding)?),+ _ => return Err("invalid SQLite field type".into()),+ };+ *body = end;+ Ok((value, len))+}++fn decode_text(bytes: &[u8], encoding: TextEncoding) -> Result<String, String> {+ match encoding {+ TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| "invalid UTF-8 SQLite text".into()),+ TextEncoding::Utf16Le | TextEncoding::Utf16Be => {+ if bytes.len() % 2 != 0 {+ return Err("odd-length UTF-16 SQLite text".into());+ }+ let units: Result<Vec<u16>, String> = bytes+ .as_chunks::<2>()+ .0+ .iter()+ .map(|pair| {+ let first = *pair.first().ok_or("truncated UTF-16 SQLite text")?;+ let second = *pair.get(1).ok_or("truncated UTF-16 SQLite text")?;+ Ok(match encoding {+ TextEncoding::Utf16Le => u16::from_le_bytes([first, second]),+ TextEncoding::Utf16Be => u16::from_be_bytes([first, second]),+ TextEncoding::Utf8 => return Err("invalid UTF-16 encoding state".into()),+ })+ })+ .collect();+ String::from_utf16(&units?).map_err(|_| "invalid UTF-16 SQLite text".into())+ }+ }+}++fn parse_create_table(sql: &str) -> Result<(Vec<String>, Option<usize>), String> {+ let tokens = tokenize_sql(sql)?;+ if !tokens.first().is_some_and(|t| t.word_is("CREATE")) {+ return Err("malformed CREATE TABLE SQL".into());+ }+ let mut kind_at = 1usize;+ if tokens.get(kind_at).is_some_and(|t| t.word_is("TEMPORARY") || t.word_is("TEMP")) {+ kind_at += 1;+ }+ if tokens.get(kind_at).is_some_and(|t| t.word_is("VIRTUAL")) {+ return Err("virtual tables are unsupported".into());+ }+ if !tokens.get(kind_at).is_some_and(|t| t.word_is("TABLE")) {+ return Err("malformed CREATE TABLE SQL".into());+ }+ let open = tokens.iter().position(|t| t.is_symbol(b'(')).ok_or("CREATE TABLE has no column list")?;+ let mut depth = 0usize;+ let mut close = None;+ for (i, token) in tokens.iter().enumerate().skip(open) {+ if token.is_symbol(b'(') {+ depth += 1;+ } else if token.is_symbol(b')') {+ depth = depth.checked_sub(1).ok_or("unbalanced CREATE TABLE parentheses")?;+ if depth == 0 {+ close = Some(i);+ break;+ }+ }+ }+ let close = close.ok_or("unterminated CREATE TABLE column list")?;+ if tokens+ .get(close + 1..)+ .ok_or("invalid CREATE TABLE suffix")?+ .windows(2)+ .any(|w| w.first().is_some_and(|t| t.word_is("WITHOUT")) && w.get(1).is_some_and(|t| t.word_is("ROWID")))+ {+ return Err("WITHOUT ROWID tables are unsupported".into());+ }+ let body = tokens.get(open + 1..close).ok_or("invalid CREATE TABLE column list")?;+ let mut columns = Vec::new();+ let mut alias = None;+ let mut integer_columns = HashSet::new();+ let mut table_primary = None;+ let mut start = 0usize;+ let mut nested = 0usize;+ for i in 0..=body.len() {+ let comma = i == body.len() || (nested == 0 && body.get(i).is_some_and(|t| t.is_symbol(b',')));+ if comma {+ let part = body.get(start..i).ok_or("invalid CREATE TABLE column range")?;+ if part.is_empty() {+ return Err("empty CREATE TABLE column definition".into());+ }+ if is_table_constraint(part) {+ if table_primary.is_none() {+ table_primary = table_primary_column(part);+ }+ } else {+ let name = part.first().and_then(Token::identifier).ok_or("column name is not an identifier")?;+ if columns.iter().any(|old| old == name) {+ return Err("duplicate SQLite column name".into());+ }+ let index = columns.len();+ if exact_integer_decl(part) {+ integer_columns.insert(name.to_ascii_lowercase());+ }+ if integer_primary_key_alias(part) {+ alias = Some(index);+ }+ columns.push(name.to_string());+ }+ start = i + 1;+ } else if body.get(i).is_some_and(|token| token.is_symbol(b'(')) {+ nested += 1;+ } else if body.get(i).is_some_and(|token| token.is_symbol(b')')) {+ nested = nested.checked_sub(1).ok_or("unbalanced CREATE TABLE column definition")?;+ }+ }+ if columns.is_empty() {+ return Err("CREATE TABLE has no columns".into());+ }+ if alias.is_none() {+ if let Some(primary) = table_primary {+ let key = primary.to_ascii_lowercase();+ if integer_columns.contains(&key) {+ alias = columns.iter().position(|column| column.eq_ignore_ascii_case(&primary));+ }+ }+ }+ Ok((columns, alias))+}++fn is_table_constraint(part: &[Token]) -> bool {+ part.first().is_some_and(|token| ["CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN"].iter().any(|word| token.word_is(word)))+}++fn exact_integer_decl(part: &[Token]) -> bool {+ if !part.get(1).is_some_and(|token| token.word_is("INTEGER")) {+ return false;+ }+ match part.get(2) {+ None => true,+ Some(token) if token.is_symbol(b'(') => false,+ Some(token) => ["PRIMARY", "NOT", "NULL", "UNIQUE", "CHECK", "DEFAULT", "COLLATE", "REFERENCES", "CONSTRAINT", "GENERATED"]+ .iter()+ .any(|word| token.word_is(word)),+ }+}++fn table_primary_column(part: &[Token]) -> Option<String> {+ let primary = part.iter().position(|token| token.word_is("PRIMARY"))?;+ if !part.get(primary + 1).is_some_and(|token| token.word_is("KEY")) {+ return None;+ }+ let open = primary + 2;+ if !part.get(open).is_some_and(|token| token.is_symbol(b'(')) {+ return None;+ }+ let close = (open + 1..part.len()).find(|index| part.get(*index).is_some_and(|token| token.is_symbol(b')')))?;+ let terms = part.get(open + 1..close)?;+ if terms.len() == 1 || (terms.len() == 2 && terms.get(1).is_some_and(|token| token.word_is("ASC") || token.word_is("DESC"))) {+ return terms.first().and_then(Token::identifier).map(str::to_owned);+ }+ None+}++fn integer_primary_key_alias(part: &[Token]) -> bool {+ if !exact_integer_decl(part) {+ return false;+ }+ let mut primary = None;+ let mut depth = 0usize;+ for i in 2..part.len() {+ if part.get(i).is_some_and(|token| token.is_symbol(b'(')) {+ depth += 1;+ } else if part.get(i).is_some_and(|token| token.is_symbol(b')')) {+ depth = depth.saturating_sub(1);+ } else if depth == 0 && part.get(i).is_some_and(|token| token.word_is("PRIMARY")) && part.get(i + 1).is_some_and(|t| t.word_is("KEY")) {+ primary = Some(i);+ break;+ }+ }+ let Some(i) = primary else { return false };+ !part.get(i + 2).is_some_and(|token| token.word_is("DESC")) && !part.get(i + 2).is_some_and(|token| token.is_symbol(b'('))+}++#[derive(Clone, Debug)]+enum Token {+ Word(String),+ Quoted(String),+ StringLiteral,+ Symbol(u8),+}++impl Token {+ fn is_symbol(&self, symbol: u8) -> bool {+ matches!(self, Token::Symbol(value) if *value == symbol)+ }++ fn word_is(&self, wanted: &str) -> bool {+ matches!(self, Token::Word(word) if word.eq_ignore_ascii_case(wanted))+ }++ fn identifier(&self) -> Option<&str> {+ match self {+ Token::Word(word) | Token::Quoted(word) => Some(word),+ _ => None,+ }+ }+}++fn tokenize_sql(sql: &str) -> Result<Vec<Token>, String> {+ let bytes = sql.as_bytes();+ let mut out = Vec::new();+ let mut at = 0usize;+ while at < bytes.len() {+ if bytes.get(at).is_some_and(u8::is_ascii_whitespace) {+ at += 1;+ } else if bytes.get(at) == Some(&b'-') && bytes.get(at + 1) == Some(&b'-') {+ at += 2;+ while at < bytes.len() && bytes.get(at) != Some(&b'\n') {+ at += 1;+ }+ } else if bytes.get(at) == Some(&b'/') && bytes.get(at + 1) == Some(&b'*') {+ at += 2;+ let mut closed = false;+ while at + 1 < bytes.len() {+ if bytes.get(at) == Some(&b'*') && bytes.get(at + 1) == Some(&b'/') {+ at += 2;+ closed = true;+ break;+ }+ at += 1;+ }+ if !closed {+ return Err("unterminated SQL comment".into());+ }+ } else if matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';')) {+ out.push(Token::Symbol(*bytes.get(at).ok_or("truncated SQL punctuation")?));+ at += 1;+ } else if matches!(bytes.get(at), Some(b'"' | b'`' | b'[')) {+ let quote = *bytes.get(at).ok_or("truncated SQL quote")?;+ let end_quote = if quote == b'[' { b']' } else { quote };+ at += 1;+ let mut value = Vec::new();+ let mut closed = false;+ while at < bytes.len() {+ if bytes.get(at) == Some(&end_quote) {+ if bytes.get(at + 1) == Some(&end_quote) {+ value.push(end_quote);+ at += 2;+ } else {+ at += 1;+ closed = true;+ break;+ }+ } else {+ value.push(*bytes.get(at).ok_or("truncated SQL identifier")?);+ at += 1;+ }+ }+ if !closed {+ return Err("unterminated quoted SQL identifier".into());+ }+ out.push(Token::Quoted(String::from_utf8(value).map_err(|_| "invalid SQL identifier encoding")?));+ } else if bytes.get(at) == Some(&b'\'') {+ at += 1;+ let mut closed = false;+ while at < bytes.len() {+ if bytes.get(at) == Some(&b'\'') {+ if bytes.get(at + 1) == Some(&b'\'') {+ at += 2;+ } else {+ at += 1;+ closed = true;+ break;+ }+ } else {+ at += 1;+ }+ }+ if !closed {+ return Err("unterminated SQL string".into());+ }+ out.push(Token::StringLiteral);+ } else {+ let start = at;+ while at < bytes.len()+ && !bytes.get(at).is_some_and(u8::is_ascii_whitespace)+ && !matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';' | b'\'' | b'"' | b'`' | b'['))+ {+ at += 1;+ }+ if start == at {+ return Err("unsupported SQL punctuation".into());+ }+ out.push(Token::Word(sql.get(start..at).ok_or("invalid SQL token boundary")?.to_string()));+ }+ }+ Ok(out)+}++fn sqlite_page_size(value: u16) -> Result<usize, String> {+ let size = if value == 1 { 65_536 } else { usize::from(value) };+ if !(512..=65_536).contains(&size) || !size.is_power_of_two() {+ return Err("invalid SQLite page size".into());+ }+ Ok(size)+}++fn read_varint(bytes: &[u8], at: &mut usize) -> Result<u64, String> {+ let mut value = 0u64;+ for i in 0..9 {+ let byte = *bytes.get(*at).ok_or("truncated SQLite varint")?;+ *at = (*at).checked_add(1).ok_or("SQLite varint offset overflow")?;+ if i == 8 {+ value = (value << 8) | u64::from(byte);+ return Ok(value);+ }+ value = (value << 7) | u64::from(byte & 0x7f);+ if byte & 0x80 == 0 {+ return Ok(value);+ }+ }+ Err("invalid SQLite varint".into())+}++fn usize_from_u64(value: u64, label: &str) -> Result<usize, String> {+ usize::try_from(value).map_err(|_| format!("{label} exceeds host size"))+}++fn be_u16(bytes: &[u8], at: usize) -> Result<u16, String> {+ let end = at.checked_add(2).ok_or("SQLite integer offset overflow")?;+ bytes.get(at..end).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite integer".into())+}++fn be_u16_at(bytes: &[u8], at: usize) -> Result<u16, String> {+ be_u16(bytes, at)+}++fn be_u32(bytes: &[u8], at: usize) -> Result<u32, String> {+ let end = at.checked_add(4).ok_or("SQLite integer offset overflow")?;+ bytes.get(at..end).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated SQLite integer".into())+}++fn be_u32_at(bytes: &[u8], at: usize) -> Result<u32, String> {+ be_u32(bytes, at)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn varint(mut value: u64) -> Vec<u8> {+ let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)];+ value >>= 7;+ while value != 0 {+ out.push(u8::try_from(value & 0x7f).unwrap_or(0) | 0x80);+ value >>= 7;+ }+ out.reverse();+ out+ }++ fn leaf(page: &mut [u8], base: usize, rowid: u64, payload: &[u8]) {+ page[base] = 0x0d;+ page[base + 3..base + 5].copy_from_slice(&1u16.to_be_bytes());+ let mut cell = varint(payload.len() as u64);+ cell.extend(varint(rowid));+ cell.extend(payload);+ let start = page.len() - cell.len();+ page[start..].copy_from_slice(&cell);+ page[base + 8..base + 10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());+ page[base + 5..base + 7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());+ }++ #[test]+ fn schema_names_and_integer_primary_key_alias_are_decoded() {+ let sql = r#"CREATE TABLE "t" ("id_local" INTEGER PRIMARY KEY, "odd,name" TEXT, PRIMARY KEY ("id_local"))"#;+ let (names, alias) = parse_create_table(sql).unwrap();+ assert_eq!(names, vec!["id_local", "odd,name"]);+ assert_eq!(alias, Some(0));+ let (_, no_alias) = parse_create_table("CREATE TABLE t (id INTEGER PRIMARY KEY DESC, value TEXT)").unwrap();+ assert_eq!(no_alias, None);+ let (table_names, table_alias) = parse_create_table("CREATE TABLE t (id INTEGER, value TEXT, PRIMARY KEY(id))").unwrap();+ assert_eq!(table_names, vec!["id", "value"]);+ assert_eq!(table_alias, Some(0));+ }++ #[test]+ fn synthetic_btree_substitutes_rowid_and_reads_schema() {+ let mut main = vec![0u8; 2 * 512];+ main[..16].copy_from_slice(b"SQLite format 3\0");+ main[16..18].copy_from_slice(&512u16.to_be_bytes());+ main[18] = 1;+ main[19] = 1;+ main[21] = 64;+ main[22] = 32;+ main[23] = 32;+ main[28..32].copy_from_slice(&2u32.to_be_bytes());+ main[44..48].copy_from_slice(&4u32.to_be_bytes());+ main[56..60].copy_from_slice(&1u32.to_be_bytes());+ let sql = "CREATE TABLE \"t\" (\"id_local\" INTEGER PRIMARY KEY, \"name\" TEXT)";+ // rootpage is an integer field, so replace fourth text value with a compact record.+ let mut header = vec![23, 15, 15, 1, u8::try_from(13 + 2 * sql.len()).unwrap_or(0)];+ let mut body = b"tablett".to_vec();+ body.push(2);+ body.extend(sql.as_bytes());+ let mut record = varint((1 + header.len()) as u64);+ record.append(&mut header);+ record.extend(body);+ let schema = record;+ leaf(&mut main[512..], 0, 1, &[3, 0, 15, b'x']);+ leaf(&mut main[..512], 100, 1, &schema);+ let db = Database::open_with_wal(main, &[]).unwrap();+ let tables = db.live_tables().unwrap();+ assert!(tables.iter().any(|table| table.name == "t"));+ let rows = db.read_table(2, 2).unwrap();+ assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1)));+ }++ #[test]+ fn interior_table_pointer_array_uses_cell_offsets() {+ let mut main = vec![0u8; 4 * 512];+ let mut interior = vec![0u8; 512];+ interior[0] = 0x05;+ interior[3..5].copy_from_slice(&1u16.to_be_bytes());+ interior[8..12].copy_from_slice(&4u32.to_be_bytes());+ interior[12..14].copy_from_slice(&500u16.to_be_bytes());+ interior[500..504].copy_from_slice(&3u32.to_be_bytes());+ interior[504] = 1;+ main[512..1024].copy_from_slice(&interior);+ leaf(&mut main[1024..1536], 0, 1, &[2, 0]);+ leaf(&mut main[1536..], 0, 2, &[2, 0]);+ let db = Database {+ main,+ overlay: HashMap::new(),+ page_size: 512,+ reserved: 0,+ encoding: TextEncoding::Utf8,+ page_count: 4,+ tables: Vec::new(),+ rowid_aliases: HashMap::new(),+ };+ assert_eq!(db.read_table(2, 1).unwrap().len(), 2);+ }++ #[test]+ fn negative_rowid_varint_is_signed() {+ let mut main = vec![0u8; 512];+ let page = &mut main[..];+ page[100] = 0x0d;+ page[103..105].copy_from_slice(&1u16.to_be_bytes());+ let start = 500usize;+ page[start] = 2;+ for offset in 1..10 {+ if let Some(byte) = page.get_mut(start + offset) {+ *byte = 0xff;+ }+ }+ page[start + 10..start + 12].copy_from_slice(&[2, 0]);+ page[108..110].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());+ page[105..107].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes());+ let mut aliases = HashMap::new();+ aliases.insert(1, 0);+ let db = Database {+ main,+ overlay: HashMap::new(),+ page_size: 512,+ reserved: 0,+ encoding: TextEncoding::Utf8,+ page_count: 1,+ tables: Vec::new(),+ rowid_aliases: aliases,+ };+ let rows = db.read_table(1, 1).unwrap();+ assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(-1)));+ }++ #[test]+ fn reserved_page_trailer_cannot_supply_cells() {+ let mut main = vec![0u8; 512];+ main[100] = 0x0d;+ main[103..105].copy_from_slice(&1u16.to_be_bytes());+ main[108..110].copy_from_slice(&490u16.to_be_bytes());+ let db = Database {+ main,+ overlay: HashMap::new(),+ page_size: 512,+ reserved: 32,+ encoding: TextEncoding::Utf8,+ page_count: 1,+ tables: Vec::new(),+ rowid_aliases: HashMap::new(),+ };+ assert!(db.read_table(1, 1).is_err());+ }++ #[test]+ fn utf16_and_overflow_payloads_are_supported() {+ let mut payload = vec![2, 17, b'A', 0];+ assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Le).unwrap(), vec![Value::Text("A".into())]);+ payload.clear();+ payload.extend([2, 17, 0, b'A']);+ assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Be).unwrap(), vec![Value::Text("A".into())]);++ let mut main = vec![0u8; 2 * 512];+ let db = Database {+ main: std::mem::take(&mut main),+ overlay: HashMap::new(),+ page_size: 512,+ reserved: 0,+ encoding: TextEncoding::Utf8,+ page_count: 2,+ tables: Vec::new(),+ rowid_aliases: HashMap::new(),+ };+ let mut cell_page = vec![b'x'; 512];+ cell_page[100..104].copy_from_slice(&2u32.to_be_bytes());+ let inline_page = vec![b'i'; 512];+ assert_eq!(db.payload(&inline_page, 0, 110).unwrap(), vec![b'i'; 110]);+ let overflow_page = vec![b'y'; 512];+ let mut db = db;+ db.main[512..].copy_from_slice(&overflow_page);+ let mut with_overflow = db.overlay;+ with_overflow.insert(2, overflow_page);+ db.overlay = with_overflow;+ let bytes = db.payload(&cell_page, 0, 600).unwrap();+ assert_eq!(bytes.len(), 600);+ }++ #[test]+ fn malformed_varint_and_wal_cycle_fail_cleanly() {+ let mut at = 0;+ assert!(read_varint(&[0x80; 8], &mut at).is_err());+ let mut wal = vec![0u8; 32 + 24 + 512];+ wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes());+ wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes());+ wal[8..12].copy_from_slice(&512u32.to_be_bytes());+ wal[16..20].copy_from_slice(&7u32.to_be_bytes());+ wal[20..24].copy_from_slice(&8u32.to_be_bytes());+ wal[32..36].copy_from_slice(&1u32.to_be_bytes());+ wal[36..40].copy_from_slice(&1u32.to_be_bytes());+ wal[40..44].copy_from_slice(&7u32.to_be_bytes());+ wal[44..48].copy_from_slice(&8u32.to_be_bytes());+ let parsed = parse_wal(&wal, 512).unwrap();+ assert_eq!(parsed.frame_count, 1);+ assert_eq!(parsed.page_count, Some(1));+ assert_eq!(parsed.pages.get(&1).map(Vec::len), Some(512));+ let mut page = vec![0u8; 512];+ page[16..18].copy_from_slice(&512u16.to_be_bytes());+ let db = Database {+ main: vec![0; 512],+ overlay: HashMap::new(),+ page_size: 512,+ reserved: 0,+ encoding: TextEncoding::Utf8,+ page_count: 1,+ tables: Vec::new(),+ rowid_aliases: HashMap::new(),+ };+ assert!(db.payload(&page, 0, 600).is_err());+ }+}diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rsindex 1b7b99a44..1252fdb35 100644--- a/crates/ui-egui/src/lib.rs+++ b/crates/ui-egui/src/lib.rs@@ -13,6 +13,7 @@ pub mod headless;pub mod i18n;pub mod icons;pub mod import;+pub mod lightroom_import;pub mod links;pub mod menubar;pub mod menus;@@ -187,6 +188,10 @@ pub struct LightcraftApp {pub import: Option<import::ImportTask>,/// A folder scan in progress (feeds the import review).pub scan: Option<import::ScanTask>,+ /// A Lightroom catalog inspect/import in progress.+ pub lightroom: Option<lightroom_import::LightroomTask>,+ /// Last terminal Lightroom result, exposed by the command's status/wait response.+ pub lightroom_last: Option<Value>,/// A background export in progress.pub export: Option<export_task::ExportTask>,/// Background file-system work of other commands (Find Missing Photos, auto import…).@@ -246,6 +251,8 @@ impl LightcraftApp {merge: merge::MergeState::default(),import: None,scan: None,+ lightroom: None,+ lightroom_last: None,export: None,tasks: Default::default(),last_export_result: None,@@ -621,6 +628,7 @@ impl LightcraftApp {merge::poll(self, ctx);import::poll_scan(self, ctx);import::tick(self, ctx);+ lightroom_import::tick(self, ctx);tasks::poll(self, ctx);self.preview_build_status(ctx);self.save_status(ctx);@@ -642,7 +650,7 @@ impl LightcraftApp {if let Some(folder) = self.session.import_defaults.auto_folder.clone() {const LABEL: &str = "Auto Import";let now = ctx.input(|i| i.time);- if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) {+ if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && self.lightroom.is_none() && !self.tasks.is_running(LABEL) {self.ui.auto_import_at = now;let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder);let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result<Vec<(String, u64)>, String>| {@@ -683,10 +691,12 @@ impl LightcraftApp {let _ = self.run("file.importPresets", serde_json::json!({"paths": presets}));}// read and added on a worker thread (dropped folders can be large, or on a slow drive)- if !photos.is_empty()- && let Err(e) = import::start_paths(self, photos)- {- self.toast(ctx, e);+ if !photos.is_empty() {+ if self.lightroom.is_some() {+ self.toast(ctx, "Finish Lightroom catalog import before adding photos");+ } else if let Err(e) = import::start_paths(self, photos) {+ self.toast(ctx, e);+ }}}}@@ -823,6 +833,7 @@ impl LightcraftApp {panels::library_problem::show(self, &ctx);import::progress(self, &ctx);import::scan_progress(self, &ctx);+ lightroom_import::progress(self, &ctx);export_task::poll(self, &ctx);panels::grid::drag_feedback(self, &ctx);panels::toast(self, &ctx);diff --git a/crates/ui-egui/src/lightroom_import.rs b/crates/ui-egui/src/lightroom_import.rsnew file mode 100644index 000000000..6b2e0a2d7--- /dev/null+++ b/crates/ui-egui/src/lightroom_import.rs@@ -0,0 +1,456 @@+//! Background Lightroom catalog inspection and import.+//!+//! Reading a catalog, probing its originals and staging its archive can take minutes on a+//! removable or network drive. This task keeps that work off the owner thread, commits prepared+//! data only on the owner thread, then finishes the archive index on a worker.++use std::path::PathBuf;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, mpsc};++use egui::Align2;+use serde_json::{Value, json};++use crate::LightcraftApp;+use crate::theme::Tokens;+use crate::widgets::register;++#[derive(Clone, Copy)]+enum Kind {+ Inspect,+ Import,+}++enum Message {+ Inspected(Result<Value, String>),+ Prepared(Result<lightcraft_engine::lightroom_job::PreparedLightroom, String>),+ Finalized(Result<(), String>),+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum Phase {+ Reading,+ Committing,+ Finalizing,+}++/// A Lightroom task in flight. The receiver is polled from the egui owner thread.+pub struct LightroomTask {+ kind: Kind,+ path: PathBuf,+ cancel: Arc<AtomicBool>,+ total: Arc<AtomicUsize>,+ done: Arc<AtomicUsize>,+ rx: mpsc::Receiver<Message>,+ phase: Phase,+ cancelled: bool,+}++impl LightroomTask {+ #[cfg(not(target_arch = "wasm32"))]+ fn status(&self) -> Value {+ let kind = match self.kind {+ Kind::Inspect => "inspect",+ Kind::Import => "import",+ };+ let phase = match self.phase {+ Phase::Reading => "reading",+ Phase::Committing => "committing",+ Phase::Finalizing => "finalizing",+ };+ json!({+ "running": true,+ "kind": kind,+ "path": self.path,+ "phase": phase,+ "done": self.done.load(Ordering::Relaxed),+ "total": self.total.load(Ordering::Relaxed),+ "cancelled": self.cancelled || self.cancel.load(Ordering::Relaxed),+ })+ }++ fn cancel(&mut self) {+ self.cancelled = true;+ self.cancel.store(true, Ordering::Relaxed);+ }+}++impl Drop for LightroomTask {+ fn drop(&mut self) {+ self.cancel.store(true, Ordering::Relaxed);+ }+}++#[cfg(target_arch = "wasm32")]+fn unsupported_wasm() -> Result<Value, String> {+ Err("Lightroom catalog import is unavailable in browser builds; open the catalog in the native LightCraft app".into())+}++/// Whether a Lightroom task currently owns the catalog transition.+pub fn is_running(app: &LightcraftApp) -> bool {+ app.lightroom.is_some()+}++#[cfg(not(target_arch = "wasm32"))]+fn busy(app: &LightcraftApp) -> Result<(), String> {+ if app.lightroom.is_some() {+ return Err("a Lightroom catalog task is already running".into());+ }+ if app.import.is_some() || app.scan.is_some() {+ return Err("an import is running".into());+ }+ Ok(())+}++#[cfg(not(target_arch = "wasm32"))]+fn spawn_inspect(+ path: PathBuf,+ cancel: Arc<AtomicBool>,+ total: Arc<AtomicUsize>,+ done: Arc<AtomicUsize>,+ tx: mpsc::Sender<Message>,+ ctx: egui::Context,+) -> Result<(), String> {+ std::thread::Builder::new()+ .name("lc-lightroom-inspect".into())+ .spawn(move || {+ let result = lightcraft_engine::guard::catch("Lightroom inspect", || {+ lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &total, &done)+ })+ .and_then(|r| r.map(inspect_report));+ let _ = tx.send(Message::Inspected(result));+ ctx.request_repaint();+ })+ .map(|_| ())+ .map_err(|e| format!("could not start Lightroom inspect: {e}"))+}++#[cfg(not(target_arch = "wasm32"))]+fn spawn_prepare(+ mut job: lightcraft_engine::lightroom_job::LightroomJob,+ cancel: Arc<AtomicBool>,+ tx: mpsc::Sender<Message>,+ ctx: egui::Context,+) -> Result<(), String> {+ std::thread::Builder::new()+ .name("lc-lightroom-import".into())+ .spawn(move || {+ let result = lightcraft_engine::guard::catch("Lightroom import", || job.prepare(&cancel)).and_then(|r| r.map_err(|e| e.to_string()));+ if let Err(mpsc::SendError(Message::Prepared(Ok(prepared)))) = tx.send(Message::Prepared(result)) {+ // The owner dropped the task (for example while closing): staged copies/moves+ // must be put back even though no UI receiver remains to accept the result.+ prepared.rollback();+ }+ ctx.request_repaint();+ })+ .map(|_| ())+ .map_err(|e| format!("could not start Lightroom import: {e}"))+}++#[cfg(not(target_arch = "wasm32"))]+fn spawn_finalize(+ finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,+ tx: mpsc::Sender<Message>,+ ctx: egui::Context,+) -> Result<(), String> {+ std::thread::Builder::new()+ .name("lc-lightroom-finalize".into())+ .spawn(move || {+ let result = lightcraft_engine::guard::catch("Lightroom archive finalization", || finalization.finish())+ .and_then(|r| r.map_err(|e| e.to_string()));+ let _ = tx.send(Message::Finalized(result));+ ctx.request_repaint();+ })+ .map(|_| ())+ .map_err(|e| format!("could not start Lightroom archive finalization: {e}"))+}++#[cfg(target_arch = "wasm32")]+fn spawn_finalize(+ _finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,+ _tx: mpsc::Sender<Message>,+ _ctx: egui::Context,+) -> Result<(), String> {+ unsupported_wasm().map(|_| ())+}++#[cfg(not(target_arch = "wasm32"))]+fn inspect_report(data: lightcraft_engine::lightroom_catalog::CatalogImport) -> Value {+ json!({+ "photos": data.photos.len(),+ "collections": data.collections.iter().filter(|r| r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0) == 0.0).count(),+ "missing": data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p| &p.path).collect::<Vec<_>>(),+ "warnings": data.warnings,+ })+}++fn terminal(app: &mut LightcraftApp, ctx: &egui::Context, kind: Kind, value: Value) {+ let message = if let Some(e) = value.get("error").and_then(Value::as_str) {+ let warning = value.get("indexWarning").and_then(Value::as_str).map_or(String::new(), |w| format!("; index warning: {w}"));+ format!("Lightroom {} failed: {e}{warning}", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })+ } else if value.get("cancelled").and_then(Value::as_bool).unwrap_or(false) {+ format!("Lightroom {} cancelled", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })+ } else if matches!(kind, Kind::Inspect) {+ let photos = value["photos"].as_u64().unwrap_or(0);+ let collections = value["collections"].as_u64().unwrap_or(0);+ let missing = value["missing"].as_array().map_or(0, Vec::len);+ format!("Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing")+ } else {+ let report = value.get("report").unwrap_or(&value);+ let imported = report["imported"].as_u64().unwrap_or(0);+ let failed = report["failed"].as_array().map_or(0, Vec::len);+ let warning = value.get("indexWarning").and_then(Value::as_str);+ if let Some(warning) = warning {+ format!("Lightroom import complete: {imported} photos added; index warning: {warning}")+ } else if failed == 0 {+ format!("Lightroom import complete: {imported} photos added")+ } else {+ format!("Lightroom import complete: {imported} photos added, {failed} failed")+ }+ };+ app.lightroom_last = Some(value);+ app.lightroom = None;+ app.toast_for(ctx, message, 6.0);+}++#[cfg(not(target_arch = "wasm32"))]+fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> Result<Value, String> {+ busy(app)?;+ let cancel = Arc::new(AtomicBool::new(false));+ let total = Arc::new(AtomicUsize::new(0));+ let done = Arc::new(AtomicUsize::new(0));+ let (tx, rx) = mpsc::channel();+ spawn_inspect(path.clone(), cancel.clone(), total.clone(), done.clone(), tx, ctx.clone())?;+ app.lightroom_last = None;+ app.lightroom = Some(LightroomTask { kind: Kind::Inspect, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });+ Ok(json!({"running": true, "kind": "inspect", "path": path}))+}++#[cfg(not(target_arch = "wasm32"))]+fn start_import(app: &mut LightcraftApp, path: PathBuf, update_existing: bool, ctx: &egui::Context) -> Result<Value, String> {+ busy(app)?;+ let job = lightcraft_engine::lightroom_job::LightroomJob::new(&mut app.session, path.clone(), update_existing).map_err(|e| e.to_string())?;+ let cancel = Arc::new(AtomicBool::new(false));+ let total = job.total_atomic();+ let done = job.done_atomic();+ let (tx, rx) = mpsc::channel();+ spawn_prepare(job, cancel.clone(), tx, ctx.clone())?;+ app.lightroom_last = None;+ app.lightroom = Some(LightroomTask { kind: Kind::Import, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });+ Ok(json!({"running": true, "kind": "import", "path": path}))+}++/// Run Lightroom UI commands before the generic engine dispatcher.+pub fn command(app: &mut LightcraftApp, id: &str, p: &Value, ctx: &egui::Context) -> Result<Value, String> {+ #[cfg(target_arch = "wasm32")]+ {+ let _ = (app, id, p, ctx);+ return unsupported_wasm();+ }+ #[cfg(not(target_arch = "wasm32"))]+ {+ let path = p.get("path").and_then(Value::as_str).map(PathBuf::from);+ let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);+ let status = p.get("status").and_then(Value::as_bool).unwrap_or(false) || path.is_none();+ if status {+ if p.get("cancel").and_then(Value::as_bool).unwrap_or(false) {+ if let Some(task) = app.lightroom.as_mut() {+ task.cancel();+ }+ }+ if wait && app.lightroom.is_some() {+ return wait_for(app, ctx, std::time::Duration::from_secs(600));+ }+ return Ok(app.lightroom.as_ref().map(LightroomTask::status).or_else(|| app.lightroom_last.clone()).unwrap_or(Value::Null));+ }+ let Some(path) = path else { return Err("missing `path`".into()) };+ let result = if id == "library.inspectLightroom" {+ start_inspect(app, path, ctx)+ } else {+ start_import(app, path, p.get("updateExisting").and_then(Value::as_bool).unwrap_or(false), ctx)+ }?;+ if wait {+ return wait_for(app, ctx, std::time::Duration::from_secs(600));+ }+ Ok(result)+ }+}++/// Poll owner-thread work, commit a prepared import, and finish its archive index.+pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {+ let Some(mut task) = app.lightroom.take() else { return };+ let message = match task.rx.try_recv() {+ Ok(m) => m,+ Err(mpsc::TryRecvError::Empty) => {+ app.lightroom = Some(task);+ return;+ }+ Err(mpsc::TryRecvError::Disconnected) => {+ terminal(app, ctx, task.kind, json!({"error": "Lightroom worker stopped unexpectedly"}));+ return;+ }+ };+ match message {+ Message::Inspected(result) => match result {+ Ok(report) if !task.cancel.load(Ordering::Relaxed) => terminal(app, ctx, task.kind, report),+ Ok(_) => terminal(app, ctx, task.kind, json!({"cancelled": true})),+ Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => terminal(app, ctx, task.kind, json!({"cancelled": true})),+ Err(e) => terminal(app, ctx, task.kind, json!({"error": e})),+ },+ Message::Prepared(result) => {+ let prepared = match result {+ Ok(p) => p,+ Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => {+ terminal(app, ctx, task.kind, json!({"cancelled": true}));+ return;+ }+ Err(e) => {+ terminal(app, ctx, task.kind, json!({"error": e}));+ return;+ }+ };+ if task.cancel.load(Ordering::Relaxed) || prepared.is_cancelled() || !prepared.token().matches(&app.session) {+ prepared.rollback();+ terminal(+ app,+ ctx,+ task.kind,+ if task.cancel.load(Ordering::Relaxed) {+ json!({"cancelled": true})+ } else {+ json!({"error": "Lightroom import belongs to a different open library"})+ },+ );+ return;+ }+ task.phase = Phase::Committing;+ let mut finalization = None;+ let mut report = None;+ let committed = app.session.execute_fn("library.importLightroom", |s| {+ let completion = lightcraft_engine::lightroom_job::commit_prepared(s, prepared)?;+ report = Some(completion.report.clone());+ finalization = Some(completion.finalization);+ Ok(completion.report)+ });+ let report = report.unwrap_or(Value::Null);+ task.phase = Phase::Finalizing;+ let (tx, rx) = mpsc::channel();+ let commit_error = committed.as_ref().err().map(ToString::to_string);+ let Some(finalization) = finalization else {+ terminal(app, ctx, task.kind, json!({"error": commit_error.clone().unwrap_or_else(|| "Lightroom import did not commit".into())}));+ return;+ };+ if let Err(e) = spawn_finalize(finalization, tx, ctx.clone()) {+ let mut result = json!({"report": report, "indexWarning": e});+ if let Some(error) = commit_error {+ result["error"] = json!(error);+ }+ terminal(app, ctx, task.kind, result);+ return;+ }+ task.rx = rx;+ task.cancelled = false;+ let mut pending = json!({"running": true, "kind": "import", "report": report});+ if let Some(error) = commit_error {+ pending["error"] = json!(error);+ }+ app.lightroom_last = Some(pending);+ app.lightroom = Some(task);+ }+ Message::Finalized(result) => {+ let pending = app.lightroom_last.take().unwrap_or(Value::Null);+ let report = pending.get("report").cloned().unwrap_or(Value::Null);+ match result {+ Ok(()) if pending.get("error").is_none() => terminal(app, ctx, task.kind, report),+ Ok(()) => terminal(app, ctx, task.kind, json!({"report": report, "error": pending["error"].clone()})),+ Err(e) => {+ let mut value = json!({"report": report, "indexWarning": e});+ if let Some(error) = pending.get("error") {+ value["error"] = error.clone();+ }+ terminal(app, ctx, task.kind, value);+ }+ }+ }+ }+ ctx.request_repaint();+}++/// Wait for a task in control/headless mode while still applying owner-thread completion.+#[cfg(not(target_arch = "wasm32"))]+pub fn wait_for(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> Result<Value, String> {+ let start = std::time::Instant::now();+ while app.lightroom.is_some() {+ tick(app, ctx);+ if start.elapsed() >= timeout {+ return Err("timed out waiting for Lightroom task".into());+ }+ std::thread::sleep(std::time::Duration::from_millis(2));+ }+ Ok(app.lightroom_last.clone().unwrap_or(Value::Null))+}++#[cfg(target_arch = "wasm32")]+pub fn wait_for(_app: &mut LightcraftApp, _ctx: &egui::Context, _timeout: std::time::Duration) -> Result<Value, String> {+ unsupported_wasm()+}++/// The progress window for inspect/import, with cancellation.+pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) {+ let Some(task) = app.lightroom.as_ref() else { return };+ let total = task.total.load(Ordering::Relaxed);+ let done = task.done.load(Ordering::Relaxed);+ let title = match task.kind {+ Kind::Inspect => "Inspecting Lightroom Catalog",+ Kind::Import => "Importing Lightroom Catalog",+ };+ let text = match task.phase {+ Phase::Reading if total == 0 => "Reading Lightroom catalog…".to_string(),+ Phase::Reading => format!("Reading Lightroom catalog… {done} of {total}"),+ Phase::Committing => "Adding Lightroom photos…".to_string(),+ Phase::Finalizing => "Saving Lightroom import index…".to_string(),+ };+ let mut cancel = false;+ let t = Tokens::get(ctx);+ egui::Window::new(title).title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([360.0, 92.0]).show(+ ctx,+ |ui| {+ ui.label(egui::RichText::new(text).color(t.text));+ ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(340.0));+ let r = ui.add_enabled(task.phase == Phase::Reading && !task.cancel.load(Ordering::Relaxed), egui::Button::new("Cancel"));+ register(ui.ctx(), "button:lightroomCancel", r.rect);+ cancel = r.clicked();+ },+ );+ if cancel {+ if let Some(task) = app.lightroom.as_mut() {+ task.cancel();+ }+ ctx.request_repaint();+ }+ if app.lightroom.is_some() {+ ctx.request_repaint_after(std::time::Duration::from_millis(100));+ }+}++#[cfg(test)]+mod tests {+ use super::inspect_report;+ use lightcraft_engine::lightroom_catalog::CatalogImport;++ #[test]+ fn inspect_report_keeps_empty_catalog_shape() {+ let report = inspect_report(CatalogImport {+ source: "synthetic.lrcat".into(),+ photos: Vec::new(),+ collections: Vec::new(),+ members: Vec::new(),+ collection_content: Vec::new(),+ warnings: vec!["warning".into()],+ });+ assert_eq!(report["photos"], 0);+ assert_eq!(report["collections"], 0);+ assert_eq!(report["missing"], serde_json::json!([]));+ assert_eq!(report["warnings"], serde_json::json!(["warning"]));+ }+}diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rsindex 5c841d9e3..b04ba2bd3 100644--- a/crates/ui-egui/src/menus.rs+++ b/crates/ui-egui/src/menus.rs@@ -236,6 +236,10 @@ pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> {/// Handle UI commands; `None` means "not a UI command — send it to the engine".pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Result<Value, String>> {+ if matches!(id, "library.inspectLightroom" | "library.importLightroom") {+ let ctx = egui::Context::default();+ return Some(crate::lightroom_import::command(app, id, p, &ctx));+ }if let Some(language) = language_from_command(id) {app.ui.language = language;// Immediately, not on the next frame: the reply and anything else run this frame@@ -948,11 +952,17 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<ReOk(Value::Null)}"library.browse" if !cfg!(target_arch = "wasm32") => {+ if crate::lightroom_import::is_running(app) {+ return Some(Err("finish Lightroom catalog import before browsing folders".into()));+ }// listed and read in the background (see `import::browse`)let path = p.get("path").and_then(Value::as_str)?;crate::import::browse(app, path, p.get("subfolders").and_then(Value::as_bool))}"file.addPhotos" => {+ if crate::lightroom_import::is_running(app) {+ return Some(Err("finish Lightroom catalog import before adding photos".into()));+ }let paths = match p.get("paths").and_then(Value::as_array) {Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(),None => app.services.pick_files.as_mut().map(|f| f()).unwrap_or_default(),@@ -1278,7 +1288,7 @@ pub fn ui_enabled(app: &LightcraftApp, id: &str) -> bool {"file.exportCurvePresets" => !app.session.curve_presets.is_empty(),"view.compare" => app.session.catalog.len() > 1,"view.fullScreenPreview" | "view.infoOverlay" | "view.navigator" => app.session.active().is_some() || app.ui.fullscreen,- "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some(),+ "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some() && !crate::lightroom_import::is_running(app),"file.backupLibrary" => app.services.backup_library.is_some(),"file.restoreLibrary" => app.services.restore_library.is_some(),"compare.swap" | "compare.makeSelect" => app.ui.view == ViewMode::Compare,diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rsindex e191668d9..745c06ad0 100644--- a/crates/ui-egui/src/panels/settings.rs+++ b/crates/ui-egui/src/panels/settings.rs@@ -530,6 +530,9 @@ fn interface_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) {/// `app.openLibrary {path?}`: close the current library and open (or create) the one at `path`,/// or a folder chosen in a dialog. Remembered as the library to open at launch.pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String> {+ if crate::lightroom_import::is_running(app) {+ return Err("wait for Lightroom catalog import to finish before switching libraries".into());+ }let path = match p.get("path").and_then(Value::as_str) {Some(x) => x.to_string(),None => match app.services.pick_folder.as_mut() {@@ -542,6 +545,7 @@ pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String>};app.session.close_library().map_err(|e| e.to_string())?;app.session.open_library(&path, false).map_err(|e| e.to_string())?;+ app.lightroom_last = None;app.renderer.forget_all();app.ui.compare = None;app.ui.settings.library_path = path.clone();diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.mdindex a5412da19..694441aa1 100644--- a/docs/lightroom-catalog-import.md+++ b/docs/lightroom-catalog-import.md@@ -19,10 +19,15 @@ identity prevents reimport from duplicating virtual copies and collections. Onecatalog changes; recovery archives remain available.Before changing records, source image records, decoded XMP, verbatim develop settings, history,-snapshots and collection content are saved to `Interop/lightroom-import-N.json` in the LightCraft-library. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals+snapshots and collection content are saved to compressed `Interop/lightroom-import-N.lca` archives+in the LightCraft library. Unchanged source data reuses its existing archive. Opaque SQL BLOBs are+skipped; decoded XMP and structured recovery data remain. Each archive is capped at 32 MiB+(uncompressed and on disk), with at most eight archives and 128 MiB total retained. Oldest managed+archives are retired; legacy `.json` archives remain untouched. `Interop/lightroom-index.json` records imported identities. Source catalogs and originalsare never overwritten. Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and-one million rows per table. The SQLite reader is Apache-2.0 `sqlite-core`, written in Rust.+one million rows per table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format specification;+it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. Unsupported+WITHOUT ROWID and virtual tables produce explicit errors.Develop settings reuse the existing XMP/preset mapper. Supported sliders, curves and supportedmask structures remain editable, but this is approximate rendering: camera profiles, Adobe AI@@ -34,5 +39,15 @@ Lightroom's `-999999` deferred-adjustment sentinel is omitted from both catalogit is reported and archived, never clamped into a real slider value. Deferred Adobe Auto Toneis not evaluated by the importer; LightCraft's Auto control remains available after migration.+Native desktop inspection/import runs on a cancellable background worker with progress. Only+prepared catalog operations commit on the owner thread; archive index writing stays on a worker.+Late results from a closed or switched library are rejected, including reopening the same path.+Ordinary edits during preparation remain intact. CLI commands complete synchronously using the+same preparation/commit pipeline; browser builds reject native catalog import.++Archive version 1 begins with the 12 bytes `LC-LRARCH\0\x01Z`, followed by zlib-compressed JSON.+Recovery readers must cap decompression at 32 MiB. JSON retains source identities, photo paths,+settings, decoded XMP, keywords, history, snapshots, collection definitions and membership.+The importer uses shared Rust code on macOS, Windows and Linux. If originals move betweencomputers or volumes, use LightCraft's missing-photo relinking tools to update their paths.diff --git a/docs/parity.md b/docs/parity.mdindex 3aee3d8e0..79ae9f5e1 100644--- a/docs/parity.md+++ b/docs/parity.md@@ -716,7 +716,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06-| LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N || LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo || LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) |-| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-process Rust SQLite reader; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Verbatim settings/history/snapshots/collection rules archived. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs |+| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Settings/history/snapshots/collection rules retained in deduplicated compressed archives (32 MiB each, eight archives/128 MiB aggregate); opaque SQL BLOBs skipped. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs || LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning; no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) || LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) || LRC-LIB-COLLECTIONS | Collections & sets | P1 | ✅ | `cmd:album.create`, `cmd:album.createSmart`, `cmd:album.toggleTarget`, `cmd:album.setTarget`, `cmd:album.clearQuick` | albums + folders (sets), smart albums (rule editor; rules incl. File Path — the whole string, either separator, case-insensitive), Quick Collection and target album (B in the grids adds / removes the selection; album menu ▸ Set as Target Album; the target is marked +) |