Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(cache): deflate large records and compact migrated databases (#8530)
Sean Aye8a9ef577e178
diff --git a/Cargo.lock b/Cargo.lockindex 8213e6adf1..81fd635b45 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3608,6 +3608,7 @@ dependencies = ["chrono","lru 0.16.4","maybe_send",+ "miniz_oxide","pollster","postcard","predicate-index",@@ -3626,6 +3627,7 @@ dependencies = ["cache-core","chrono","maybe_send",+ "miniz_oxide","pollster","predicate-index","proptest",@@ -3640,7 +3642,7 @@ dependencies = [[[package]]name = "cache-wasm"-version = "0.6.48"+version = "0.6.49"dependencies = ["async-lock","cache-core",diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.mdindex f63510e843..d8eb70577e 100644--- a/apps/web/docs/graphql-normalized-cache-plan.md+++ b/apps/web/docs/graphql-normalized-cache-plan.md@@ -322,7 +322,15 @@ paths remain behind the same `CacheHost` interface.| Turso over filesystem IO | Tauri native | records, search projection, mutation queue, optimistic layers, and metadata; WAL mode || Turso over OPFS | browser | the same frozen schema behind the OPFS IO adapter and coordinated worker lifecycle |-Normalized records are serialized with `postcard` into `records.value`.+Normalized records are serialized with `postcard` into `records.value`, behind a+one-byte encoding tag: records of 256 bytes or more are stored deflated when that+is smaller, and everything else stays raw. Other persisted payloads (queued+mutations, optimistic layers, projections) remain plain postcard. A format bump+that only changes the record encoding carries the previous database forward by+dropping records, search projections, index facts, and calendar ranges — all+refetchable — while keeping queued mutations, optimistic layers, and their+projections. Native storage then compacts the migrated file with `VACUUM INTO`, so+the upgrade reclaims the space instead of leaving freed pages behind.Both hosts use the same `Storage` implementation and compound entity keys.Wasm futures are not `Send`, so the `Storage` trait's futures are bound by`MaybeSend` (`crates/maybe_send`): `Send` on native targets, where Tauri drivesdiff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.tsindex f8883b1df1..14919d469d 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts@@ -828,10 +828,10 @@ describe('coordinator runtime protocol', () => {// The current versions moved past the unversioned name, so the lock// names the versioned database that builds before them never held.expect(databaseOwnerLockName('scope')).toBe(- 'macro:turso-opfs:v1:29:graphql-cache:scope:s4.v3.t11'+ 'macro:turso-opfs:v1:29:graphql-cache:scope:s4.v4.t11');expect(databaseOwnerLockName('é')).toBe(- 'macro:turso-opfs:v1:26:graphql-cache:é:s4.v3.t11'+ 'macro:turso-opfs:v1:26:graphql-cache:é:s4.v4.t11');});diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.tsindex 40f10c2159..45f8aa5509 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts@@ -1379,7 +1379,7 @@ export function tabIdFromLivenessLockName(*/export const CACHE_STORAGE_VERSION = {schemaCompatibilityEpoch: 4,- formatVersion: 3,+ formatVersion: 4,storageSchemaVersion: 11,} as const;diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.tsindex f56099eb7d..ddec88cf41 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts@@ -26,7 +26,7 @@ describe('cache takeover messages', () => {`graphql-cache-takeover:${cacheDatabaseIdentity('scope')}`);expect(cacheTakeoverChannelName('scope')).toBe(- 'graphql-cache-takeover:graphql-cache:scope:s4.v3.t11'+ 'graphql-cache-takeover:graphql-cache:scope:s4.v4.t11');});diff --git a/apps/web/tauri/Cargo.lock b/apps/web/tauri/Cargo.lockindex 4fbf4b6482..db5a09680c 100644--- a/apps/web/tauri/Cargo.lock+++ b/apps/web/tauri/Cargo.lock@@ -859,6 +859,7 @@ dependencies = ["chrono","lru","maybe_send",+ "miniz_oxide","postcard","predicate-index","serde",diff --git a/apps/web/tauri/src-tauri/tauri.android.conf.json b/apps/web/tauri/src-tauri/tauri.android.conf.jsonindex c6908f4741..7ddcad1204 100644--- a/apps/web/tauri/src-tauri/tauri.android.conf.json+++ b/apps/web/tauri/src-tauri/tauri.android.conf.json@@ -2,7 +2,7 @@"version": "2.5.0","bundle": {"android": {- "versionCode": 12+ "versionCode": 13}},"$schema": "https://schema.tauri.app/config/2",diff --git a/crates/client/README.md b/crates/client/README.mdindex 110bdadac4..1708af141d 100644--- a/crates/client/README.md+++ b/crates/client/README.md@@ -61,6 +61,29 @@ scheduled automatically in the background. Failures latch the existing storagehealth state without deleting records or pending mutations. Recovery/reset remainsan explicit caller decision after closing the storage.+## Record encoding++`records.value` is postcard behind a one-byte tag: payloads of 256 bytes or more are+deflated (`miniz_oxide`, level 3) when that is smaller, and everything else stays+raw. Large text records — foreign-entity source metadata and email bodies —+dominate the database and shrink about 3×; a real 155 MB record payload became+48 MB. Level 3 keeps that size win with markedly less CPU in the slowest batches+than level 6, which matters because deflate runs while the engine lock is held.+Decompression is bounded per record. Queued mutations, optimistic layers, and+projections keep their unframed postcard encoding, so a format upgrade never+re-encodes them.++Only the immediately prior cache format version is migratable+(`MIGRATABLE_CACHE_FORMAT_VERSIONS`). Opening one drops records, search+projections, index facts, and calendar ranges — all refetchable — and keeps+queued mutations, optimistic layers, and their projections, then rewrites the+namespace in the same transaction. An interrupted migration therefore retries on+the next open. Native owners then `VACUUM INTO` a fresh file and swap it in, so an+upgrade also releases the space the dropped records occupied instead of leaving+free pages in the old file. The browser's physical database name embeds the format+version, so a browser upgrade starts a new file instead of migrating; the previous+file is left for any tab still running the older bundle.+## Network refreshesNormalized refreshes persist only records whose merged contents changed. The hotdiff --git a/crates/client/cache-core/Cargo.toml b/crates/client/cache-core/Cargo.tomlindex fa8ebc1214..d03e6f0f02 100644--- a/crates/client/cache-core/Cargo.toml+++ b/crates/client/cache-core/Cargo.toml@@ -10,6 +10,7 @@ apollo-parser = { workspace = true }chrono = { workspace = true }lru = { workspace = true }maybe_send = { path = "../../maybe_send" }+miniz_oxide = { workspace = true }postcard = { workspace = true }predicate-index = { path = "../../predicate_index" }serde = { workspace = true }diff --git a/crates/client/cache-core/src/codec.rs b/crates/client/cache-core/src/codec.rsindex 8fb63fbd05..e0b68e6254 100644--- a/crates/client/cache-core/src/codec.rs+++ b/crates/client/cache-core/src/codec.rs@@ -1,5 +1,7 @@//! Binary record codec shared by all persistent Turso backends (browser and-//! native hosts). Records are stored as postcard bytes; the logical namespace+//! native hosts). Records are stored as tagged postcard bytes — raw, or+//! deflated when that is smaller — while every other persisted payload stays+//! plain postcard. The logical namespace//! embeds [`CACHE_FORMAT_VERSION`] and [`CACHE_SCHEMA_COMPATIBILITY_EPOCH`],//! and so does the browser's physical database name after//! [`UNVERSIONED_STORAGE_VERSIONS`]. A healthy, graceful@@ -10,12 +12,32 @@use crate::normalize::RecordUpdates;use crate::queue::{PersistedOptimisticLayer, StoredMutation};use crate::value::Record;+use miniz_oxide::{deflate, inflate};use serde::{Serialize, de::DeserializeOwned};use thiserror::Error;/// Bump when the stored representation of [`Record`]/[`CacheValue`]/// (or anything else persisted) changes incompatibly.-pub const CACHE_FORMAT_VERSION: u32 = 3;+pub const CACHE_FORMAT_VERSION: u32 = 4;++/// Records at or above this size may be stored deflated. Smaller payloads stay+/// raw so deflate framing never makes them larger.+const RECORD_COMPRESSION_MIN_BYTES: usize = 256;++/// Deflate level for record payloads.+///+/// On a 155 MB sample of real records, level 6 buys 2% more compression than+/// level 3 (3.29x vs 3.22x) but costs 36% more CPU in the slowest batches, and+/// deflate runs while the cache engine lock is held. Level 3 keeps the size win+/// without stalling other cache operations behind a large record.+const RECORD_DEFLATE_LEVEL: u8 = 3;++/// Upper bound for one inflated record. Real records are a few hundred KiB at+/// most; the bound turns a corrupt length prefix into a codec error.+const RECORD_MAX_INFLATED_BYTES: usize = 64 * 1024 * 1024;++const RECORD_ENCODING_RAW: u8 = 0;+const RECORD_ENCODING_DEFLATE: u8 = 1;/// Bump when a GraphQL schema change makes existing normalized records unsafe.///@@ -31,6 +53,10 @@ pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 4;pub enum CodecError {#[error("corrupt cache payload: {0}")]Corrupt(#[from] postcard::Error),+ #[error("corrupt compressed cache payload: {0}")]+ Inflate(String),+ #[error("unknown record encoding: {0}")]+ UnknownEncoding(u8),}fn encode<T: Serialize>(value: &T) -> Vec<u8> {@@ -42,12 +68,38 @@ fn decode<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, CodecError> {Ok(postcard::from_bytes(bytes)?)}+/// Encodes a normalized record, deflating it when that makes the payload smaller.pub fn encode_record(record: &Record) -> Vec<u8> {- encode(record)+ let raw = encode(record);+ let compressed = (raw.len() >= RECORD_COMPRESSION_MIN_BYTES)+ .then(|| deflate::compress_to_vec(&raw, RECORD_DEFLATE_LEVEL));+ match compressed {+ Some(compressed) if compressed.len() < raw.len() => {+ tagged(RECORD_ENCODING_DEFLATE, compressed)+ }+ _ => tagged(RECORD_ENCODING_RAW, raw),+ }}+/// Decodes a normalized record written by [`encode_record`].pub fn decode_record(bytes: &[u8]) -> Result<Record, CodecError> {- decode(bytes)+ let (encoding, payload) = bytes.split_first().ok_or(CodecError::UnknownEncoding(0))?;+ match *encoding {+ RECORD_ENCODING_RAW => decode(payload),+ RECORD_ENCODING_DEFLATE => {+ let raw = inflate::decompress_to_vec_with_limit(payload, RECORD_MAX_INFLATED_BYTES)+ .map_err(|error| CodecError::Inflate(format!("{error:?}")))?;+ decode(&raw)+ }+ other => Err(CodecError::UnknownEncoding(other)),+ }+}++fn tagged(encoding: u8, payload: Vec<u8>) -> Vec<u8> {+ let mut bytes = Vec::with_capacity(payload.len() + 1);+ bytes.push(encoding);+ bytes.extend_from_slice(&payload);+ bytes}/// Encodes one queued mutation's request and retry metadata.@@ -85,7 +137,19 @@ pub fn decode_record_updates(bytes: &[u8]) -> Result<RecordUpdates, CodecError>/// `scope` is an anonymous, client-generated token supplied by the host. It is/// neither user nor workspace identity.pub fn cache_namespace(scope: &str) -> String {- format!("graphql-cache:{scope}:s{CACHE_SCHEMA_COMPATIBILITY_EPOCH}:v{CACHE_FORMAT_VERSION}")+ cache_namespace_at(scope, CACHE_FORMAT_VERSION)+}++/// Cache format versions whose physical databases this build carries forward.+///+/// Opening one of these drops the derived records (they are refetched) while+/// keeping queued mutations and optimistic layers, then rewrites the namespace.+/// A format that is neither current nor listed here is reset instead.+pub const MIGRATABLE_CACHE_FORMAT_VERSIONS: &[u32] = &[3];++/// Namespace a cache scope used at one cache format version.+pub fn cache_namespace_at(scope: &str, format_version: u32) -> String {+ format!("graphql-cache:{scope}:s{CACHE_SCHEMA_COMPATIBILITY_EPOCH}:v{format_version}")}/// Storage versions `(schema compatibility epoch, cache format, storage@@ -164,14 +228,59 @@ mod tests {#[test]fn rejects_garbage() {assert!(decode_record(&[0xff, 0x00, 0x13, 0x37]).is_err());+ assert!(decode_record(&[]).is_err());+ assert!(decode_record(&[RECORD_ENCODING_DEFLATE, 0x13, 0x37]).is_err());+ }++ #[test]+ fn small_records_stay_raw() {+ let record = record_with_body("short body");+ let bytes = encode_record(&record);+ assert_eq!(bytes.first(), Some(&RECORD_ENCODING_RAW));+ assert_eq!(decode_record(&bytes).unwrap(), record);+ }++ #[test]+ fn large_text_records_deflate_and_round_trip() {+ let body = "review comment line\n".repeat(2_000);+ let record = record_with_body(&body);+ let bytes = encode_record(&record);+ assert_eq!(bytes.first(), Some(&RECORD_ENCODING_DEFLATE));+ assert!(+ bytes.len() < body.len() / 4,+ "compressed {} bytes",+ bytes.len()+ );+ assert_eq!(decode_record(&bytes).unwrap(), record);+ }++ fn record_with_body(body: &str) -> Record {+ let mut record = Record::default();+ record.fields.insert(+ "__typename".into(),+ CacheValue::String("GraphqlSoupForeignEntity".into()),+ );+ record+ .fields+ .insert("sourceMetadata".into(), CacheValue::String(body.into()));+ record}#[test]fn namespace_uses_schema_compatibility_epoch_not_schema_hash() {assert_eq!(cache_namespace("client-token-1"),+ "graphql-cache:client-token-1:s4:v4"+ );+ }++ #[test]+ fn migratable_versions_name_their_own_namespace() {+ assert_eq!(+ cache_namespace_at("client-token-1", 3),"graphql-cache:client-token-1:s4:v3");+ assert!(!MIGRATABLE_CACHE_FORMAT_VERSIONS.contains(&CACHE_FORMAT_VERSION));}#[test]@@ -180,11 +289,11 @@ mod tests {// later epoch embeds its versions, even at that storage schema.assert_eq!(cache_database_name("client-token-1", 11),- "graphql-cache:client-token-1:s4.v3.t11"+ "graphql-cache:client-token-1:s4.v4.t11");assert_eq!(cache_database_name("client-token-1", 12),- "graphql-cache:client-token-1:s4.v3.t12"+ "graphql-cache:client-token-1:s4.v4.t12");assert_eq!(legacy_cache_database_name("client-token-1"),diff --git a/crates/client/cache-turso/AGENTS.md b/crates/client/cache-turso/AGENTS.mdindex d301af51aa..13492ff997 100644--- a/crates/client/cache-turso/AGENTS.md+++ b/crates/client/cache-turso/AGENTS.md@@ -46,6 +46,11 @@ GraphQL cache — native for tests/Tauri, OPFS-backed on wasm32.- The compound-key schema and payload encoding are frozen; an incompatible oruncertain on-disk state is classified for physical reset, never silentlymigrated, copied, or redirected to a fallback backend.+- The one exception is a cache format version listed in+ `MIGRATABLE_CACHE_FORMAT_VERSIONS`: `migrate_previous_format` drops the+ refetchable records, search projections, index facts, and calendar ranges in+ one transaction, keeps queued mutations, optimistic layers, and their+ projections, and rewrites the namespace. Everything else still resets.## Relateddiff --git a/crates/client/cache-turso/Cargo.toml b/crates/client/cache-turso/Cargo.tomlindex f202dd06ff..baaca7f58c 100644--- a/crates/client/cache-turso/Cargo.toml+++ b/crates/client/cache-turso/Cargo.toml@@ -24,6 +24,7 @@ turso_core = { workspace = true, features = ["fs", "pure-rust-crypto", "uuid"] }[dev-dependencies]chrono = { workspace = true }+miniz_oxide = { workspace = true }pollster = { workspace = true }proptest = { workspace = true }serde_json = { workspace = true }diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rsindex 434db449af..57a81b8ad0 100644--- a/crates/client/cache-turso/src/storage.rs+++ b/crates/client/cache-turso/src/storage.rs@@ -2,7 +2,8 @@ use crate::driver;use crate::key::RecordKey;use crate::{PhysicalResetReason, TursoStorageError};use cache_core::codec::{- cache_namespace, decode_record, decode_record_updates, encode_record, encode_record_updates,+ MIGRATABLE_CACHE_FORMAT_VERSIONS, cache_namespace, cache_namespace_at, decode_record,+ decode_record_updates, encode_record, encode_record_updates,};use cache_core::predicate::reconciliation::{PredicateBaselineEntry, PredicateReconciliation, predicate_membership,@@ -226,7 +227,7 @@ impl TursoStorage {let result = initialize(&connection, scope, fresh);drop(connection);match result {- Ok(()) => Ok((+ Ok(_) => Ok((Self {health: AtomicU8::new(0),session,@@ -612,12 +613,107 @@ impl TursoFileDatabase {/// Opens this database, validating compatibility and queued writes for `scope`./// Full-file integrity scans are explicit via [`TursoStorage::check_integrity`].+ ///+ /// A database carried forward from a migratable format is compacted first, so+ /// the upgrade also releases the space its dropped records occupied.pub fn open(&self, scope: &str) -> Result<TursoStorage, TursoStorageError> {let fresh = !self.path.exists();+ let (storage, migrated) = self.open_with_outcome(scope, fresh)?;+ if !migrated {+ return Ok(storage);+ }+ self.compact_after_migration(storage, scope)+ }++ fn open_with_outcome(+ &self,+ scope: &str,+ fresh: bool,+ ) -> Result<(TursoStorage, bool), TursoStorageError> {let io: Arc<dyn IO> = Arc::new(PlatformIO::new().map_err(TursoStorageError::turso)?);open_native_database(io, &self.turso_path, scope, fresh)}+ /// Copy the migrated database's small live set into a fresh file and swap it+ /// in. Freed pages are reused but never returned, so without this an upgrade+ /// would keep the previous file size indefinitely.+ ///+ /// Compaction is an optimization on top of an already-committed migration:+ /// every failure falls back to the migrated database, because reporting a+ /// reset-required error here would make `open_or_reset` delete the queued+ /// mutations the migration just preserved.+ fn compact_after_migration(+ &self,+ storage: TursoStorage,+ scope: &str,+ ) -> Result<TursoStorage, TursoStorageError> {+ let compact_path = self.path.with_extension("migrated");+ let _ = remove_native_file(&compact_path);+ let vacuumed = compact_path+ .to_str()+ .ok_or(TursoStorageError::InvalidInput)+ .and_then(|compact_path| {+ driver::execute(+ &storage.connection(),+ &format!("VACUUM INTO {}", sql_literal(compact_path)),+ Vec::new(),+ )+ })+ .is_ok();+ if !vacuumed {+ let _ = remove_native_file(&compact_path);+ return Ok(storage);+ }+ // The compacted file is complete; a close failure cannot invalidate it.+ let _ = storage.try_close();+ if self.install_compacted(&compact_path).is_err() {+ let _ = remove_native_file(&compact_path);+ return Ok(self.open_with_outcome(scope, false)?.0);+ }+ Ok(self.open_with_outcome(scope, false)?.0)+ }++ /// Move the compacted database into place while keeping the original+ /// recoverable, so an interrupted swap never leaves no database behind.+ fn install_compacted(&self, compact_path: &Path) -> std::io::Result<()> {+ let backup_path = self.path.with_extension("backup");+ for suffix in COMPANION_SUFFIXES {+ let _ = std::fs::remove_file(companion_path(&backup_path, suffix));+ }+ std::fs::rename(&self.path, &backup_path)?;+ let mut installed = Ok(());+ for suffix in COMPANION_SUFFIXES {+ let source = companion_path(compact_path, suffix);+ if !source.exists() {+ continue;+ }+ if let Err(error) = std::fs::rename(source, companion_path(&self.path, suffix)) {+ installed = Err(error);+ break;+ }+ }+ match installed {+ Ok(()) => {+ // The compacted file is installed; the original is disposable now.+ for suffix in COMPANION_SUFFIXES {+ let _ = std::fs::remove_file(companion_path(&backup_path, suffix));+ }+ Ok(())+ }+ Err(error) => {+ // Restore the migrated original so the next open still sees it.+ for suffix in COMPANION_SUFFIXES {+ let _ = std::fs::remove_file(companion_path(&self.path, suffix));+ let backup = companion_path(&backup_path, suffix);+ if backup.exists() {+ let _ = std::fs::rename(backup, companion_path(&self.path, suffix));+ }+ }+ Err(error)+ }+ }+ }+/// Opens this database, physically replacing incompatible or uncertain/// storage before retrying once.pub fn open_or_reset(&self, scope: &str) -> Result<TursoStorage, TursoStorageError> {@@ -639,6 +735,18 @@ impl TursoFileDatabase {}}+/// Main, WAL, and shared-memory files that belong to one native database path.+#[cfg(not(target_arch = "wasm32"))]+const COMPANION_SUFFIXES: [&str; 3] = ["", "-wal", "-shm"];++/// `path`, `path-wal`, or `path-shm` for a native database path.+#[cfg(not(target_arch = "wasm32"))]+fn companion_path(path: &Path, suffix: &str) -> PathBuf {+ let mut value = path.as_os_str().to_owned();+ value.push(suffix);+ PathBuf::from(value)+}+#[cfg(not(target_arch = "wasm32"))]fn remove_native_file(path: &Path) -> Result<(), TursoStorageError> {match std::fs::remove_file(path) {@@ -654,7 +762,7 @@ fn open_native_database(path: &str,scope: &str,fresh: bool,-) -> Result<TursoStorage, TursoStorageError> {+) -> Result<(TursoStorage, bool), TursoStorageError> {let database = Database::open(io, path, OpenOptions::new(Arc::new(SqliteDialect))).map_err(TursoStorageError::turso).map_err(|error| if fresh { error } else { error.initialization() })?;@@ -662,21 +770,27 @@ fn open_native_database(.connect().map_err(TursoStorageError::turso).map_err(|error| if fresh { error } else { error.initialization() })?;- if let Err(error) = initialize(&connection, scope, fresh) {- if connection.close().is_err() {- return Err(TursoStorageError::reset(- PhysicalResetReason::TransactionOutcomeUncertain,- ));+ let migrated = match initialize(&connection, scope, fresh) {+ Ok(migrated) => migrated,+ Err(error) => {+ if connection.close().is_err() {+ return Err(TursoStorageError::reset(+ PhysicalResetReason::TransactionOutcomeUncertain,+ ));+ }+ return Err(error);}- return Err(error);- }- Ok(TursoStorage {- health: AtomicU8::new(0),- database,- connection,- #[cfg(all(test, not(target_arch = "wasm32")))]- fault: Mutex::new(None),- })+ };+ Ok((+ TursoStorage {+ health: AtomicU8::new(0),+ database,+ connection,+ #[cfg(all(test, not(target_arch = "wasm32")))]+ fault: Mutex::new(None),+ },+ migrated,+ ))}/// Reopenable Turso `MemoryIO` database used by native conformance tests.@@ -712,7 +826,7 @@ impl TursoMemoryDatabase {(state.io.clone(), !state.initialized)};let io_trait: Arc<dyn IO> = io;- let storage = open_native_database(io_trait, &self.path, scope, fresh)?;+ let (storage, _) = open_native_database(io_trait, &self.path, scope, fresh)?;if fresh {self.state.lock()@@ -2918,11 +3032,47 @@ fn sql_bound(bound: RangeBound, lower: bool) -> (&'static str, i64) {}}+/// Carry a cache written by a migratable older format forward.+///+/// Records and the derived rows they feed are refetchable, so they are dropped.+/// Queued mutations, optimistic layers, and their projections are kept, which is+/// why this is not a physical reset. The namespace moves to this build inside the+/// same transaction, so an interrupted migration leaves the previous state intact+/// and is retried on the next open.+fn migrate_previous_format(+ connection: &Arc<Connection>,+ scope: &str,+ previous_namespace: &str,+) -> Result<(), TursoStorageError> {+ let migratable = MIGRATABLE_CACHE_FORMAT_VERSIONS+ .iter()+ .any(|version| cache_namespace_at(scope, *version) == previous_namespace);+ if !migratable {+ return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));+ }+ driver::write_transaction(connection, || {+ driver::execute(connection, "DELETE FROM search_documents", Vec::new())?;+ // Facts cascade from their index documents.+ driver::execute(connection, "DELETE FROM index_documents", Vec::new())?;+ driver::execute(connection, "DELETE FROM records", Vec::new())?;+ calendar::clear(connection)?;+ driver::execute(+ connection,+ "UPDATE meta SET value = ?1 WHERE key = 'namespace'",+ vec![text(&cache_namespace(scope))],+ )?;+ Ok(())+ })+ .map_err(TursoStorageError::initialization)+}++/// Validates or initializes a database. Returns whether a previous format was+/// carried forward, which native owners compact after opening.fn initialize(connection: &Arc<Connection>,scope: &str,fresh: bool,-) -> Result<(), TursoStorageError> {+) -> Result<bool, TursoStorageError> {enable_foreign_keys(connection).map_err(TursoStorageError::initialization)?;if fresh {driver::write_transaction(connection, || {@@ -2952,7 +3102,7 @@ fn initialize(.map_err(TursoStorageError::initialization)?;enable_foreign_keys(connection).map_err(TursoStorageError::initialization)?;validate_frozen_schema(connection)?;- return Ok(());+ return Ok(false);}// Ordinary reopen only validates compatibility and pending writes. A changed@@ -2972,15 +3122,30 @@ fn initialize(if metadata.len() != expected.len() {return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));}+ let mut previous_namespace = None;for (row, (expected_key, expected_value)) in metadata.iter().zip(expected) {let key = required_text(row, 0).map_err(|_| TursoStorageError::reset(PhysicalResetReason::Compatibility))?;let value = required_text(row, 1).map_err(|_| TursoStorageError::reset(PhysicalResetReason::Compatibility))?;- if key != expected_key || value != expected_value {+ if key != expected_key {return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));}+ if value != expected_value {+ // Only the namespace may differ, and only for a migratable format.+ if key != "namespace" {+ return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));+ }+ previous_namespace = Some(value.to_owned());+ }}+ let migrated = match previous_namespace {+ Some(previous_namespace) => {+ migrate_previous_format(connection, scope, &previous_namespace)?;+ true+ }+ None => false,+ };for sql in [RECORD_GET,RECORD_UPSERT,@@ -3024,7 +3189,8 @@ fn initialize(validate_optimistic_shadow_consistency(connection)?;page_retention::compact_legacy_pages(connection)?;ensure_search_projection_version(connection)?;- calendar::ensure_projection_version(connection)+ calendar::ensure_projection_version(connection)?;+ Ok(migrated)}fn save_search_projection_version(connection: &Arc<Connection>) -> Result<(), TursoStorageError> {@@ -4847,6 +5013,13 @@ fn text(value: &str) -> Value {Value::from_text(value.to_owned())}+/// SQL string literal for statements that accept no bound parameters+/// (`VACUUM INTO <path>`).+#[cfg(not(target_arch = "wasm32"))]+fn sql_literal(value: &str) -> String {+ format!("'{}'", value.replace('\'', "''"))+}+fn optional_text(value: Option<&str>) -> Value {value.map_or(Value::Null, text)}diff --git a/crates/client/cache-turso/src/storage/test/startup.rs b/crates/client/cache-turso/src/storage/test/startup.rsindex 7daa09241a..0d50af05e0 100644--- a/crates/client/cache-turso/src/storage/test/startup.rs+++ b/crates/client/cache-turso/src/storage/test/startup.rs@@ -63,3 +63,210 @@ fn explicit_integrity_check_preserves_records_and_pending_writes() {);});}++#[test]+fn previous_format_drops_records_and_keeps_pending_writes() {+ block_on(async {+ let database = TursoMemoryDatabase::new("previous-format-migration.db");+ let mut storage = database.open("scope").unwrap();+ storage+ .put_batch(vec![(key("Thing:1"), record("persisted"))])+ .await+ .unwrap();+ storage.enqueue_mutation(queued("Pending")).await.unwrap();+ let pending = storage.load_mutation_queue().await.unwrap();++ mark_previous_format(&storage, "scope");+ storage.try_close().unwrap();++ let storage = database.open("scope").unwrap();+ assert_eq!(+ storage.get_batch(&[key("Thing:1")]).await.unwrap(),+ vec![None],+ "records from the previous format are dropped"+ );+ assert_eq!(+ storage.load_mutation_queue().await.unwrap(),+ pending,+ "queued mutations and their optimistic layers survive"+ );+ assert_eq!(+ namespace(&storage),+ cache_namespace("scope"),+ "the migration rewrites the namespace"+ );+ assert_eq!(+ storage.try_close().unwrap(),+ TursoStorageCloseOutcome::Healthy+ );++ // The migrated database reopens without another migration or reset.+ let storage = database.open("scope").unwrap();+ assert_eq!(+ storage.load_mutation_queue().await.unwrap(),+ pending,+ "a second open keeps the migrated queue"+ );+ assert_eq!(namespace(&storage), cache_namespace("scope"));+ assert_eq!(+ storage.try_close().unwrap(),+ TursoStorageCloseOutcome::Healthy+ );+ });+}++/// Rewrites every record as the previous format stored it: untagged postcard+/// bytes under the previous namespace.+fn mark_previous_format(storage: &TursoStorage, scope: &str) {+ let rows = driver::query(+ &storage.connection(),+ "SELECT __typename, id, value FROM records",+ Vec::new(),+ )+ .unwrap();+ for row in rows {+ let typename = required_text(&row, 0).unwrap();+ let id = required_text(&row, 1).unwrap();+ let Value::Blob(bytes) = required_value(&row, 2).unwrap() else {+ panic!("records.value must be a blob");+ };+ let legacy = match bytes.first() {+ Some(0) => bytes[1..].to_vec(),+ Some(1) => miniz_oxide::inflate::decompress_to_vec(&bytes[1..]).unwrap(),+ other => panic!("unexpected record encoding {other:?} for {typename}:{id}"),+ };+ raw_execute(+ storage,+ "UPDATE records SET value = ?1 WHERE __typename = ?2 AND id = ?3",+ vec![Value::from_blob(legacy), text(&typename), text(&id)],+ );+ }+ raw_execute(+ storage,+ "UPDATE meta SET value = ?1 WHERE key = 'namespace'",+ vec![text(&cache_namespace_at(scope, 3))],+ );+}++fn namespace(storage: &TursoStorage) -> String {+ let rows = driver::query(+ &storage.connection(),+ "SELECT value FROM meta WHERE key = 'namespace'",+ Vec::new(),+ )+ .unwrap();+ required_text(&rows[0], 0).unwrap()+}++#[cfg(not(target_arch = "wasm32"))]+#[test]+fn migrating_a_file_reclaims_space_and_keeps_pending_writes() {+ block_on(async {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("cache.turso");+ let database = TursoFileDatabase::new(&path).unwrap();+ let mut storage = database.open("scope").unwrap();+ storage+ .put_batch(vec![(+ key("Thing:1"),+ record(&"review line\n".repeat(20_000)),+ )])+ .await+ .unwrap();+ storage.enqueue_mutation(queued("Pending")).await.unwrap();+ let pending = storage.load_mutation_queue().await.unwrap();+ mark_previous_format(&storage, "scope");+ storage.try_close().unwrap();+ let before = std::fs::metadata(&path).unwrap().len();+ assert!(before > 100_000, "fixture file was only {before} bytes");++ let storage = database.open("scope").unwrap();+ assert_eq!(+ storage.get_batch(&[key("Thing:0")]).await.unwrap(),+ vec![None],+ "records from the previous format are dropped"+ );+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ assert_eq!(namespace(&storage), cache_namespace("scope"));+ storage.try_close().unwrap();++ let after = std::fs::metadata(&path).unwrap().len();+ assert!(+ after < before / 2,+ "compaction should reclaim space: {before} -> {after}"+ );+ let storage = database.open("scope").unwrap();+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ storage.try_close().unwrap();+ });+}++#[cfg(not(target_arch = "wasm32"))]+#[test]+fn failed_compaction_keeps_the_migrated_database() {+ block_on(async {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("cache.turso");+ let database = TursoFileDatabase::new(&path).unwrap();+ let mut storage = database.open("scope").unwrap();+ storage+ .put_batch(vec![(key("Thing:1"), record("persisted"))])+ .await+ .unwrap();+ storage.enqueue_mutation(queued("Pending")).await.unwrap();+ let pending = storage.load_mutation_queue().await.unwrap();+ mark_previous_format(&storage, "scope");+ storage.try_close().unwrap();++ // A directory at the compaction target makes VACUUM INTO fail. The+ // migration is already committed, so its retained queue must survive.+ let compact_path = path.with_extension("migrated");+ std::fs::create_dir(&compact_path).unwrap();+ let storage = database.open("scope").unwrap();+ assert_eq!(+ storage.get_batch(&[key("Thing:1")]).await.unwrap(),+ vec![None]+ );+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ storage.try_close().unwrap();+ std::fs::remove_dir(&compact_path).unwrap();++ let storage = database.open("scope").unwrap();+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ assert_eq!(namespace(&storage), cache_namespace("scope"));+ storage.try_close().unwrap();+ });+}++#[cfg(not(target_arch = "wasm32"))]+#[test]+fn failed_swap_keeps_the_migrated_database() {+ block_on(async {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("cache.turso");+ let database = TursoFileDatabase::new(&path).unwrap();+ let mut storage = database.open("scope").unwrap();+ storage+ .put_batch(vec![(key("Thing:1"), record("persisted"))])+ .await+ .unwrap();+ storage.enqueue_mutation(queued("Pending")).await.unwrap();+ let pending = storage.load_mutation_queue().await.unwrap();+ mark_previous_format(&storage, "scope");+ storage.try_close().unwrap();++ // A directory at the backup path fails the move of the original, which+ // must leave the migrated database openable.+ let backup_path = path.with_extension("backup");+ std::fs::create_dir(&backup_path).unwrap();+ let storage = database.open("scope").unwrap();+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ storage.try_close().unwrap();+ std::fs::remove_dir(&backup_path).unwrap();++ let storage = database.open("scope").unwrap();+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);+ assert_eq!(namespace(&storage), cache_namespace("scope"));+ storage.try_close().unwrap();+ });+}diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.tomlindex 6f973a7113..85c1e60a12 100644--- a/crates/client/cache-wasm/Cargo.toml+++ b/crates/client/cache-wasm/Cargo.toml@@ -3,7 +3,7 @@ edition = "2024"name = "cache-wasm"publish = false# Rebuild with thread-local email participant cache semantics.-version = "0.6.48"+version = "0.6.49"[features]default = []