Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(cache): deflate large records and compact migrated databases (#8530)

Sean Aye8a9ef577e178
diff --git a/Cargo.lock b/Cargo.lock
index 8213e6adf1..81fd635b45 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3608,6 +3608,7 @@ dependencies = [
"chrono",
"lru 0.16.4",
"maybe_send",
+ "miniz_oxide",
"pollster",
"postcard",
"predicate-index",
@@ -3626,6 +3627,7 @@ dependencies = [
"cache-core",
"chrono",
"maybe_send",
+ "miniz_oxide",
"pollster",
"predicate-index",
"proptest",
@@ -3640,7 +3642,7 @@ dependencies = [
[[package]]
name = "cache-wasm"
-version = "0.6.48"
+version = "0.6.49"
dependencies = [
"async-lock",
"cache-core",
diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.md
index f63510e843..d8eb70577e 100644
--- a/apps/web/docs/graphql-normalized-cache-plan.md
+++ b/apps/web/docs/graphql-normalized-cache-plan.md
@@ -322,7 +322,15 @@ paths remain behind the same `CacheHost` interface.
| Turso over filesystem IO | Tauri native | records, search projection, mutation queue, optimistic layers, and metadata; WAL mode |
| Turso over OPFS | browser | the same frozen schema behind the OPFS IO adapter and coordinated worker lifecycle |
-Normalized records are serialized with `postcard` into `records.value`.
+Normalized records are serialized with `postcard` into `records.value`, behind a
+one-byte encoding tag: records of 256 bytes or more are stored deflated when that
+is smaller, and everything else stays raw. Other persisted payloads (queued
+mutations, optimistic layers, projections) remain plain postcard. A format bump
+that only changes the record encoding carries the previous database forward by
+dropping records, search projections, index facts, and calendar ranges — all
+refetchable — while keeping queued mutations, optimistic layers, and their
+projections. Native storage then compacts the migrated file with `VACUUM INTO`, so
+the upgrade reclaims the space instead of leaving freed pages behind.
Both hosts use the same `Storage` implementation and compound entity keys.
Wasm futures are not `Send`, so the `Storage` trait's futures are bound by
`MaybeSend` (`crates/maybe_send`): `Send` on native targets, where Tauri drives
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
index f8883b1df1..14919d469d 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
@@ -828,10 +828,10 @@ describe('coordinator runtime protocol', () => {
// The current versions moved past the unversioned name, so the lock
// names the versioned database that builds before them never held.
expect(databaseOwnerLockName('scope')).toBe(
- 'macro:turso-opfs:v1:29:graphql-cache:scope:s4.v3.t11'
+ 'macro:turso-opfs:v1:29:graphql-cache:scope:s4.v4.t11'
);
expect(databaseOwnerLockName('é')).toBe(
- 'macro:turso-opfs:v1:26:graphql-cache:é:s4.v3.t11'
+ 'macro:turso-opfs:v1:26:graphql-cache:é:s4.v4.t11'
);
});
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
index 40f10c2159..45f8aa5509 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
@@ -1379,7 +1379,7 @@ export function tabIdFromLivenessLockName(
*/
export const CACHE_STORAGE_VERSION = {
schemaCompatibilityEpoch: 4,
- formatVersion: 3,
+ formatVersion: 4,
storageSchemaVersion: 11,
} as const;
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts
index f56099eb7d..ddec88cf41 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-takeover.test.ts
@@ -26,7 +26,7 @@ describe('cache takeover messages', () => {
`graphql-cache-takeover:${cacheDatabaseIdentity('scope')}`
);
expect(cacheTakeoverChannelName('scope')).toBe(
- 'graphql-cache-takeover:graphql-cache:scope:s4.v3.t11'
+ 'graphql-cache-takeover:graphql-cache:scope:s4.v4.t11'
);
});
diff --git a/apps/web/tauri/Cargo.lock b/apps/web/tauri/Cargo.lock
index 4fbf4b6482..db5a09680c 100644
--- a/apps/web/tauri/Cargo.lock
+++ b/apps/web/tauri/Cargo.lock
@@ -859,6 +859,7 @@ dependencies = [
"chrono",
"lru",
"maybe_send",
+ "miniz_oxide",
"postcard",
"predicate-index",
"serde",
diff --git a/apps/web/tauri/src-tauri/tauri.android.conf.json b/apps/web/tauri/src-tauri/tauri.android.conf.json
index c6908f4741..7ddcad1204 100644
--- a/apps/web/tauri/src-tauri/tauri.android.conf.json
+++ b/apps/web/tauri/src-tauri/tauri.android.conf.json
@@ -2,7 +2,7 @@
"version": "2.5.0",
"bundle": {
"android": {
- "versionCode": 12
+ "versionCode": 13
}
},
"$schema": "https://schema.tauri.app/config/2",
diff --git a/crates/client/README.md b/crates/client/README.md
index 110bdadac4..1708af141d 100644
--- a/crates/client/README.md
+++ b/crates/client/README.md
@@ -61,6 +61,29 @@ scheduled automatically in the background. Failures latch the existing storage
health state without deleting records or pending mutations. Recovery/reset remains
an explicit caller decision after closing the storage.
+## Record encoding
+
+`records.value` is postcard behind a one-byte tag: payloads of 256 bytes or more are
+deflated (`miniz_oxide`, level 3) when that is smaller, and everything else stays
+raw. Large text records — foreign-entity source metadata and email bodies —
+dominate the database and shrink about 3×; a real 155 MB record payload became
+48 MB. Level 3 keeps that size win with markedly less CPU in the slowest batches
+than level 6, which matters because deflate runs while the engine lock is held.
+Decompression is bounded per record. Queued mutations, optimistic layers, and
+projections keep their unframed postcard encoding, so a format upgrade never
+re-encodes them.
+
+Only the immediately prior cache format version is migratable
+(`MIGRATABLE_CACHE_FORMAT_VERSIONS`). Opening one drops records, search
+projections, index facts, and calendar ranges — all refetchable — and keeps
+queued mutations, optimistic layers, and their projections, then rewrites the
+namespace in the same transaction. An interrupted migration therefore retries on
+the next open. Native owners then `VACUUM INTO` a fresh file and swap it in, so an
+upgrade also releases the space the dropped records occupied instead of leaving
+free pages in the old file. The browser's physical database name embeds the format
+version, so a browser upgrade starts a new file instead of migrating; the previous
+file is left for any tab still running the older bundle.
+
## Network refreshes
Normalized refreshes persist only records whose merged contents changed. The hot
diff --git a/crates/client/cache-core/Cargo.toml b/crates/client/cache-core/Cargo.toml
index fa8ebc1214..d03e6f0f02 100644
--- a/crates/client/cache-core/Cargo.toml
+++ b/crates/client/cache-core/Cargo.toml
@@ -10,6 +10,7 @@ apollo-parser = { workspace = true }
chrono = { workspace = true }
lru = { workspace = true }
maybe_send = { path = "../../maybe_send" }
+miniz_oxide = { workspace = true }
postcard = { workspace = true }
predicate-index = { path = "../../predicate_index" }
serde = { workspace = true }
diff --git a/crates/client/cache-core/src/codec.rs b/crates/client/cache-core/src/codec.rs
index 8fb63fbd05..e0b68e6254 100644
--- a/crates/client/cache-core/src/codec.rs
+++ b/crates/client/cache-core/src/codec.rs
@@ -1,5 +1,7 @@
//! Binary record codec shared by all persistent Turso backends (browser and
-//! native hosts). Records are stored as postcard bytes; the logical namespace
+//! native hosts). Records are stored as tagged postcard bytes — raw, or
+//! deflated when that is smaller — while every other persisted payload stays
+//! plain postcard. The logical namespace
//! embeds [`CACHE_FORMAT_VERSION`] and [`CACHE_SCHEMA_COMPATIBILITY_EPOCH`],
//! and so does the browser's physical database name after
//! [`UNVERSIONED_STORAGE_VERSIONS`]. A healthy, graceful
@@ -10,12 +12,32 @@
use crate::normalize::RecordUpdates;
use crate::queue::{PersistedOptimisticLayer, StoredMutation};
use crate::value::Record;
+use miniz_oxide::{deflate, inflate};
use serde::{Serialize, de::DeserializeOwned};
use thiserror::Error;
/// Bump when the stored representation of [`Record`]/[`CacheValue`]
/// (or anything else persisted) changes incompatibly.
-pub const CACHE_FORMAT_VERSION: u32 = 3;
+pub const CACHE_FORMAT_VERSION: u32 = 4;
+
+/// Records at or above this size may be stored deflated. Smaller payloads stay
+/// raw so deflate framing never makes them larger.
+const RECORD_COMPRESSION_MIN_BYTES: usize = 256;
+
+/// Deflate level for record payloads.
+///
+/// On a 155 MB sample of real records, level 6 buys 2% more compression than
+/// level 3 (3.29x vs 3.22x) but costs 36% more CPU in the slowest batches, and
+/// deflate runs while the cache engine lock is held. Level 3 keeps the size win
+/// without stalling other cache operations behind a large record.
+const RECORD_DEFLATE_LEVEL: u8 = 3;
+
+/// Upper bound for one inflated record. Real records are a few hundred KiB at
+/// most; the bound turns a corrupt length prefix into a codec error.
+const RECORD_MAX_INFLATED_BYTES: usize = 64 * 1024 * 1024;
+
+const RECORD_ENCODING_RAW: u8 = 0;
+const RECORD_ENCODING_DEFLATE: u8 = 1;
/// Bump when a GraphQL schema change makes existing normalized records unsafe.
///
@@ -31,6 +53,10 @@ pub const CACHE_SCHEMA_COMPATIBILITY_EPOCH: u32 = 4;
pub enum CodecError {
#[error("corrupt cache payload: {0}")]
Corrupt(#[from] postcard::Error),
+ #[error("corrupt compressed cache payload: {0}")]
+ Inflate(String),
+ #[error("unknown record encoding: {0}")]
+ UnknownEncoding(u8),
}
fn encode<T: Serialize>(value: &T) -> Vec<u8> {
@@ -42,12 +68,38 @@ fn decode<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, CodecError> {
Ok(postcard::from_bytes(bytes)?)
}
+/// Encodes a normalized record, deflating it when that makes the payload smaller.
pub fn encode_record(record: &Record) -> Vec<u8> {
- encode(record)
+ let raw = encode(record);
+ let compressed = (raw.len() >= RECORD_COMPRESSION_MIN_BYTES)
+ .then(|| deflate::compress_to_vec(&raw, RECORD_DEFLATE_LEVEL));
+ match compressed {
+ Some(compressed) if compressed.len() < raw.len() => {
+ tagged(RECORD_ENCODING_DEFLATE, compressed)
+ }
+ _ => tagged(RECORD_ENCODING_RAW, raw),
+ }
}
+/// Decodes a normalized record written by [`encode_record`].
pub fn decode_record(bytes: &[u8]) -> Result<Record, CodecError> {
- decode(bytes)
+ let (encoding, payload) = bytes.split_first().ok_or(CodecError::UnknownEncoding(0))?;
+ match *encoding {
+ RECORD_ENCODING_RAW => decode(payload),
+ RECORD_ENCODING_DEFLATE => {
+ let raw = inflate::decompress_to_vec_with_limit(payload, RECORD_MAX_INFLATED_BYTES)
+ .map_err(|error| CodecError::Inflate(format!("{error:?}")))?;
+ decode(&raw)
+ }
+ other => Err(CodecError::UnknownEncoding(other)),
+ }
+}
+
+fn tagged(encoding: u8, payload: Vec<u8>) -> Vec<u8> {
+ let mut bytes = Vec::with_capacity(payload.len() + 1);
+ bytes.push(encoding);
+ bytes.extend_from_slice(&payload);
+ bytes
}
/// Encodes one queued mutation's request and retry metadata.
@@ -85,7 +137,19 @@ pub fn decode_record_updates(bytes: &[u8]) -> Result<RecordUpdates, CodecError>
/// `scope` is an anonymous, client-generated token supplied by the host. It is
/// neither user nor workspace identity.
pub fn cache_namespace(scope: &str) -> String {
- format!("graphql-cache:{scope}:s{CACHE_SCHEMA_COMPATIBILITY_EPOCH}:v{CACHE_FORMAT_VERSION}")
+ cache_namespace_at(scope, CACHE_FORMAT_VERSION)
+}
+
+/// Cache format versions whose physical databases this build carries forward.
+///
+/// Opening one of these drops the derived records (they are refetched) while
+/// keeping queued mutations and optimistic layers, then rewrites the namespace.
+/// A format that is neither current nor listed here is reset instead.
+pub const MIGRATABLE_CACHE_FORMAT_VERSIONS: &[u32] = &[3];
+
+/// Namespace a cache scope used at one cache format version.
+pub fn cache_namespace_at(scope: &str, format_version: u32) -> String {
+ format!("graphql-cache:{scope}:s{CACHE_SCHEMA_COMPATIBILITY_EPOCH}:v{format_version}")
}
/// Storage versions `(schema compatibility epoch, cache format, storage
@@ -164,14 +228,59 @@ mod tests {
#[test]
fn rejects_garbage() {
assert!(decode_record(&[0xff, 0x00, 0x13, 0x37]).is_err());
+ assert!(decode_record(&[]).is_err());
+ assert!(decode_record(&[RECORD_ENCODING_DEFLATE, 0x13, 0x37]).is_err());
+ }
+
+ #[test]
+ fn small_records_stay_raw() {
+ let record = record_with_body("short body");
+ let bytes = encode_record(&record);
+ assert_eq!(bytes.first(), Some(&RECORD_ENCODING_RAW));
+ assert_eq!(decode_record(&bytes).unwrap(), record);
+ }
+
+ #[test]
+ fn large_text_records_deflate_and_round_trip() {
+ let body = "review comment line\n".repeat(2_000);
+ let record = record_with_body(&body);
+ let bytes = encode_record(&record);
+ assert_eq!(bytes.first(), Some(&RECORD_ENCODING_DEFLATE));
+ assert!(
+ bytes.len() < body.len() / 4,
+ "compressed {} bytes",
+ bytes.len()
+ );
+ assert_eq!(decode_record(&bytes).unwrap(), record);
+ }
+
+ fn record_with_body(body: &str) -> Record {
+ let mut record = Record::default();
+ record.fields.insert(
+ "__typename".into(),
+ CacheValue::String("GraphqlSoupForeignEntity".into()),
+ );
+ record
+ .fields
+ .insert("sourceMetadata".into(), CacheValue::String(body.into()));
+ record
}
#[test]
fn namespace_uses_schema_compatibility_epoch_not_schema_hash() {
assert_eq!(
cache_namespace("client-token-1"),
+ "graphql-cache:client-token-1:s4:v4"
+ );
+ }
+
+ #[test]
+ fn migratable_versions_name_their_own_namespace() {
+ assert_eq!(
+ cache_namespace_at("client-token-1", 3),
"graphql-cache:client-token-1:s4:v3"
);
+ assert!(!MIGRATABLE_CACHE_FORMAT_VERSIONS.contains(&CACHE_FORMAT_VERSION));
}
#[test]
@@ -180,11 +289,11 @@ mod tests {
// later epoch embeds its versions, even at that storage schema.
assert_eq!(
cache_database_name("client-token-1", 11),
- "graphql-cache:client-token-1:s4.v3.t11"
+ "graphql-cache:client-token-1:s4.v4.t11"
);
assert_eq!(
cache_database_name("client-token-1", 12),
- "graphql-cache:client-token-1:s4.v3.t12"
+ "graphql-cache:client-token-1:s4.v4.t12"
);
assert_eq!(
legacy_cache_database_name("client-token-1"),
diff --git a/crates/client/cache-turso/AGENTS.md b/crates/client/cache-turso/AGENTS.md
index d301af51aa..13492ff997 100644
--- a/crates/client/cache-turso/AGENTS.md
+++ b/crates/client/cache-turso/AGENTS.md
@@ -46,6 +46,11 @@ GraphQL cache — native for tests/Tauri, OPFS-backed on wasm32.
- The compound-key schema and payload encoding are frozen; an incompatible or
uncertain on-disk state is classified for physical reset, never silently
migrated, copied, or redirected to a fallback backend.
+- The one exception is a cache format version listed in
+ `MIGRATABLE_CACHE_FORMAT_VERSIONS`: `migrate_previous_format` drops the
+ refetchable records, search projections, index facts, and calendar ranges in
+ one transaction, keeps queued mutations, optimistic layers, and their
+ projections, and rewrites the namespace. Everything else still resets.
## Related
diff --git a/crates/client/cache-turso/Cargo.toml b/crates/client/cache-turso/Cargo.toml
index f202dd06ff..baaca7f58c 100644
--- a/crates/client/cache-turso/Cargo.toml
+++ b/crates/client/cache-turso/Cargo.toml
@@ -24,6 +24,7 @@ turso_core = { workspace = true, features = ["fs", "pure-rust-crypto", "uuid"] }
[dev-dependencies]
chrono = { workspace = true }
+miniz_oxide = { workspace = true }
pollster = { workspace = true }
proptest = { workspace = true }
serde_json = { workspace = true }
diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rs
index 434db449af..57a81b8ad0 100644
--- a/crates/client/cache-turso/src/storage.rs
+++ b/crates/client/cache-turso/src/storage.rs
@@ -2,7 +2,8 @@ use crate::driver;
use crate::key::RecordKey;
use crate::{PhysicalResetReason, TursoStorageError};
use cache_core::codec::{
- cache_namespace, decode_record, decode_record_updates, encode_record, encode_record_updates,
+ MIGRATABLE_CACHE_FORMAT_VERSIONS, cache_namespace, cache_namespace_at, decode_record,
+ decode_record_updates, encode_record, encode_record_updates,
};
use cache_core::predicate::reconciliation::{
PredicateBaselineEntry, PredicateReconciliation, predicate_membership,
@@ -226,7 +227,7 @@ impl TursoStorage {
let result = initialize(&connection, scope, fresh);
drop(connection);
match result {
- Ok(()) => Ok((
+ Ok(_) => Ok((
Self {
health: AtomicU8::new(0),
session,
@@ -612,12 +613,107 @@ impl TursoFileDatabase {
/// Opens this database, validating compatibility and queued writes for `scope`.
/// Full-file integrity scans are explicit via [`TursoStorage::check_integrity`].
+ ///
+ /// A database carried forward from a migratable format is compacted first, so
+ /// the upgrade also releases the space its dropped records occupied.
pub fn open(&self, scope: &str) -> Result<TursoStorage, TursoStorageError> {
let fresh = !self.path.exists();
+ let (storage, migrated) = self.open_with_outcome(scope, fresh)?;
+ if !migrated {
+ return Ok(storage);
+ }
+ self.compact_after_migration(storage, scope)
+ }
+
+ fn open_with_outcome(
+ &self,
+ scope: &str,
+ fresh: bool,
+ ) -> Result<(TursoStorage, bool), TursoStorageError> {
let io: Arc<dyn IO> = Arc::new(PlatformIO::new().map_err(TursoStorageError::turso)?);
open_native_database(io, &self.turso_path, scope, fresh)
}
+ /// Copy the migrated database's small live set into a fresh file and swap it
+ /// in. Freed pages are reused but never returned, so without this an upgrade
+ /// would keep the previous file size indefinitely.
+ ///
+ /// Compaction is an optimization on top of an already-committed migration:
+ /// every failure falls back to the migrated database, because reporting a
+ /// reset-required error here would make `open_or_reset` delete the queued
+ /// mutations the migration just preserved.
+ fn compact_after_migration(
+ &self,
+ storage: TursoStorage,
+ scope: &str,
+ ) -> Result<TursoStorage, TursoStorageError> {
+ let compact_path = self.path.with_extension("migrated");
+ let _ = remove_native_file(&compact_path);
+ let vacuumed = compact_path
+ .to_str()
+ .ok_or(TursoStorageError::InvalidInput)
+ .and_then(|compact_path| {
+ driver::execute(
+ &storage.connection(),
+ &format!("VACUUM INTO {}", sql_literal(compact_path)),
+ Vec::new(),
+ )
+ })
+ .is_ok();
+ if !vacuumed {
+ let _ = remove_native_file(&compact_path);
+ return Ok(storage);
+ }
+ // The compacted file is complete; a close failure cannot invalidate it.
+ let _ = storage.try_close();
+ if self.install_compacted(&compact_path).is_err() {
+ let _ = remove_native_file(&compact_path);
+ return Ok(self.open_with_outcome(scope, false)?.0);
+ }
+ Ok(self.open_with_outcome(scope, false)?.0)
+ }
+
+ /// Move the compacted database into place while keeping the original
+ /// recoverable, so an interrupted swap never leaves no database behind.
+ fn install_compacted(&self, compact_path: &Path) -> std::io::Result<()> {
+ let backup_path = self.path.with_extension("backup");
+ for suffix in COMPANION_SUFFIXES {
+ let _ = std::fs::remove_file(companion_path(&backup_path, suffix));
+ }
+ std::fs::rename(&self.path, &backup_path)?;
+ let mut installed = Ok(());
+ for suffix in COMPANION_SUFFIXES {
+ let source = companion_path(compact_path, suffix);
+ if !source.exists() {
+ continue;
+ }
+ if let Err(error) = std::fs::rename(source, companion_path(&self.path, suffix)) {
+ installed = Err(error);
+ break;
+ }
+ }
+ match installed {
+ Ok(()) => {
+ // The compacted file is installed; the original is disposable now.
+ for suffix in COMPANION_SUFFIXES {
+ let _ = std::fs::remove_file(companion_path(&backup_path, suffix));
+ }
+ Ok(())
+ }
+ Err(error) => {
+ // Restore the migrated original so the next open still sees it.
+ for suffix in COMPANION_SUFFIXES {
+ let _ = std::fs::remove_file(companion_path(&self.path, suffix));
+ let backup = companion_path(&backup_path, suffix);
+ if backup.exists() {
+ let _ = std::fs::rename(backup, companion_path(&self.path, suffix));
+ }
+ }
+ Err(error)
+ }
+ }
+ }
+
/// Opens this database, physically replacing incompatible or uncertain
/// storage before retrying once.
pub fn open_or_reset(&self, scope: &str) -> Result<TursoStorage, TursoStorageError> {
@@ -639,6 +735,18 @@ impl TursoFileDatabase {
}
}
+/// Main, WAL, and shared-memory files that belong to one native database path.
+#[cfg(not(target_arch = "wasm32"))]
+const COMPANION_SUFFIXES: [&str; 3] = ["", "-wal", "-shm"];
+
+/// `path`, `path-wal`, or `path-shm` for a native database path.
+#[cfg(not(target_arch = "wasm32"))]
+fn companion_path(path: &Path, suffix: &str) -> PathBuf {
+ let mut value = path.as_os_str().to_owned();
+ value.push(suffix);
+ PathBuf::from(value)
+}
+
#[cfg(not(target_arch = "wasm32"))]
fn remove_native_file(path: &Path) -> Result<(), TursoStorageError> {
match std::fs::remove_file(path) {
@@ -654,7 +762,7 @@ fn open_native_database(
path: &str,
scope: &str,
fresh: bool,
-) -> Result<TursoStorage, TursoStorageError> {
+) -> Result<(TursoStorage, bool), TursoStorageError> {
let database = Database::open(io, path, OpenOptions::new(Arc::new(SqliteDialect)))
.map_err(TursoStorageError::turso)
.map_err(|error| if fresh { error } else { error.initialization() })?;
@@ -662,21 +770,27 @@ fn open_native_database(
.connect()
.map_err(TursoStorageError::turso)
.map_err(|error| if fresh { error } else { error.initialization() })?;
- if let Err(error) = initialize(&connection, scope, fresh) {
- if connection.close().is_err() {
- return Err(TursoStorageError::reset(
- PhysicalResetReason::TransactionOutcomeUncertain,
- ));
+ let migrated = match initialize(&connection, scope, fresh) {
+ Ok(migrated) => migrated,
+ Err(error) => {
+ if connection.close().is_err() {
+ return Err(TursoStorageError::reset(
+ PhysicalResetReason::TransactionOutcomeUncertain,
+ ));
+ }
+ return Err(error);
}
- return Err(error);
- }
- Ok(TursoStorage {
- health: AtomicU8::new(0),
- database,
- connection,
- #[cfg(all(test, not(target_arch = "wasm32")))]
- fault: Mutex::new(None),
- })
+ };
+ Ok((
+ TursoStorage {
+ health: AtomicU8::new(0),
+ database,
+ connection,
+ #[cfg(all(test, not(target_arch = "wasm32")))]
+ fault: Mutex::new(None),
+ },
+ migrated,
+ ))
}
/// Reopenable Turso `MemoryIO` database used by native conformance tests.
@@ -712,7 +826,7 @@ impl TursoMemoryDatabase {
(state.io.clone(), !state.initialized)
};
let io_trait: Arc<dyn IO> = io;
- let storage = open_native_database(io_trait, &self.path, scope, fresh)?;
+ let (storage, _) = open_native_database(io_trait, &self.path, scope, fresh)?;
if fresh {
self.state
.lock()
@@ -2918,11 +3032,47 @@ fn sql_bound(bound: RangeBound, lower: bool) -> (&'static str, i64) {
}
}
+/// Carry a cache written by a migratable older format forward.
+///
+/// Records and the derived rows they feed are refetchable, so they are dropped.
+/// Queued mutations, optimistic layers, and their projections are kept, which is
+/// why this is not a physical reset. The namespace moves to this build inside the
+/// same transaction, so an interrupted migration leaves the previous state intact
+/// and is retried on the next open.
+fn migrate_previous_format(
+ connection: &Arc<Connection>,
+ scope: &str,
+ previous_namespace: &str,
+) -> Result<(), TursoStorageError> {
+ let migratable = MIGRATABLE_CACHE_FORMAT_VERSIONS
+ .iter()
+ .any(|version| cache_namespace_at(scope, *version) == previous_namespace);
+ if !migratable {
+ return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));
+ }
+ driver::write_transaction(connection, || {
+ driver::execute(connection, "DELETE FROM search_documents", Vec::new())?;
+ // Facts cascade from their index documents.
+ driver::execute(connection, "DELETE FROM index_documents", Vec::new())?;
+ driver::execute(connection, "DELETE FROM records", Vec::new())?;
+ calendar::clear(connection)?;
+ driver::execute(
+ connection,
+ "UPDATE meta SET value = ?1 WHERE key = 'namespace'",
+ vec![text(&cache_namespace(scope))],
+ )?;
+ Ok(())
+ })
+ .map_err(TursoStorageError::initialization)
+}
+
+/// Validates or initializes a database. Returns whether a previous format was
+/// carried forward, which native owners compact after opening.
fn initialize(
connection: &Arc<Connection>,
scope: &str,
fresh: bool,
-) -> Result<(), TursoStorageError> {
+) -> Result<bool, TursoStorageError> {
enable_foreign_keys(connection).map_err(TursoStorageError::initialization)?;
if fresh {
driver::write_transaction(connection, || {
@@ -2952,7 +3102,7 @@ fn initialize(
.map_err(TursoStorageError::initialization)?;
enable_foreign_keys(connection).map_err(TursoStorageError::initialization)?;
validate_frozen_schema(connection)?;
- return Ok(());
+ return Ok(false);
}
// Ordinary reopen only validates compatibility and pending writes. A changed
@@ -2972,15 +3122,30 @@ fn initialize(
if metadata.len() != expected.len() {
return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));
}
+ let mut previous_namespace = None;
for (row, (expected_key, expected_value)) in metadata.iter().zip(expected) {
let key = required_text(row, 0)
.map_err(|_| TursoStorageError::reset(PhysicalResetReason::Compatibility))?;
let value = required_text(row, 1)
.map_err(|_| TursoStorageError::reset(PhysicalResetReason::Compatibility))?;
- if key != expected_key || value != expected_value {
+ if key != expected_key {
return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));
}
+ if value != expected_value {
+ // Only the namespace may differ, and only for a migratable format.
+ if key != "namespace" {
+ return Err(TursoStorageError::reset(PhysicalResetReason::Compatibility));
+ }
+ previous_namespace = Some(value.to_owned());
+ }
}
+ let migrated = match previous_namespace {
+ Some(previous_namespace) => {
+ migrate_previous_format(connection, scope, &previous_namespace)?;
+ true
+ }
+ None => false,
+ };
for sql in [
RECORD_GET,
RECORD_UPSERT,
@@ -3024,7 +3189,8 @@ fn initialize(
validate_optimistic_shadow_consistency(connection)?;
page_retention::compact_legacy_pages(connection)?;
ensure_search_projection_version(connection)?;
- calendar::ensure_projection_version(connection)
+ calendar::ensure_projection_version(connection)?;
+ Ok(migrated)
}
fn save_search_projection_version(connection: &Arc<Connection>) -> Result<(), TursoStorageError> {
@@ -4847,6 +5013,13 @@ fn text(value: &str) -> Value {
Value::from_text(value.to_owned())
}
+/// SQL string literal for statements that accept no bound parameters
+/// (`VACUUM INTO <path>`).
+#[cfg(not(target_arch = "wasm32"))]
+fn sql_literal(value: &str) -> String {
+ format!("'{}'", value.replace('\'', "''"))
+}
+
fn optional_text(value: Option<&str>) -> Value {
value.map_or(Value::Null, text)
}
diff --git a/crates/client/cache-turso/src/storage/test/startup.rs b/crates/client/cache-turso/src/storage/test/startup.rs
index 7daa09241a..0d50af05e0 100644
--- a/crates/client/cache-turso/src/storage/test/startup.rs
+++ b/crates/client/cache-turso/src/storage/test/startup.rs
@@ -63,3 +63,210 @@ fn explicit_integrity_check_preserves_records_and_pending_writes() {
);
});
}
+
+#[test]
+fn previous_format_drops_records_and_keeps_pending_writes() {
+ block_on(async {
+ let database = TursoMemoryDatabase::new("previous-format-migration.db");
+ let mut storage = database.open("scope").unwrap();
+ storage
+ .put_batch(vec![(key("Thing:1"), record("persisted"))])
+ .await
+ .unwrap();
+ storage.enqueue_mutation(queued("Pending")).await.unwrap();
+ let pending = storage.load_mutation_queue().await.unwrap();
+
+ mark_previous_format(&storage, "scope");
+ storage.try_close().unwrap();
+
+ let storage = database.open("scope").unwrap();
+ assert_eq!(
+ storage.get_batch(&[key("Thing:1")]).await.unwrap(),
+ vec![None],
+ "records from the previous format are dropped"
+ );
+ assert_eq!(
+ storage.load_mutation_queue().await.unwrap(),
+ pending,
+ "queued mutations and their optimistic layers survive"
+ );
+ assert_eq!(
+ namespace(&storage),
+ cache_namespace("scope"),
+ "the migration rewrites the namespace"
+ );
+ assert_eq!(
+ storage.try_close().unwrap(),
+ TursoStorageCloseOutcome::Healthy
+ );
+
+ // The migrated database reopens without another migration or reset.
+ let storage = database.open("scope").unwrap();
+ assert_eq!(
+ storage.load_mutation_queue().await.unwrap(),
+ pending,
+ "a second open keeps the migrated queue"
+ );
+ assert_eq!(namespace(&storage), cache_namespace("scope"));
+ assert_eq!(
+ storage.try_close().unwrap(),
+ TursoStorageCloseOutcome::Healthy
+ );
+ });
+}
+
+/// Rewrites every record as the previous format stored it: untagged postcard
+/// bytes under the previous namespace.
+fn mark_previous_format(storage: &TursoStorage, scope: &str) {
+ let rows = driver::query(
+ &storage.connection(),
+ "SELECT __typename, id, value FROM records",
+ Vec::new(),
+ )
+ .unwrap();
+ for row in rows {
+ let typename = required_text(&row, 0).unwrap();
+ let id = required_text(&row, 1).unwrap();
+ let Value::Blob(bytes) = required_value(&row, 2).unwrap() else {
+ panic!("records.value must be a blob");
+ };
+ let legacy = match bytes.first() {
+ Some(0) => bytes[1..].to_vec(),
+ Some(1) => miniz_oxide::inflate::decompress_to_vec(&bytes[1..]).unwrap(),
+ other => panic!("unexpected record encoding {other:?} for {typename}:{id}"),
+ };
+ raw_execute(
+ storage,
+ "UPDATE records SET value = ?1 WHERE __typename = ?2 AND id = ?3",
+ vec![Value::from_blob(legacy), text(&typename), text(&id)],
+ );
+ }
+ raw_execute(
+ storage,
+ "UPDATE meta SET value = ?1 WHERE key = 'namespace'",
+ vec![text(&cache_namespace_at(scope, 3))],
+ );
+}
+
+fn namespace(storage: &TursoStorage) -> String {
+ let rows = driver::query(
+ &storage.connection(),
+ "SELECT value FROM meta WHERE key = 'namespace'",
+ Vec::new(),
+ )
+ .unwrap();
+ required_text(&rows[0], 0).unwrap()
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn migrating_a_file_reclaims_space_and_keeps_pending_writes() {
+ block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("cache.turso");
+ let database = TursoFileDatabase::new(&path).unwrap();
+ let mut storage = database.open("scope").unwrap();
+ storage
+ .put_batch(vec![(
+ key("Thing:1"),
+ record(&"review line\n".repeat(20_000)),
+ )])
+ .await
+ .unwrap();
+ storage.enqueue_mutation(queued("Pending")).await.unwrap();
+ let pending = storage.load_mutation_queue().await.unwrap();
+ mark_previous_format(&storage, "scope");
+ storage.try_close().unwrap();
+ let before = std::fs::metadata(&path).unwrap().len();
+ assert!(before > 100_000, "fixture file was only {before} bytes");
+
+ let storage = database.open("scope").unwrap();
+ assert_eq!(
+ storage.get_batch(&[key("Thing:0")]).await.unwrap(),
+ vec![None],
+ "records from the previous format are dropped"
+ );
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ assert_eq!(namespace(&storage), cache_namespace("scope"));
+ storage.try_close().unwrap();
+
+ let after = std::fs::metadata(&path).unwrap().len();
+ assert!(
+ after < before / 2,
+ "compaction should reclaim space: {before} -> {after}"
+ );
+ let storage = database.open("scope").unwrap();
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ storage.try_close().unwrap();
+ });
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn failed_compaction_keeps_the_migrated_database() {
+ block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("cache.turso");
+ let database = TursoFileDatabase::new(&path).unwrap();
+ let mut storage = database.open("scope").unwrap();
+ storage
+ .put_batch(vec![(key("Thing:1"), record("persisted"))])
+ .await
+ .unwrap();
+ storage.enqueue_mutation(queued("Pending")).await.unwrap();
+ let pending = storage.load_mutation_queue().await.unwrap();
+ mark_previous_format(&storage, "scope");
+ storage.try_close().unwrap();
+
+ // A directory at the compaction target makes VACUUM INTO fail. The
+ // migration is already committed, so its retained queue must survive.
+ let compact_path = path.with_extension("migrated");
+ std::fs::create_dir(&compact_path).unwrap();
+ let storage = database.open("scope").unwrap();
+ assert_eq!(
+ storage.get_batch(&[key("Thing:1")]).await.unwrap(),
+ vec![None]
+ );
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ storage.try_close().unwrap();
+ std::fs::remove_dir(&compact_path).unwrap();
+
+ let storage = database.open("scope").unwrap();
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ assert_eq!(namespace(&storage), cache_namespace("scope"));
+ storage.try_close().unwrap();
+ });
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+#[test]
+fn failed_swap_keeps_the_migrated_database() {
+ block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("cache.turso");
+ let database = TursoFileDatabase::new(&path).unwrap();
+ let mut storage = database.open("scope").unwrap();
+ storage
+ .put_batch(vec![(key("Thing:1"), record("persisted"))])
+ .await
+ .unwrap();
+ storage.enqueue_mutation(queued("Pending")).await.unwrap();
+ let pending = storage.load_mutation_queue().await.unwrap();
+ mark_previous_format(&storage, "scope");
+ storage.try_close().unwrap();
+
+ // A directory at the backup path fails the move of the original, which
+ // must leave the migrated database openable.
+ let backup_path = path.with_extension("backup");
+ std::fs::create_dir(&backup_path).unwrap();
+ let storage = database.open("scope").unwrap();
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ storage.try_close().unwrap();
+ std::fs::remove_dir(&backup_path).unwrap();
+
+ let storage = database.open("scope").unwrap();
+ assert_eq!(storage.load_mutation_queue().await.unwrap(), pending);
+ assert_eq!(namespace(&storage), cache_namespace("scope"));
+ storage.try_close().unwrap();
+ });
+}
diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.toml
index 6f973a7113..85c1e60a12 100644
--- a/crates/client/cache-wasm/Cargo.toml
+++ b/crates/client/cache-wasm/Cargo.toml
@@ -3,7 +3,7 @@ edition = "2024"
name = "cache-wasm"
publish = false
# Rebuild with thread-local email participant cache semantics.
-version = "0.6.48"
+version = "0.6.49"
[features]
default = []