Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

refactor(profile): simplify imports with a read-only LevelDB parser

ldm04e3f174ab13e
diff --git a/Cargo.lock b/Cargo.lock
index 2bbb32e20..683d0081e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2498,8 +2498,10 @@ dependencies = [
"moli-cookie-cache",
"moli-cookie-jar",
"moli-crypto",
+ "moli-leveldb-parser",
"rusqlite",
"rusty-leveldb",
+ "serde",
"serde_json",
"snap",
"tempfile",
@@ -2771,6 +2773,18 @@ dependencies = [
"wuff",
]
+[[package]]
+name = "moli-leveldb-parser"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "crc32c",
+ "rusty-leveldb",
+ "snap",
+ "tempfile",
+ "zerocopy",
+]
+
[[package]]
name = "moli-local-executor"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 9aa15d2e6..8934278c7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,6 +27,7 @@ members = [
"moli-indexeddb",
"moli-local-executor",
"moli-layout",
+ "moli-leveldb-parser",
"moli-message-port",
"moli-multipart",
"moli-xpath",
diff --git a/moli-cookie-import/Cargo.toml b/moli-cookie-import/Cargo.toml
index 4c2f52933..e50613725 100644
--- a/moli-cookie-import/Cargo.toml
+++ b/moli-cookie-import/Cargo.toml
@@ -11,13 +11,18 @@ moli-browser-profile = { path = "../moli-browser-profile" }
moli-cookie-cache = { path = "../moli-cookie-cache" }
moli-cookie-jar = { path = "../moli-cookie-jar" }
moli-crypto = { path = "../moli-crypto" }
+moli-leveldb-parser = { path = "../moli-leveldb-parser" }
# Keep imports self-contained; .cargo/config.toml trims unused SQLite C extensions.
rusqlite = { version = "0.37.0", features = ["bundled"] }
-rusty-leveldb = "4.0.1"
+serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
snap = "1.1.1"
tempfile = "3.23.0"
time = "0.3.44"
+[dev-dependencies]
+# Generate real LevelDB fixtures and compare the read-only importer with the engine.
+rusty-leveldb = "4.0.1"
+
[lints]
workspace = true
diff --git a/moli-cookie-import/src/chrome.rs b/moli-cookie-import/src/chrome.rs
index a91a2bd16..98c88d2dd 100644
--- a/moli-cookie-import/src/chrome.rs
+++ b/moli-cookie-import/src/chrome.rs
@@ -1,4 +1,4 @@
-use std::{collections::BTreeMap, fs, path::Path};
+use std::{fs, path::Path};
#[cfg(target_os = "macos")]
use std::process::Command;
@@ -13,24 +13,13 @@ use moli_cookie_jar::{
use moli_crypto::derive_pbkdf2_hmac_sha1;
use moli_crypto::{aes_128_cbc_pkcs7_decrypt, aes_256_gcm_decrypt, sha256_digest};
use rusqlite::Connection;
-use rusty_leveldb::{DB, LdbIterator, Options};
use tempfile::TempDir;
use time::OffsetDateTime;
-use crate::ChromeCryptoKey;
+use crate::{ChromeCryptoKey, sqlite::SqliteSnapshot, storage::LocalStorage};
const CHROME_EPOCH_OFFSET_MICROS: i64 = 11_644_473_600_000_000;
-pub(crate) fn validate_source(path: &Path) -> Result<()> {
- if !path.is_dir() {
- bail!(
- "Chrome profile directory `{}` does not exist",
- path.display()
- );
- }
- Ok(())
-}
-
pub(crate) fn import_cookies(
profile: &Path,
key_source: &ChromeCryptoKey,
@@ -44,15 +33,8 @@ pub(crate) fn import_cookies(
profile.display()
)
})?;
- let temp = tempfile::NamedTempFile::new().context("failed to create Cookies snapshot")?;
- fs::copy(&source, temp.path()).with_context(|| {
- format!(
- "failed to snapshot Chrome Cookies database `{}`",
- source.display()
- )
- })?;
- let connection =
- Connection::open(temp.path()).context("failed to open Chrome Cookies snapshot")?;
+ let snapshot = SqliteSnapshot::open(&source, "Chrome Cookies")?;
+ let connection = &snapshot.connection;
let schema_version = connection
.query_row("SELECT value FROM meta WHERE key = 'version'", [], |row| {
row.get::<_, String>(0)
@@ -60,18 +42,7 @@ pub(crate) fn import_cookies(
.ok()
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0);
- let has_partition_key = cookie_column_exists(&connection, "top_frame_site_key")?;
- let has_cross_site_ancestor = cookie_column_exists(&connection, "has_cross_site_ancestor")?;
- let partition_key_column = if has_partition_key {
- "top_frame_site_key"
- } else {
- "NULL"
- };
- let cross_site_column = if has_cross_site_ancestor {
- "has_cross_site_ancestor"
- } else {
- "0"
- };
+ let (partition_key_column, cross_site_column) = cookie_partition_columns(connection)?;
let sql = format!(
"SELECT host_key,name,value,encrypted_value,path,expires_utc,is_secure,is_httponly,samesite,priority,source_scheme,source_port,{partition_key_column},{cross_site_column} FROM cookies"
);
@@ -179,15 +150,19 @@ impl ChromeCookieRow {
}
}
-fn cookie_column_exists(connection: &Connection, name: &str) -> Result<bool> {
+fn cookie_partition_columns(connection: &Connection) -> Result<(&'static str, &'static str)> {
let mut statement = connection.prepare("PRAGMA table_info(cookies)")?;
let columns = statement.query_map([], |row| row.get::<_, String>(1))?;
+ let mut partition_key = "NULL";
+ let mut cross_site_ancestor = "0";
for column in columns {
- if column? == name {
- return Ok(true);
+ match column?.as_str() {
+ "top_frame_site_key" => partition_key = "top_frame_site_key",
+ "has_cross_site_ancestor" => cross_site_ancestor = "has_cross_site_ancestor",
+ _ => {}
}
}
- Ok(false)
+ Ok((partition_key, cross_site_ancestor))
}
fn chrome_expiry(micros: i64) -> Option<OffsetDateTime> {
@@ -265,26 +240,17 @@ fn decrypt_cookie_value(
String::from_utf8(plaintext.to_vec()).context("decrypted cookie value is not UTF-8")
}
-pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result<usize> {
+pub(crate) fn read_local_storage(profile: &Path) -> Result<LocalStorage> {
let source = profile.join("Local Storage/leveldb");
if !source.is_dir() {
- return Ok(0);
+ return Ok(LocalStorage::new());
}
let snapshot = TempDir::new().context("failed to create localStorage snapshot directory")?;
copy_dir(&source, snapshot.path())?;
- let options = Options {
- create_if_missing: false,
- ..Options::default()
- };
- let mut database = DB::open(snapshot.path(), options)
- .map_err(|error| anyhow!("failed to open Chrome localStorage LevelDB snapshot: {error}"))?;
- let mut origins = BTreeMap::<String, BTreeMap<Vec<u16>, Vec<u16>>>::new();
- let mut iterator = database
- .new_iter()
- .map_err(|error| anyhow!("failed to iterate Chrome localStorage: {error}"))?;
- iterator.seek_to_first();
- let mut imported = 0;
- while let Some((key, value)) = iterator.next() {
+ let entries = moli_leveldb_parser::read_snapshot(snapshot.path())
+ .context("failed to read Chrome localStorage LevelDB snapshot")?;
+ let mut origins = LocalStorage::new();
+ for (key, value) in entries {
let Some((origin, item_key)) = decode_local_storage_key(&key) else {
continue;
};
@@ -294,37 +260,9 @@ pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result
origins
.entry(origin)
.or_default()
- .insert(item_key, item_value);
- imported += 1;
- }
- let origins = origins
- .into_iter()
- .map(|(origin, entries)| {
- let entries = entries
- .into_iter()
- .map(|(key, value)| {
- serde_json::json!({
- "key": json_dom_string(key),
- "value": json_dom_string(value),
- })
- })
- .collect::<Vec<_>>();
- (origin, serde_json::json!({ "entries": entries }))
- })
- .collect::<serde_json::Map<_, _>>();
- let bytes = serde_json::to_vec_pretty(&serde_json::json!({
- "version": 1,
- "origins": origins,
- }))?;
- moli_browser_profile::write_file_atomically(destination, &bytes, "Chrome localStorage import")?;
- Ok(imported)
-}
-
-fn json_dom_string(units: Vec<u16>) -> serde_json::Value {
- match String::from_utf16(&units) {
- Ok(text) => serde_json::Value::String(text),
- Err(_) => serde_json::json!({ "utf16": units }),
+ .insert(item_key, item_value.into());
}
+ Ok(origins)
}
fn decode_local_storage_key(key: &[u8]) -> Option<(String, Vec<u16>)> {
@@ -374,13 +312,88 @@ fn copy_dir(source: &Path, destination: &Path) -> Result<()> {
mod tests {
use super::*;
+ #[test]
+ fn imports_chrome_local_storage_from_tables_and_wal_without_resurrecting_deleted_keys()
+ -> Result<()> {
+ let profile = tempfile::tempdir()?;
+ let source = profile.path().join("Local Storage/leveldb");
+ fs::create_dir_all(&source)?;
+ let mut database = rusty_leveldb::DB::open(
+ &source,
+ rusty_leveldb::Options {
+ compressor: 1,
+ ..rusty_leveldb::Options::default()
+ },
+ )?;
+ let token = b"_https://example.com\0\x01token";
+ let deleted = b"_https://example.com\0\x01deleted";
+ let surrogate = b"_https://example.com\0\x00\x00\xd8";
+ database.put(b"VERSION", b"1")?;
+ database.put(token, b"\x01old")?;
+ database.put(deleted, b"\x01removed")?;
+ database.put(surrogate, b"\x00\x00\xdc")?;
+ database.compact_range(b"", b"\xff")?;
+ database.put(token, b"\x01v\xff")?;
+ database.delete(deleted)?;
+ database.flush()?;
+
+ let destination = profile.path().join("moli-profile");
+ let request = crate::ImportRequest {
+ profile_dir: &destination,
+ includes: "storage".parse().unwrap(),
+ chrome_profile_dir: Some(profile.path()),
+ chrome_crypto_key: None,
+ firefox_profile_dir: None,
+ cookie_jars: &[],
+ };
+ assert_eq!(crate::import_session_state(&request)?.storage, 2);
+ let storage_path = destination.join("partitions/default/localstorage.json");
+ let bytes = fs::read(&storage_path)?;
+ let json: serde_json::Value = serde_json::from_slice(&bytes)?;
+ assert_eq!(
+ json["origins"]["https://example.com"]["entries"],
+ serde_json::json!([
+ { "key": "token", "value": "vÿ" },
+ { "key": { "utf16": [0xd800] }, "value": { "utf16": [0xdc00] } },
+ ])
+ );
+
+ // An unreadable live SST must fail the import before replacing a
+ // previously imported profile with partial data.
+ let table = fs::read_dir(&source)?
+ .map(|entry| entry.map(|entry| entry.path()))
+ .collect::<std::io::Result<Vec<_>>>()?
+ .into_iter()
+ .find(|path| path.extension().is_some_and(|extension| extension == "ldb"))
+ .unwrap();
+ let mut corrupt = fs::read(&table)?;
+ corrupt[0] ^= 1;
+ fs::write(table, corrupt)?;
+ assert!(crate::import_session_state(&request).is_err());
+ assert_eq!(fs::read(storage_path)?, bytes);
+ Ok(())
+ }
+
#[test]
fn imports_chrome_cookies_with_and_without_partition_columns() -> Result<()> {
- for partitioned in [false, true] {
+ check_chrome_cookie_imports("DELETE")
+ }
+
+ #[test]
+ fn imports_chrome_cookies_from_wal() -> Result<()> {
+ check_chrome_cookie_imports("WAL")
+ }
+
+ fn check_chrome_cookie_imports(journal_mode: &str) -> Result<()> {
+ for (partitioned, cross_site) in
+ [(false, false), (true, false), (false, true), (true, true)]
+ {
let profile = tempfile::tempdir()?;
let network = profile.path().join("Network");
fs::create_dir(&network)?;
let connection = Connection::open(network.join("Cookies"))?;
+ connection.pragma_update(None, "journal_mode", journal_mode)?;
+ connection.pragma_update(None, "wal_autocheckpoint", 0)?;
connection.execute_batch(
"CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta VALUES ('version', '24');
@@ -398,12 +411,19 @@ mod tests {
if partitioned {
connection.execute_batch(
"ALTER TABLE cookies ADD COLUMN top_frame_site_key TEXT;
- ALTER TABLE cookies ADD COLUMN has_cross_site_ancestor INTEGER;
- UPDATE cookies SET top_frame_site_key = 'https://example.org',
- has_cross_site_ancestor = 1;",
+ UPDATE cookies SET top_frame_site_key = 'https://example.org';",
)?;
}
- drop(connection);
+ if cross_site {
+ connection.execute_batch(
+ "ALTER TABLE cookies ADD COLUMN has_cross_site_ancestor INTEGER;
+ UPDATE cookies SET has_cross_site_ancestor = 1;",
+ )?;
+ }
+ // Keep the writer open so uncheckpointed data remains in the WAL.
+ if journal_mode == "WAL" {
+ assert!(network.join("Cookies-wal").is_file());
+ }
let cookies = import_cookies(profile.path(), &ChromeCryptoKey::System)?;
assert_eq!(cookies.len(), 1);
@@ -423,7 +443,7 @@ mod tests {
cookie.partition_key,
partitioned.then(|| StoredCookiePartitionKey::site(
"https://example.org".to_owned(),
- true,
+ cross_site,
)),
);
}
diff --git a/moli-cookie-import/src/firefox.rs b/moli-cookie-import/src/firefox.rs
index a0a9f844c..acbe2481f 100644
--- a/moli-cookie-import/src/firefox.rs
+++ b/moli-cookie-import/src/firefox.rs
@@ -1,31 +1,20 @@
-use std::{collections::BTreeMap, fs, path::Path};
+use std::{fs, path::Path};
use anyhow::{Context, Result, bail};
use moli_cookie_jar::{
CookiePriority, StoredCookie, StoredCookieSameSite, StoredCookieSourceScheme,
};
-use rusqlite::Connection;
use time::OffsetDateTime;
-pub(crate) fn validate_source(path: &Path) -> Result<()> {
- if !path.is_dir() {
- bail!(
- "Firefox profile directory `{}` does not exist",
- path.display()
- );
- }
- Ok(())
-}
+use crate::{sqlite::SqliteSnapshot, storage::LocalStorage};
pub(crate) fn import_cookies(profile: &Path) -> Result<Vec<StoredCookie>> {
let source = profile.join("cookies.sqlite");
if !source.is_file() {
return Ok(Vec::new());
}
- let snapshot = snapshot_sqlite(&source, "Firefox cookies")?;
- let connection = Connection::open(snapshot.path().join("database.sqlite"))
- .context("failed to open Firefox cookies snapshot")?;
- let mut statement = connection
+ let snapshot = SqliteSnapshot::open(&source, "Firefox cookies")?;
+ let mut statement = snapshot.connection
.prepare("SELECT name,value,host,path,expiry,isSecure,isHttpOnly,sameSite,schemeMap,originAttributes FROM moz_cookies")
.context("Firefox cookies database has an unsupported schema")?;
let rows = statement.query_map([], |row| {
@@ -90,18 +79,18 @@ pub(crate) fn import_cookies(profile: &Path) -> Result<Vec<StoredCookie>> {
Ok(cookies)
}
-pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result<usize> {
+pub(crate) fn read_local_storage(profile: &Path) -> Result<LocalStorage> {
let root = profile.join("storage/default");
if !root.is_dir() {
- return Ok(0);
+ return Ok(LocalStorage::new());
}
let mut databases = Vec::new();
collect_local_storage_databases(&root, &mut databases)?;
- let mut origins = BTreeMap::<String, BTreeMap<String, String>>::new();
- let mut imported = 0;
+ databases.sort();
+ let mut origins = LocalStorage::new();
for source in databases {
- let snapshot = snapshot_sqlite(&source, "Firefox localStorage")?;
- let connection = Connection::open(snapshot.path().join("database.sqlite"))?;
+ let snapshot = SqliteSnapshot::open(&source, "Firefox localStorage")?;
+ let connection = &snapshot.connection;
let origin: String = connection
.query_row("SELECT origin FROM database LIMIT 1", [], |row| row.get(0))
.with_context(|| format!("failed to read origin from `{}`", source.display()))?;
@@ -115,19 +104,15 @@ pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result
row.get::<_, Vec<u8>>(3)?,
))
})?;
+ let entries = origins.entry(origin).or_default();
for row in rows {
let (key, conversion, compression, value) = row?;
- let value = decode_local_storage_value(&value, conversion, compression)
+ let value = decode_local_storage_value(value, conversion, compression)
.with_context(|| format!("failed to decode Firefox localStorage `{key}`"))?;
- origins
- .entry(origin.clone())
- .or_default()
- .insert(key, value);
- imported += 1;
+ entries.insert(key.encode_utf16().collect(), value.into());
}
}
- write_local_storage(destination, origins)?;
- Ok(imported)
+ Ok(origins)
}
struct FirefoxCookieRow {
@@ -169,11 +154,11 @@ fn collect_local_storage_databases(
Ok(())
}
-fn decode_local_storage_value(value: &[u8], conversion: i64, compression: i64) -> Result<String> {
+fn decode_local_storage_value(value: Vec<u8>, conversion: i64, compression: i64) -> Result<String> {
let value = match compression {
- 0 => value.to_vec(),
+ 0 => value,
1 => snap::raw::Decoder::new()
- .decompress_vec(value)
+ .decompress_vec(&value)
.context("invalid Snappy payload")?,
other => bail!("unsupported Firefox localStorage compression type {other}"),
};
@@ -193,67 +178,19 @@ fn decode_local_storage_value(value: &[u8], conversion: i64, compression: i64) -
}
}
-fn write_local_storage(
- destination: &Path,
- origins: BTreeMap<String, BTreeMap<String, String>>,
-) -> Result<()> {
- let mut persisted_origins = if destination.is_file() {
- let existing: serde_json::Value = serde_json::from_slice(&fs::read(destination)?)
- .with_context(|| {
- format!(
- "failed to parse existing Moli localStorage `{}`",
- destination.display()
- )
- })?;
- existing
- .get("origins")
- .and_then(serde_json::Value::as_object)
- .cloned()
- .unwrap_or_default()
- } else {
- serde_json::Map::new()
- };
- for (origin, entries) in origins {
- let entries = entries
- .into_iter()
- .map(|(key, value)| serde_json::json!({ "key": key, "value": value }))
- .collect::<Vec<_>>();
- persisted_origins.insert(origin, serde_json::json!({ "entries": entries }));
- }
- let bytes = serde_json::to_vec_pretty(&serde_json::json!({
- "version": 1,
- "origins": persisted_origins,
- }))?;
- moli_browser_profile::write_file_atomically(destination, &bytes, "Firefox localStorage import")
-}
-
-fn snapshot_sqlite(source: &Path, label: &str) -> Result<tempfile::TempDir> {
- let snapshot =
- tempfile::tempdir().with_context(|| format!("failed to create {label} snapshot"))?;
- fs::copy(source, snapshot.path().join("database.sqlite"))
- .with_context(|| format!("failed to snapshot {label} `{}`", source.display()))?;
- for suffix in ["-wal", "-shm"] {
- let companion = std::path::PathBuf::from(format!("{}{suffix}", source.display()));
- if companion.is_file() {
- fs::copy(
- &companion,
- snapshot.path().join(format!("database.sqlite{suffix}")),
- )?;
- }
- }
- Ok(snapshot)
-}
-
#[cfg(test)]
mod tests {
use super::*;
- use rusqlite::params;
+ use rusqlite::{Connection, params};
#[test]
fn decodes_utf8_and_snappy_values() -> Result<()> {
- assert_eq!(decode_local_storage_value(b"hello", 1, 0)?, "hello");
+ assert_eq!(
+ decode_local_storage_value(b"hello".to_vec(), 1, 0)?,
+ "hello"
+ );
let compressed = snap::raw::Encoder::new().compress_vec("世界".as_bytes())?;
- assert_eq!(decode_local_storage_value(&compressed, 1, 1)?, "世界");
+ assert_eq!(decode_local_storage_value(compressed, 1, 1)?, "世界");
Ok(())
}
@@ -263,7 +200,7 @@ mod tests {
.encode_utf16()
.flat_map(u16::to_ne_bytes)
.collect::<Vec<_>>();
- assert_eq!(decode_local_storage_value(&bytes, 0, 0)?, "hello");
+ assert_eq!(decode_local_storage_value(bytes, 0, 0)?, "hello");
Ok(())
}
@@ -338,8 +275,10 @@ mod tests {
assert_eq!(cookies[0].value, "value");
assert_eq!(cookies[0].same_site, StoredCookieSameSite::Lax);
let destination = temp.path().join("localstorage.json");
- assert_eq!(import_local_storage(&profile, &destination)?, 1);
- let json: serde_json::Value = serde_json::from_slice(&fs::read(destination)?)?;
+ let origins = read_local_storage(&profile)?;
+ assert_eq!(origins["https://example.com"].len(), 1);
+ let bytes = crate::storage::prepare(&destination, origins)?.unwrap();
+ let json: serde_json::Value = serde_json::from_slice(&bytes)?;
assert_eq!(
json["origins"]["https://example.com"]["entries"][0]["value"],
"stored-value"
diff --git a/moli-cookie-import/src/lib.rs b/moli-cookie-import/src/lib.rs
index 301f4b4fc..5f6b769c5 100644
--- a/moli-cookie-import/src/lib.rs
+++ b/moli-cookie-import/src/lib.rs
@@ -2,6 +2,8 @@
mod chrome;
mod firefox;
+mod sqlite;
+mod storage;
use std::{
fmt,
@@ -109,11 +111,10 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary
)
})?;
let partition = profile.default_partition();
- let mut summary = ImportSummary::default();
let mut imported_cookies = Vec::new();
+ let mut imported_storage = storage::LocalStorage::new();
if let Some(source) = request.chrome_profile_dir {
- chrome::validate_source(source)?;
if request.includes.contains(ImportIncludes::COOKIES) {
imported_cookies.extend(chrome::import_cookies(
source,
@@ -123,19 +124,16 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary
)?);
}
if request.includes.contains(ImportIncludes::STORAGE) {
- summary.storage +=
- chrome::import_local_storage(source, partition.local_storage_path())?;
+ storage::merge(&mut imported_storage, chrome::read_local_storage(source)?);
}
}
if let Some(source) = request.firefox_profile_dir {
- firefox::validate_source(source)?;
if request.includes.contains(ImportIncludes::COOKIES) {
imported_cookies.extend(firefox::import_cookies(source)?);
}
if request.includes.contains(ImportIncludes::STORAGE) {
- summary.storage +=
- firefox::import_local_storage(source, partition.local_storage_path())?;
+ storage::merge(&mut imported_storage, firefox::read_local_storage(source)?);
}
}
@@ -146,12 +144,31 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary
.with_context(|| format!("failed to import cookie jar `{}`", jar.display()))?,
);
}
- summary.cookies = imported_cookies.len();
- if summary.cookies > 0 {
- let mut cookies = moli_cookie_cache::load_cookie_cache(partition.cookies_path())?;
- cookies.extend(imported_cookies);
- moli_cookie_cache::save_cookie_cache(partition.cookies_path(), cookies)?;
- }
+ }
+
+ let summary = ImportSummary {
+ cookies: imported_cookies.len(),
+ storage: imported_storage.values().map(|entries| entries.len()).sum(),
+ };
+ // Read and merge every source and existing destination before writing.
+ // Each file is replaced atomically; the two files are not a transaction.
+ let cookies = if imported_cookies.is_empty() {
+ None
+ } else {
+ let mut cookies = moli_cookie_cache::load_cookie_cache(partition.cookies_path())?;
+ cookies.extend(imported_cookies);
+ Some(cookies)
+ };
+ let storage = storage::prepare(partition.local_storage_path(), imported_storage)?;
+ if let Some(bytes) = storage {
+ moli_browser_profile::write_file_atomically(
+ partition.local_storage_path(),
+ &bytes,
+ "localStorage import",
+ )?;
+ }
+ if let Some(cookies) = cookies {
+ moli_cookie_cache::save_cookie_cache(partition.cookies_path(), cookies)?;
}
Ok(summary)
@@ -166,72 +183,21 @@ fn validate_request(request: &ImportRequest<'_>) -> Result<()> {
if !request.cookie_jars.is_empty() && !request.includes.contains(ImportIncludes::COOKIES) {
bail!("cookie jars can only import cookies; include `cookies` in the import selection");
}
+ for (browser, source) in [
+ ("Chrome", request.chrome_profile_dir),
+ ("Firefox", request.firefox_profile_dir),
+ ] {
+ if let Some(source) = source
+ && !source.is_dir()
+ {
+ bail!(
+ "{browser} profile directory `{}` does not exist",
+ source.display()
+ );
+ }
+ }
Ok(())
}
#[cfg(test)]
-mod tests {
- use std::fs;
-
- use anyhow::Result;
-
- use super::*;
-
- #[test]
- fn bundled_sqlite_omits_unused_extensions() -> Result<()> {
- // Inspect the linked engine so dependency upgrades cannot silently
- // reintroduce extensions excluded by .cargo/config.toml.
- let connection = rusqlite::Connection::open_in_memory()?;
- for option in [
- "ENABLE_FTS3",
- "ENABLE_FTS3_PARENTHESIS",
- "ENABLE_FTS4",
- "ENABLE_FTS5",
- "ENABLE_RTREE",
- "ENABLE_DBSTAT_VTAB",
- "ENABLE_STAT4",
- "ENABLE_LOAD_EXTENSION",
- ] {
- let enabled: bool =
- connection.query_row("SELECT sqlite_compileoption_used(?1)", [option], |row| {
- row.get(0)
- })?;
- assert!(!enabled, "bundled SQLite unexpectedly enables {option}");
- }
- let omits_extension_loading: bool = connection.query_row(
- "SELECT sqlite_compileoption_used('OMIT_LOAD_EXTENSION')",
- [],
- |row| row.get(0),
- )?;
- assert!(omits_extension_loading);
- Ok(())
- }
-
- #[test]
- fn imports_a_netscape_cookie_jar() -> Result<()> {
- let temp = tempfile::tempdir()?;
- let jar = temp.path().join("cookies.txt");
- let destination = temp.path().join("moli-profile");
- fs::write(
- &jar,
- "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tFALSE\t0\tsession\tvalue\n",
- )?;
-
- let summary = import_session_state(&ImportRequest {
- profile_dir: &destination,
- includes: "all".parse().expect("valid import selection"),
- chrome_profile_dir: None,
- chrome_crypto_key: None,
- firefox_profile_dir: None,
- cookie_jars: &[jar],
- })?;
-
- let profile = BrowserProfile::open(&destination)?;
- let cookies =
- moli_cookie_cache::load_cookie_cache(profile.default_partition().cookies_path())?;
- assert_eq!(summary.cookies, 1);
- assert_eq!(cookies.len(), 1);
- assert_eq!(cookies[0].name, "session");
- Ok(())
- }
-}
+mod tests;
diff --git a/moli-cookie-import/src/sqlite.rs b/moli-cookie-import/src/sqlite.rs
new file mode 100644
index 000000000..ef202d31a
--- /dev/null
+++ b/moli-cookie-import/src/sqlite.rs
@@ -0,0 +1,106 @@
+use std::{fs, io::ErrorKind, path::Path};
+
+use anyhow::{Context, Result};
+use rusqlite::{Connection, OpenFlags};
+use tempfile::TempDir;
+
+pub(crate) struct SqliteSnapshot {
+ // Close the connection before removing its files (fields drop in order).
+ pub connection: Connection,
+ _directory: TempDir,
+}
+
+impl SqliteSnapshot {
+ /// Copy a database and its WAL companions. The source files must stay
+ /// stable during copying; this is not an online-backup transaction.
+ pub fn open(source: &Path, label: &str) -> Result<Self> {
+ let directory =
+ tempfile::tempdir().with_context(|| format!("failed to create {label} snapshot"))?;
+ let database = directory.path().join("database.sqlite");
+ fs::copy(source, &database)
+ .with_context(|| format!("failed to snapshot {label} `{}`", source.display()))?;
+ for suffix in ["-wal", "-shm"] {
+ let mut companion = source.as_os_str().to_owned();
+ companion.push(suffix);
+ match fs::copy(
+ &companion,
+ directory.path().join(format!("database.sqlite{suffix}")),
+ ) {
+ Ok(_) => {}
+ Err(error) if error.kind() == ErrorKind::NotFound => {}
+ Err(error) => {
+ return Err(error).with_context(|| {
+ format!(
+ "failed to snapshot {label} `{}`",
+ Path::new(&companion).display()
+ )
+ });
+ }
+ }
+ }
+ let connection = Connection::open_with_flags(&database, OpenFlags::SQLITE_OPEN_READ_ONLY)
+ .with_context(|| format!("failed to open {label} snapshot"))?;
+ Ok(Self {
+ connection,
+ _directory: directory,
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn snapshot_reads_wal_without_changing_source_and_rejects_writes() -> Result<()> {
+ check_wal_snapshot(Path::new("cookies.sqlite"))
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn snapshot_preserves_non_utf8_paths_when_finding_wal_files() -> Result<()> {
+ use std::os::unix::ffi::OsStrExt;
+ check_wal_snapshot(Path::new(std::ffi::OsStr::from_bytes(
+ b"cookies-\xff.sqlite",
+ )))
+ }
+
+ fn check_wal_snapshot(name: &Path) -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let source = directory.path().join(name);
+ let writer = Connection::open(&source)?;
+ writer.execute_batch(
+ "PRAGMA journal_mode=WAL;
+ PRAGMA wal_autocheckpoint=0;
+ CREATE TABLE entries (value TEXT);
+ PRAGMA wal_checkpoint(TRUNCATE);
+ INSERT INTO entries VALUES ('only in WAL');",
+ )?;
+ let files = fs::read_dir(directory.path())?
+ .map(|entry| {
+ let path = entry?.path();
+ let bytes = fs::read(&path)?;
+ Ok((path, bytes))
+ })
+ .collect::<std::io::Result<Vec<_>>>()?;
+ let snapshot = SqliteSnapshot::open(&source, "test database")?;
+ let value: String =
+ snapshot
+ .connection
+ .query_row("SELECT value FROM entries", [], |row| row.get(0))?;
+ assert_eq!(value, "only in WAL");
+ let error = snapshot
+ .connection
+ .execute("INSERT INTO entries VALUES ('forbidden')", [])
+ .unwrap_err();
+ assert_eq!(
+ error.sqlite_error_code(),
+ Some(rusqlite::ErrorCode::ReadOnly)
+ );
+ drop(snapshot);
+ for (path, bytes) in files {
+ assert_eq!(fs::read(path)?, bytes);
+ }
+ Ok(())
+ }
+}
diff --git a/moli-cookie-import/src/storage.rs b/moli-cookie-import/src/storage.rs
new file mode 100644
index 000000000..8a28354d5
--- /dev/null
+++ b/moli-cookie-import/src/storage.rs
@@ -0,0 +1,126 @@
+use std::{collections::BTreeMap, fs, io::ErrorKind, path::Path};
+
+use anyhow::{Context, Result, ensure};
+use serde::{Deserialize, Serialize};
+
+// Keys use UTF-16 units so JSON text and explicit UTF-16 keys merge identically.
+// Values keep owned UTF-8 strings when possible, including Firefox's SQL values.
+pub(crate) type LocalStorage = BTreeMap<String, BTreeMap<Vec<u16>, DomString>>;
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum DomString {
+ Text(String),
+ Utf16 { utf16: Vec<u16> },
+}
+
+impl From<String> for DomString {
+ fn from(value: String) -> Self {
+ Self::Text(value)
+ }
+}
+
+impl From<Vec<u16>> for DomString {
+ fn from(units: Vec<u16>) -> Self {
+ match String::from_utf16(&units) {
+ Ok(text) => Self::Text(text),
+ Err(_) => Self::Utf16 { utf16: units },
+ }
+ }
+}
+
+impl DomString {
+ fn into_units(self) -> Vec<u16> {
+ match self {
+ Self::Text(text) => text.encode_utf16().collect(),
+ Self::Utf16 { utf16 } => utf16,
+ }
+ }
+}
+
+#[derive(Serialize, Deserialize)]
+struct StorageFile {
+ version: u8,
+ origins: BTreeMap<String, StorageArea>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct StorageArea {
+ entries: Vec<StorageEntry>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct StorageEntry {
+ key: DomString,
+ value: DomString,
+}
+
+pub(crate) fn merge(target: &mut LocalStorage, source: LocalStorage) {
+ for (origin, entries) in source {
+ if !entries.is_empty() {
+ target.entry(origin).or_default().extend(entries);
+ }
+ }
+}
+
+/// Prepare the merged file without writing it, so other inputs can be checked
+/// before any destination data is replaced. Later sources win on matching keys.
+pub(crate) fn prepare(destination: &Path, imported: LocalStorage) -> Result<Option<Vec<u8>>> {
+ if imported.values().all(BTreeMap::is_empty) {
+ return Ok(None);
+ }
+ let mut origins = load(destination)?;
+ merge(&mut origins, imported);
+ let file = StorageFile {
+ version: 1,
+ origins: origins
+ .into_iter()
+ .map(|(origin, entries)| {
+ let entries = entries
+ .into_iter()
+ .map(|(key, value)| StorageEntry {
+ key: key.into(),
+ value,
+ })
+ .collect();
+ (origin, StorageArea { entries })
+ })
+ .collect(),
+ };
+ serde_json::to_vec_pretty(&file)
+ .context("failed to serialize imported localStorage")
+ .map(Some)
+}
+
+fn load(path: &Path) -> Result<LocalStorage> {
+ let bytes = match fs::read(path) {
+ Ok(bytes) => bytes,
+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(LocalStorage::new()),
+ Err(error) => {
+ return Err(error)
+ .with_context(|| format!("failed to read Moli localStorage `{}`", path.display()));
+ }
+ };
+ if bytes.is_empty() {
+ return Ok(LocalStorage::new());
+ }
+ let file: StorageFile = serde_json::from_slice(&bytes)
+ .with_context(|| format!("failed to parse Moli localStorage `{}`", path.display()))?;
+ ensure!(
+ file.version == 1,
+ "unsupported Moli localStorage version {}",
+ file.version
+ );
+ Ok(file
+ .origins
+ .into_iter()
+ .map(|(origin, area)| {
+ let entries = area
+ .entries
+ .into_iter()
+ .map(|entry| (entry.key.into_units(), entry.value))
+ .collect();
+ (origin, entries)
+ })
+ .collect())
+}
diff --git a/moli-cookie-import/src/tests.rs b/moli-cookie-import/src/tests.rs
new file mode 100644
index 000000000..c7838e206
--- /dev/null
+++ b/moli-cookie-import/src/tests.rs
@@ -0,0 +1,307 @@
+use std::fs;
+
+use anyhow::Result;
+use rusqlite::{Connection, params};
+
+use super::*;
+
+fn storage_request(profile_dir: &Path) -> ImportRequest<'_> {
+ ImportRequest {
+ profile_dir,
+ includes: "storage".parse().unwrap(),
+ chrome_profile_dir: None,
+ chrome_crypto_key: None,
+ firefox_profile_dir: None,
+ cookie_jars: &[],
+ }
+}
+
+fn chrome_storage(profile: &Path, entries: &[(&[u8], &[u8])]) -> Result<()> {
+ let path = profile.join("Local Storage/leveldb");
+ fs::create_dir_all(&path)?;
+ let mut database = rusty_leveldb::DB::open(path, rusty_leveldb::Options::default())?;
+ database.put(b"VERSION", b"1")?;
+ for (key, value) in entries {
+ database.put(key, value)?;
+ }
+ database.flush()?;
+ Ok(())
+}
+
+fn firefox_storage(profile: &Path, entries: &[(&str, &str)]) -> Result<PathBuf> {
+ let directory = profile.join("storage/default/https+++example.com/ls");
+ fs::create_dir_all(&directory)?;
+ let path = directory.join("data.sqlite");
+ let connection = Connection::open(&path)?;
+ connection.execute_batch(
+ "CREATE TABLE database (origin TEXT);
+ INSERT INTO database VALUES ('https://example.com');
+ CREATE TABLE data (key TEXT, conversion_type INTEGER, compression_type INTEGER, value BLOB);",
+ )?;
+ for (key, value) in entries {
+ connection.execute(
+ "INSERT INTO data VALUES (?1, 1, 0, ?2)",
+ params![key, value.as_bytes()],
+ )?;
+ }
+ Ok(path)
+}
+
+fn existing_storage(destination: &Path) -> Result<PathBuf> {
+ let path = destination.join("partitions/default/localstorage.json");
+ fs::create_dir_all(path.parent().unwrap())?;
+ fs::write(
+ &path,
+ serde_json::to_vec_pretty(&serde_json::json!({
+ "version": 1,
+ "origins": {
+ "https://keep.example": {"entries": [{"key": "keep", "value": "untouched"}]},
+ "https://example.com": {"entries": [
+ {"key": "preserved", "value": "keep"},
+ {"key": {"utf16": [115, 104, 97, 114, 101, 100]}, "value": "old shared"},
+ {"key": "raw-value", "value": {"utf16": [55296]}},
+ {"key": {"utf16": [55296]}, "value": "old surrogate"}
+ ]}
+ }
+ }))?,
+ )?;
+ Ok(path)
+}
+
+#[test]
+fn merges_both_browsers_with_existing_storage_by_utf16_key() -> Result<()> {
+ let temp = tempfile::tempdir()?;
+ let destination = temp.path().join("moli");
+ let chrome = temp.path().join("chrome");
+ let firefox = temp.path().join("firefox");
+ let storage = existing_storage(&destination)?;
+ chrome_storage(
+ &chrome,
+ &[
+ (b"_https://example.com\0\x01chrome-only", b"\x01chrome"),
+ (b"_https://example.com\0\x01shared", b"\x01chrome"),
+ (b"_https://example.com\0\x00\x00\xd8", b"\x00\x00\xdc"),
+ (b"_https://chrome.example\0\x01token", b"\x01chrome"),
+ ],
+ )?;
+ firefox_storage(
+ &firefox,
+ &[("shared", "firefox"), ("firefox-only", "firefox")],
+ )?;
+ let mut request = storage_request(&destination);
+ request.chrome_profile_dir = Some(&chrome);
+ request.firefox_profile_dir = Some(&firefox);
+ let summary = import_session_state(&request)?;
+ assert_eq!(
+ summary,
+ ImportSummary {
+ cookies: 0,
+ storage: 5
+ }
+ );
+ let bytes = fs::read(&storage)?;
+ let actual: serde_json::Value = serde_json::from_slice(&bytes)?;
+ assert_eq!(
+ actual,
+ serde_json::json!({
+ "version": 1,
+ "origins": {
+ "https://chrome.example": {"entries": [{"key": "token", "value": "chrome"}]},
+ "https://keep.example": {"entries": [{"key": "keep", "value": "untouched"}]},
+ "https://example.com": {"entries": [
+ {"key": "chrome-only", "value": "chrome"},
+ {"key": "firefox-only", "value": "firefox"},
+ {"key": "preserved", "value": "keep"},
+ {"key": "raw-value", "value": {"utf16": [55296]}},
+ {"key": "shared", "value": "firefox"},
+ {"key": {"utf16": [55296]}, "value": {"utf16": [56320]}}
+ ]}
+ }
+ })
+ );
+ assert_eq!(import_session_state(&request)?, summary);
+ assert_eq!(fs::read(storage)?, bytes);
+ Ok(())
+}
+
+#[test]
+fn empty_browser_storage_does_not_create_or_replace_a_file() -> Result<()> {
+ for existing in [false, true] {
+ let temp = tempfile::tempdir()?;
+ let destination = temp.path().join("moli");
+ let chrome = temp.path().join("chrome");
+ let firefox = temp.path().join("firefox");
+ chrome_storage(&chrome, &[])?;
+ firefox_storage(&firefox, &[])?;
+ let storage = destination.join("partitions/default/localstorage.json");
+ let before = if existing {
+ Some(fs::read(existing_storage(&destination)?)?)
+ } else {
+ None
+ };
+ let mut request = storage_request(&destination);
+ request.chrome_profile_dir = Some(&chrome);
+ request.firefox_profile_dir = Some(&firefox);
+ assert_eq!(import_session_state(&request)?, ImportSummary::default());
+ assert_eq!(fs::read(storage).ok(), before);
+ }
+ Ok(())
+}
+
+#[test]
+fn rejects_missing_source_directories_before_creating_a_profile() -> Result<()> {
+ for browser in ["Chrome", "Firefox"] {
+ let temp = tempfile::tempdir()?;
+ let destination = temp.path().join("moli");
+ let missing = temp.path().join("missing");
+ let mut request = storage_request(&destination);
+ if browser == "Chrome" {
+ request.chrome_profile_dir = Some(&missing);
+ } else {
+ request.firefox_profile_dir = Some(&missing);
+ }
+ let error = import_session_state(&request).unwrap_err();
+ assert!(error.to_string().contains(browser));
+ assert!(!destination.exists());
+ }
+ Ok(())
+}
+
+#[test]
+fn later_source_failures_leave_existing_storage_and_cookies_unchanged() -> Result<()> {
+ for corrupt_firefox in [true, false] {
+ let temp = tempfile::tempdir()?;
+ let destination = temp.path().join("moli");
+ let chrome = temp.path().join("chrome");
+ let firefox = temp.path().join("firefox");
+ let storage = existing_storage(&destination)?;
+ let cookies = storage.with_file_name("cookies.json");
+ moli_cookie_cache::save_cookie_cache(&cookies, [])?;
+ let before_storage = fs::read(&storage)?;
+ let before_cookies = fs::read(&cookies)?;
+ let database = firefox_storage(&firefox, &[("new", "firefox")])?;
+ let jars = [temp.path().join("missing-cookie-jar")];
+ let mut request = storage_request(&destination);
+ request.firefox_profile_dir = Some(&firefox);
+ if corrupt_firefox {
+ chrome_storage(
+ &chrome,
+ &[(b"_https://example.com\0\x01new", b"\x01chrome")],
+ )?;
+ request.chrome_profile_dir = Some(&chrome);
+ fs::write(database, b"corrupt SQLite file")?;
+ } else {
+ request.includes = "all".parse().unwrap();
+ request.cookie_jars = &jars;
+ }
+ let error = format!("{:#}", import_session_state(&request).unwrap_err());
+ assert!(
+ error.contains(if corrupt_firefox {
+ "data.sqlite"
+ } else {
+ "missing-cookie-jar"
+ }),
+ "{error}"
+ );
+ assert_eq!(fs::read(storage)?, before_storage);
+ assert_eq!(fs::read(cookies)?, before_cookies);
+ }
+ Ok(())
+}
+
+#[test]
+fn invalid_destination_files_are_rejected_before_either_file_is_written() -> Result<()> {
+ for (target, invalid) in [
+ ("cookies.json", "{"),
+ ("localstorage.json", "{"),
+ ("localstorage.json", r#"{"version":2,"origins":{}}"#),
+ (
+ "localstorage.json",
+ r#"{"version":1,"origins":{"https://example.com":{"entries":[{"key":{},"value":"x"}]}}}"#,
+ ),
+ ] {
+ let temp = tempfile::tempdir()?;
+ let destination = temp.path().join("moli");
+ let firefox = temp.path().join("firefox");
+ let storage = existing_storage(&destination)?;
+ let cookies = storage.with_file_name("cookies.json");
+ moli_cookie_cache::save_cookie_cache(&cookies, [])?;
+ fs::write(storage.with_file_name(target), invalid)?;
+ let before_storage = fs::read(&storage)?;
+ let before_cookies = fs::read(&cookies)?;
+ firefox_storage(&firefox, &[("new", "firefox")])?;
+ let jars = [temp.path().join("cookies.txt")];
+ fs::write(
+ &jars[0],
+ "example.com\tFALSE\t/\tFALSE\t0\tsession\tnew-value\n",
+ )?;
+ let mut request = storage_request(&destination);
+ request.includes = "all".parse().unwrap();
+ request.firefox_profile_dir = Some(&firefox);
+ request.cookie_jars = &jars;
+ assert!(
+ import_session_state(&request).is_err(),
+ "accepted {target}: {invalid}"
+ );
+ assert_eq!(fs::read(storage)?, before_storage);
+ assert_eq!(fs::read(cookies)?, before_cookies);
+ }
+ Ok(())
+}
+
+#[test]
+fn bundled_sqlite_omits_unused_extensions() -> Result<()> {
+ // Inspect the linked engine so dependency upgrades cannot silently
+ // reintroduce extensions excluded by .cargo/config.toml.
+ let connection = rusqlite::Connection::open_in_memory()?;
+ for option in [
+ "ENABLE_FTS3",
+ "ENABLE_FTS3_PARENTHESIS",
+ "ENABLE_FTS4",
+ "ENABLE_FTS5",
+ "ENABLE_RTREE",
+ "ENABLE_DBSTAT_VTAB",
+ "ENABLE_STAT4",
+ "ENABLE_LOAD_EXTENSION",
+ ] {
+ let enabled: bool =
+ connection.query_row("SELECT sqlite_compileoption_used(?1)", [option], |row| {
+ row.get(0)
+ })?;
+ assert!(!enabled, "bundled SQLite unexpectedly enables {option}");
+ }
+ let omits_extension_loading: bool = connection.query_row(
+ "SELECT sqlite_compileoption_used('OMIT_LOAD_EXTENSION')",
+ [],
+ |row| row.get(0),
+ )?;
+ assert!(omits_extension_loading);
+ Ok(())
+}
+
+#[test]
+fn imports_a_netscape_cookie_jar() -> Result<()> {
+ let temp = tempfile::tempdir()?;
+ let jar = temp.path().join("cookies.txt");
+ let destination = temp.path().join("moli-profile");
+ fs::write(
+ &jar,
+ "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tFALSE\t0\tsession\tvalue\n",
+ )?;
+
+ let summary = import_session_state(&ImportRequest {
+ profile_dir: &destination,
+ includes: "all".parse().expect("valid import selection"),
+ chrome_profile_dir: None,
+ chrome_crypto_key: None,
+ firefox_profile_dir: None,
+ cookie_jars: &[jar],
+ })?;
+
+ let profile = BrowserProfile::open(&destination)?;
+ let cookies = moli_cookie_cache::load_cookie_cache(profile.default_partition().cookies_path())?;
+ assert_eq!(summary.cookies, 1);
+ assert_eq!(cookies.len(), 1);
+ assert_eq!(cookies[0].name, "session");
+ Ok(())
+}
diff --git a/moli-leveldb-parser/AUTHORS-LevelDB b/moli-leveldb-parser/AUTHORS-LevelDB
new file mode 100644
index 000000000..2439d7a45
--- /dev/null
+++ b/moli-leveldb-parser/AUTHORS-LevelDB
@@ -0,0 +1,12 @@
+# Names should be added to this file like so:
+# Name or Organization <email address>
+
+Google Inc.
+
+# Initial version authors:
+Jeffrey Dean <[email protected]>
+Sanjay Ghemawat <[email protected]>
+
+# Partial list of contributors:
+Kevin Regan <[email protected]>
+Johan Bilien <[email protected]>
diff --git a/moli-leveldb-parser/Cargo.toml b/moli-leveldb-parser/Cargo.toml
new file mode 100644
index 000000000..687dc7bc1
--- /dev/null
+++ b/moli-leveldb-parser/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+license = "(MIT OR Apache-2.0) AND BSD-3-Clause"
+name = "moli-leveldb-parser"
+version = "0.1.0"
+edition = "2024"
+description = "Read-only LevelDB snapshot reader"
+
+[dependencies]
+anyhow = "1.0.100"
+crc32c = "0.6.8"
+snap = "1.1.1"
+zerocopy = { version = "0.8.46", default-features = false, features = ["derive"] }
+
+[dev-dependencies]
+# Generate database fixtures and compare snapshot contents with an independent engine.
+rusty-leveldb = "4.0.1"
+tempfile = "3.23.0"
+
+[lints]
+workspace = true
diff --git a/moli-leveldb-parser/LICENSE-LevelDB b/moli-leveldb-parser/LICENSE-LevelDB
new file mode 100644
index 000000000..8e80208cd
--- /dev/null
+++ b/moli-leveldb-parser/LICENSE-LevelDB
@@ -0,0 +1,27 @@
+Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/moli-leveldb-parser/TESTS.md b/moli-leveldb-parser/TESTS.md
new file mode 100644
index 000000000..660bd4932
--- /dev/null
+++ b/moli-leveldb-parser/TESTS.md
@@ -0,0 +1,166 @@
+# Chromium LevelDB test ports
+
+The source is Chromium's actual LevelDB dependency, pinned to
+[`7ee830d02b623e8ffe0b95d59a74db1e58da04c5`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/).
+Chromium revision
+[`d9fa5a15858b7189aa1a70cebe811169154ebcf0`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/)
+selects that gitlink and identifies it as LevelDB 1.23 in
+[`README.chromium`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/README.chromium).
+The tests retain the upstream [license](LICENSE-LevelDB) and
+[authors](AUTHORS-LevelDB).
+
+The suite has 133 tests: 68 ports/adaptations of the scenarios listed below,
+56 additional parser regressions, and the 9 original snapshot tests. This is
+coverage of the read-only parser, not the complete Chromium LevelDB engine
+suite. Each original scenario keeps its snake_case name, with a suffix where
+its observable behavior was adapted. Supporting fixtures are test-only code.
+
+```sh
+cargo test -p moli-leveldb-parser -p moli-cookie-import --lib
+cargo fmt --all
+cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo nextest run --no-fail-fast
+```
+
+## WAL physical records
+
+Source: [`db/log_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/log_test.cc).
+Port: [`src/tests/log.rs`](src/tests/log.rs), 31 tests, including these 23 upstream cases:
+
+| Upstream cases | What is checked |
+| --- | --- |
+| `Empty`, `ReadWrite`, `ManyBlocks` | Empty records and 100,000 records over many physical blocks |
+| `Fragmentation`, `RandomRead` | 50/100 KB values and 500 skewed-size records, fixed seed 301 |
+| `MarginalTrailer`, `MarginalTrailer2`, `ShortTrailer`, `AlignedEof` | Exact block ends, short padding and empty FIRST fragments |
+| `OpenForAppend` | Reading a log whose writer resumed at a nonzero block offset |
+| `ReadError` | An injected I/O error propagates |
+| `BadRecordType`, `BadLength`, `ChecksumMismatch` | Unknown types, records crossing a block, and bad CRC32C |
+| `UnexpectedMiddleType`, `UnexpectedLastType`, `UnexpectedFullType`, `UnexpectedFirstType` | Invalid fragment ordering |
+| `TruncatedTrailingRecordIsIgnored`, `BadLengthAtEndIsIgnored`, `MissingLastIsIgnored`, `PartialLastIsIgnored` | Torn final appends never emit partial records |
+| `ErrorJoinsRecords` | Corruption must not join fragments belonging to different logical records |
+
+Upstream's reader reports corruption and can continue salvaging records. The
+parser returns `Err` at complete corruption; the public snapshot API exposes
+no partial result. Tests assert errors instead of upstream dropped-byte counts
+and salvaged data. Incomplete final appends retain upstream EOF behavior.
+
+The remaining 15 upstream cases (`SkipIntoMultiRecord`, `ReadStart`,
+`ReadSecond*`, `ReadThird*`, `ReadFourth*`, `ReadInitialOffsetIntoBlockPadding`,
+`ReadEnd`, `ReadPastEnd`) exercise an initial-offset/seek API. This parser always
+starts at byte zero, so those API-specific cases are outside its scope.
+
+Additional tests cover the empty-FIRST compatibility exception documented in
+[`db/log_reader.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/log_reader.cc),
+checksums of every fragment kind, zero-filled preallocation, corrupt padding,
+short/interrupted reads, callback errors, and cuts around every fragment boundary.
+
+## Coding, CRC and WriteBatch
+
+The four additional [layout tests](src/tests/layout.rs) use literal wire bytes
+at every alignment modulo 8 to check borrowed headers, trailers and restart
+arrays, little-endian values, truncation without cursor advancement, and
+nonempty blocks with zero restarts. Pointer checks ensure views refer to the
+original input; the existing corruption tests continue to validate semantics.
+
+| Source | Ported cases | Local tests |
+| --- | --- | --- |
+| [`util/coding_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/util/coding_test.cc) | All 10: `Fixed32`, `Fixed64`, `EncodingOutput`, `Varint32`, `Varint64`, both overflow cases, both truncation cases, `Strings` | [coding.rs](src/tests/coding.rs) |
+| [`util/crc32c_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/util/crc32c_test.cc) | `StandardResults`, `Values`, `Mask` | [coding.rs](src/tests/coding.rs) |
+| [`db/write_batch_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/write_batch_test.cc) | `Empty`, `Multiple`, `Corruption`, `Append` | [write_batch.rs](src/tests/write_batch.rs) |
+
+`EncodingOutput` checks little-endian decoding against literal bytes; the
+parser has no encoder. `CRC.Mask` checks the production masking wrapper
+against its unmasked checksum. `CRC.Extend` and `WriteBatch.ApproximateSize`
+test APIs absent from this crate. `Append` checks the combined on-disk batch;
+the parser retains the latest version per key rather than all memtable history.
+
+Added regressions cover truncated fixed fields and length prefixes, overlong
+varint32 encodings, high bits that would overflow, every batch truncation point,
+counts too small/large, unsupported tags, 56-bit sequence limits, empty keys and
+values, ordered deletes, and conflicting values at the same sequence.
+
+## MANIFEST / VersionEdit
+
+Source: [`db/version_edit_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/version_edit_test.cc).
+Port: [`src/tests/manifest.rs`](src/tests/manifest.rs), 16 tests.
+
+`VersionEditTest.EncodeDecode` preserves the large `(1 << 50)` file, size and
+sequence values, additions, removals and compaction pointers. Assertions check
+decoded manifest state instead of encoding it again. Its arbitrary `foo`
+comparator is tested as an explicit unsupported-comparator error; supported
+snapshots use `leveldb.BytewiseComparator`.
+
+Additional cases cover metadata split across records, missing metadata,
+previous-log updates, every VersionEdit field's truncation, unknown tags,
+invalid levels/internal keys, sequence overflow, file deletion and level moves,
+deletions-before-additions in one edit, CURRENT validation/selection, incomplete
+final edits, and complete edits with invalid checksums. `ComparatorCheck` also
+corresponds to the custom-comparator mismatch scenario in `db/db_test.cc`.
+
+## SST blocks and internal keys
+
+Sources: [`table/table_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/table/table_test.cc),
+[`db/dbformat_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/dbformat_test.cc).
+Port: [`src/tests/table.rs`](src/tests/table.rs), 23 tests.
+
+| Upstream cases | Adaptation |
+| --- | --- |
+| `Harness.Empty`, `SimpleEmptyKey`, `SimpleSingle`, `SimpleMulti`, `SimpleSpecialKey` | Block and table forward scans; restart intervals 1/16/1024; raw and Snappy data/index blocks |
+| `Harness.ZeroRestartPointsInBlock` | The four-byte empty block emitted by Java LevelDB |
+| `Harness.Randomized` | Original 0..2000 entry-count schedule, binary-key alphabet and skewed lengths; fixed seed 306 |
+| `Harness.RandomizedLongDB` | Checked-in native fixture from 100,000 writes, fixed seed 301; Rust independently reconstructs all 25,479 final entries |
+| `TableTest.ApproximateOffsetOfPlain` | Original values up to 300 KB; verifies full contents instead of an offset-estimation API |
+| `CompressionTableTest.ApproximateOffsetOfCompressed` | Original 25%-compressible workload; verifies Snappy-decoded contents |
+| `FormatTest.InternalKey_EncodeDecode`, `InternalKey_DecodeFromEmpty` | Snapshot decoding of empty/long keys and sequence boundaries; rejection of short internal keys |
+
+The read-only API exposes a full scan. Reverse/custom comparators, backward
+iterators, seeks, memtable constructors, key-shortening/debug formatting, and
+approximate offsets have no corresponding API here. The pinned Chromium
+[`port_chromium.h`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/port/port_chromium.h)
+returns false for Zstd support, so the Zstd parameter is not ported; compression
+types other than raw/Snappy are explicitly rejected.
+
+Added regressions cover footer size/magic, file-size disagreement with MANIFEST,
+out-of-bounds block handles, data/index CRCs, invalid compression with valid
+CRCs, restart counts/offsets/order, illegal shared prefixes, entries crossing
+restart arrays, unknown internal value types, and callback errors. Existing
+snapshot tests also cover malicious Snappy lengths and missing/legacy tables.
+
+## Recovery, versions and tombstones
+
+Sources: [`db/recovery_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/recovery_test.cc),
+[`db/db_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/db_test.cc),
+[`db/corruption_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/corruption_test.cc).
+Port: [`src/tests/recovery.rs`](src/tests/recovery.rs), 21 tests.
+
+| Upstream cases | Read-only expectation |
+| --- | --- |
+| `RecoveryTest.ManifestReused`, `LargeManifestCompacted` | Appended edits and a 3 MB zero-padded MANIFEST read successfully; source files stay unchanged |
+| `RecoveryTest.NoLogFiles`, `LogFileReuse` | Missing logs on a valid snapshot and empty/nonempty current WALs |
+| `RecoveryTest.MultipleMemTables` | Original 1,000 padded keys recovered entirely in memory |
+| `RecoveryTest.MultipleLogFiles` | Recover newer unregistered WALs while ignoring obsolete logs, even with larger sequences |
+| `RecoveryTest.ManifestMissing` | Fail without creating/replacing files |
+| `DBTest.GetLevel0Ordering`, `GetOrderedByLevels` | Newest visible version wins across overlapping L0 files and different levels |
+| `DBTest.Recover`, `RecoveryWithEmptyLog` | Repeated updates across WAL/SST transitions |
+| `DBTest.IterMultiWithDelete`, `IterMultiWithDeleteAndCompaction` | Deleted entries absent from output with both WAL-only and SST+WAL data |
+| `CorruptionTest.SequenceNumberRecovery` | Recovered sequence ordering keeps subsequent updates visible; repair itself is outside scope |
+| `CorruptionTest.Recovery` | Corruption in the original two WAL locations returns an error instead of salvaged partial data |
+
+These are 15 upstream scenarios. Added regressions cover native Chromium
+SST/WAL compatibility, future SST versions, tombstones newer than WAL values,
+empty batches, malformed batches without partial output, and duplicate numeric
+WAL filenames. Recovery checks compare the input files before and after reads.
+
+The nine existing [snapshot tests](src/tests/snapshot.rs) moved from the Chrome
+importer. They retain engine comparison, mtime/content checks, a held writer
+lock, read-only filesystem permissions, obsolete-file resurrection checks,
+`.sst` fallback/missing live tables (`DBTest.StillReadSST`/`MissingSSTFile`
+scenarios), corruption, previous WAL selection, and UTF-agnostic byte handling.
+Chrome string/JSON import tests remain in `moli-cookie-import`.
+
+Native fixtures, their C++ generator, checksums and regeneration instructions
+are in [tests/fixtures](tests/fixtures/README.md). Normal tests require no C++
+compiler or network. Compaction scheduling, writable recovery/repair,
+concurrency, caches/Bloom-filter performance, locking policy, and write-failure
+injection for a database engine are outside the parser's API and are not
+claimed as ported.
diff --git a/moli-leveldb-parser/src/coding.rs b/moli-leveldb-parser/src/coding.rs
new file mode 100644
index 000000000..9d9e26f0f
--- /dev/null
+++ b/moli-leveldb-parser/src/coding.rs
@@ -0,0 +1,63 @@
+use anyhow::{Result, anyhow, bail, ensure};
+use zerocopy::{FromBytes, Immutable, KnownLayout, Unaligned};
+
+pub(super) struct Decoder<'a> {
+ pub bytes: &'a [u8],
+}
+
+impl<'a> Decoder<'a> {
+ pub fn new(bytes: &'a [u8]) -> Self {
+ Self { bytes }
+ }
+
+ pub fn take(&mut self, length: usize) -> Result<&'a [u8]> {
+ ensure!(length <= self.bytes.len(), "truncated LevelDB field");
+ let (value, rest) = self.bytes.split_at(length);
+ self.bytes = rest;
+ Ok(value)
+ }
+
+ pub fn fixed<T: FromBytes + KnownLayout + Immutable + Unaligned>(&mut self) -> Result<&'a T> {
+ let (value, rest) =
+ T::ref_from_prefix(self.bytes).map_err(|_| anyhow!("truncated LevelDB field"))?;
+ self.bytes = rest;
+ Ok(value)
+ }
+
+ pub fn varint(&mut self) -> Result<u64> {
+ let mut value = 0;
+ for shift in (0..70).step_by(7) {
+ let byte = self.take(1)?[0];
+ ensure!(shift != 63 || byte <= 1, "LevelDB varint overflow");
+ value |= u64::from(byte & 0x7f) << shift;
+ if byte < 0x80 {
+ return Ok(value);
+ }
+ }
+ bail!("invalid LevelDB varint")
+ }
+
+ pub fn varint32(&mut self) -> Result<u32> {
+ let mut value = 0;
+ for shift in (0..35).step_by(7) {
+ let byte = self.take(1)?[0];
+ ensure!(shift != 28 || byte <= 0x0f, "LevelDB varint32 overflow");
+ value |= u32::from(byte & 0x7f) << shift;
+ if byte < 0x80 {
+ return Ok(value);
+ }
+ }
+ bail!("invalid LevelDB varint32")
+ }
+
+ pub fn slice(&mut self) -> Result<&'a [u8]> {
+ let length = self.varint32()? as usize;
+ self.take(length)
+ }
+}
+
+pub(super) fn masked_crc(bytes: &[u8]) -> u32 {
+ crc32c::crc32c(bytes)
+ .rotate_right(15)
+ .wrapping_add(0xa282_ead8)
+}
diff --git a/moli-leveldb-parser/src/layout.rs b/moli-leveldb-parser/src/layout.rs
new file mode 100644
index 000000000..555217892
--- /dev/null
+++ b/moli-leveldb-parser/src/layout.rs
@@ -0,0 +1,49 @@
+//! Fixed on-disk layouts shared by the log, batch and table readers.
+//!
+//! Little-endian numeric wrappers have alignment 1, so these views work at
+//! arbitrary byte offsets without packed fields or unsafe code. All bit
+//! patterns are representable; readers validate tags, lengths and checksums.
+
+use zerocopy::{
+ FromBytes, Immutable, KnownLayout, Unaligned,
+ byteorder::little_endian::{U16, U32, U64},
+};
+
+/// Physical record header used by both WAL and MANIFEST files.
+#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]
+#[repr(C)]
+pub(super) struct LogHeader {
+ pub checksum: U32,
+ pub length: U16,
+ pub record_type: u8,
+}
+
+#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]
+#[repr(C)]
+pub(super) struct WriteBatchHeader {
+ pub sequence: U64,
+ pub count: U32,
+}
+
+#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]
+#[repr(C)]
+pub(super) struct BlockTrailer {
+ pub compression: u8,
+ pub checksum: U32,
+}
+
+#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]
+#[repr(C)]
+pub(super) struct TableFooter {
+ /// Two varint-encoded block handles followed by padding to 40 bytes.
+ pub block_handles: [u8; 40],
+ pub magic: U64,
+}
+
+// Keep the structs tied to the file format even if fields are edited later.
+const _: () = {
+ assert!(size_of::<LogHeader>() == 7);
+ assert!(size_of::<WriteBatchHeader>() == 12);
+ assert!(size_of::<BlockTrailer>() == 5);
+ assert!(size_of::<TableFooter>() == 48);
+};
diff --git a/moli-leveldb-parser/src/lib.rs b/moli-leveldb-parser/src/lib.rs
new file mode 100644
index 000000000..c124fbf11
--- /dev/null
+++ b/moli-leveldb-parser/src/lib.rs
@@ -0,0 +1,151 @@
+//! Read the current contents of a stable LevelDB snapshot without opening
+//! a database engine. Only CURRENT, its MANIFEST, live tables and recoverable WAL
+//! files are read. Recovery happens in memory; no locks, logs, tables, manifests
+//! or compactions are created, including when the reader is dropped.
+//!
+//! Formats: https://github.com/google/leveldb/tree/main/doc
+//! Manifest tags: https://github.com/google/leveldb/blob/main/db/version_edit.cc
+
+mod coding;
+mod layout;
+mod log;
+mod manifest;
+mod table;
+
+use std::{
+ collections::{BTreeMap, btree_map::Entry},
+ fs,
+ path::Path,
+};
+
+use anyhow::{Context, Result, anyhow, bail, ensure};
+use zerocopy::{FromBytes, byteorder::little_endian::U64};
+
+use coding::Decoder;
+use layout::WriteBatchHeader;
+use manifest::Manifest;
+
+const MAX_SEQUENCE: u64 = (1 << 56) - 1;
+type Versions = BTreeMap<Vec<u8>, (u64, Option<Vec<u8>>)>;
+
+/// Read the latest key/value pairs from an existing, stable LevelDB directory.
+///
+/// The caller must keep the directory unchanged for the duration of this call
+/// (for example, by copying a closed database). This function acquires no locks
+/// and never modifies the directory. It supports the bytewise comparator and
+/// uncompressed or Snappy-compressed tables, including legacy `.sst` filenames.
+/// Deleted keys and obsolete files are excluded; complete WAL records are
+/// replayed in memory. An unfinished append at EOF is ignored.
+///
+/// # Errors
+///
+/// Returns an error for I/O failures, missing live tables, corrupt complete
+/// records, or unsupported formats. No partial result is returned on failure.
+pub fn read_snapshot(directory: &Path) -> Result<BTreeMap<Vec<u8>, Vec<u8>>> {
+ let manifest = Manifest::read(directory)?;
+ let mut versions = Versions::new();
+ let mut logs = BTreeMap::new();
+ for entry in fs::read_dir(directory)? {
+ let entry = entry?;
+ let name = entry.file_name();
+ let Some(name) = name.to_str().and_then(|name| name.strip_suffix(".log")) else {
+ continue;
+ };
+ if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) {
+ continue;
+ }
+ let number = name.parse::<u64>().context("invalid LevelDB log number")?;
+ if number >= manifest.log_number || number == manifest.previous_log_number {
+ ensure!(
+ logs.insert(number, entry.path()).is_none(),
+ "duplicate LevelDB log number"
+ );
+ }
+ }
+ let mut last_sequence = manifest.last_sequence;
+ for path in logs.values() {
+ log::read_records(path, |record| {
+ if let Some(sequence) = read_batch(record, &mut versions)? {
+ last_sequence = last_sequence.max(sequence);
+ }
+ Ok(())
+ })
+ .with_context(|| format!("failed to read LevelDB WAL `{}`", path.display()))?;
+ }
+ // WAL recovery establishes the latest visible sequence, just as a normal
+ // database open does. SST versions newer than that snapshot are invisible.
+ for ((_, number), size) in manifest.tables {
+ let mut path = directory.join(format!("{number:06}.ldb"));
+ if !path.try_exists()? {
+ path = directory.join(format!("{number:06}.sst"));
+ }
+ table::read_table(&path, size, |key, value| {
+ let (key, tag) =
+ U64::ref_from_suffix(key).map_err(|_| anyhow!("truncated LevelDB internal key"))?;
+ let tag = tag.get();
+ let value = match tag & 0xff {
+ 0 => None,
+ 1 => Some(value),
+ other => bail!("unsupported LevelDB value type {other}"),
+ };
+ if tag >> 8 <= last_sequence {
+ apply(&mut versions, key, tag >> 8, value)?;
+ }
+ Ok(())
+ })
+ .with_context(|| format!("failed to read LevelDB table `{}`", path.display()))?;
+ }
+ Ok(versions
+ .into_iter()
+ .filter_map(|(key, (_, value))| value.map(|value| (key, value)))
+ .collect())
+}
+
+fn read_batch(record: &[u8], versions: &mut Versions) -> Result<Option<u64>> {
+ let mut input = Decoder::new(record);
+ let header = input.fixed::<WriteBatchHeader>()?;
+ let sequence = header.sequence.get();
+ let count = header.count.get();
+ ensure!(
+ sequence <= MAX_SEQUENCE && u64::from(count.saturating_sub(1)) <= MAX_SEQUENCE - sequence,
+ "LevelDB batch sequence overflow"
+ );
+ for index in 0..count {
+ let kind = input.take(1)?[0];
+ let key = input.slice()?;
+ let value = match kind {
+ 0 => None,
+ 1 => Some(input.slice()?),
+ other => bail!("unsupported LevelDB batch value type {other}"),
+ };
+ apply(versions, key, sequence + u64::from(index), value)?;
+ }
+ ensure!(
+ input.bytes.is_empty(),
+ "LevelDB batch record count mismatch"
+ );
+ Ok((count > 0).then_some(sequence + u64::from(count.saturating_sub(1))))
+}
+
+fn apply(versions: &mut Versions, key: &[u8], sequence: u64, value: Option<&[u8]>) -> Result<()> {
+ match versions.entry(key.to_vec()) {
+ Entry::Vacant(entry) => {
+ entry.insert((sequence, value.map(<[u8]>::to_vec)));
+ }
+ Entry::Occupied(mut entry) => {
+ let (previous_sequence, previous_value) = entry.get();
+ if *previous_sequence == sequence {
+ ensure!(
+ previous_value.as_deref() == value,
+ "conflicting LevelDB values at the same sequence"
+ );
+ } else if *previous_sequence < sequence {
+ entry.insert((sequence, value.map(<[u8]>::to_vec)));
+ }
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/moli-leveldb-parser/src/log.rs b/moli-leveldb-parser/src/log.rs
new file mode 100644
index 000000000..3bf4bbff4
--- /dev/null
+++ b/moli-leveldb-parser/src/log.rs
@@ -0,0 +1,114 @@
+use std::{fs::File, io::Read, path::Path};
+
+use anyhow::{Context, Result, bail, ensure};
+
+use super::{
+ coding::{Decoder, masked_crc},
+ layout::LogHeader,
+};
+
+const BLOCK_SIZE: usize = 32 * 1024;
+const HEADER_SIZE: usize = size_of::<LogHeader>();
+
+/// Read complete logical records, including records spanning physical blocks.
+/// Like LevelDB recovery, ignore an unfinished append at EOF. A complete record
+/// with a bad checksum is an error, rather than a silently incomplete import.
+pub(super) fn read_records(path: &Path, visit: impl FnMut(&[u8]) -> Result<()>) -> Result<()> {
+ read_records_from(File::open(path)?, visit)
+}
+
+pub(super) fn read_records_from(
+ mut file: impl Read,
+ mut visit: impl FnMut(&[u8]) -> Result<()>,
+) -> Result<()> {
+ let mut block = Vec::with_capacity(BLOCK_SIZE);
+ let mut fragment = Vec::new();
+ let mut fragmented = false;
+ loop {
+ block.clear();
+ file.by_ref()
+ .take(BLOCK_SIZE as u64)
+ .read_to_end(&mut block)?;
+ if block.is_empty() {
+ return Ok(());
+ }
+ let mut offset = 0;
+ while offset < block.len() {
+ let remaining = &block[offset..];
+ if remaining.len() < HEADER_SIZE {
+ ensure!(
+ block.len() < BLOCK_SIZE || remaining.iter().all(|byte| *byte == 0),
+ "nonzero LevelDB log block trailer"
+ );
+ break;
+ }
+ let header = Decoder::new(remaining).fixed::<LogHeader>()?;
+ let length = usize::from(header.length.get());
+ let kind = header.record_type;
+ if kind == 0 && length == 0 {
+ // Writers may preallocate zero-filled blocks.
+ ensure!(
+ remaining.iter().all(|byte| *byte == 0),
+ "invalid LevelDB log padding"
+ );
+ fragment.clear();
+ fragmented = false;
+ break;
+ }
+ if HEADER_SIZE + length > remaining.len() {
+ ensure!(
+ block.len() < BLOCK_SIZE,
+ "LevelDB log record crosses a block boundary"
+ );
+ return Ok(());
+ }
+ let payload = &remaining[HEADER_SIZE..HEADER_SIZE + length];
+ ensure!(
+ // The CRC covers the record type and payload, excluding length.
+ masked_crc(
+ &remaining[std::mem::offset_of!(LogHeader, record_type)..HEADER_SIZE + length]
+ ) == header.checksum.get(),
+ "LevelDB log checksum mismatch"
+ );
+ match kind {
+ 1 => {
+ // Older writers could leave an empty FIRST at a block's
+ // end, then start a new record in the next block. Match
+ // LevelDB's compatibility exception only for empty data.
+ ensure!(
+ fragment.is_empty(),
+ "unfinished LevelDB log record before FULL"
+ );
+ fragmented = false;
+ visit(payload)?;
+ }
+ 2 => {
+ ensure!(
+ fragment.is_empty(),
+ "unfinished LevelDB log record before FIRST"
+ );
+ fragment.clear();
+ fragment
+ .try_reserve(payload.len())
+ .context("LevelDB log record is too large")?;
+ fragment.extend_from_slice(payload);
+ fragmented = true;
+ }
+ 3 | 4 => {
+ ensure!(fragmented, "LevelDB log fragment without FIRST");
+ fragment
+ .try_reserve(payload.len())
+ .context("LevelDB log record is too large")?;
+ fragment.extend_from_slice(payload);
+ if kind == 4 {
+ visit(&fragment)?;
+ fragment.clear();
+ fragmented = false;
+ }
+ }
+ _ => bail!("unsupported LevelDB log record type {kind}"),
+ }
+ offset += HEADER_SIZE + length;
+ }
+ }
+}
diff --git a/moli-leveldb-parser/src/manifest.rs b/moli-leveldb-parser/src/manifest.rs
new file mode 100644
index 000000000..4d1fa4b80
--- /dev/null
+++ b/moli-leveldb-parser/src/manifest.rs
@@ -0,0 +1,97 @@
+use std::{collections::BTreeMap, fs, path::Path};
+
+use anyhow::{Context, Result, bail, ensure};
+
+use super::{MAX_SEQUENCE, coding::Decoder, log::read_records};
+
+#[derive(Default)]
+pub(super) struct Manifest {
+ pub tables: BTreeMap<(u32, u64), u64>,
+ pub log_number: u64,
+ pub previous_log_number: u64,
+ pub last_sequence: u64,
+}
+
+impl Manifest {
+ pub fn read(directory: &Path) -> Result<Self> {
+ let current = fs::read_to_string(directory.join("CURRENT"))
+ .context("failed to read LevelDB CURRENT")?;
+ let name = current
+ .strip_suffix('\n')
+ .context("LevelDB CURRENT must end with a newline")?;
+ let number = name
+ .strip_prefix("MANIFEST-")
+ .context("invalid LevelDB CURRENT manifest name")?;
+ ensure!(
+ !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()),
+ "invalid LevelDB CURRENT manifest name"
+ );
+ let mut manifest = Self::default();
+ let mut has_log = false;
+ let mut has_next_file = false;
+ let mut has_last_sequence = false;
+ read_records(&directory.join(name), |record| {
+ let mut input = Decoder::new(record);
+ let mut deleted = Vec::new();
+ let mut added = Vec::new();
+ while !input.bytes.is_empty() {
+ match input.varint32()? {
+ 1 => ensure!(
+ input.slice()? == b"leveldb.BytewiseComparator",
+ "unsupported LevelDB comparator"
+ ),
+ 2 => {
+ manifest.log_number = input.varint()?;
+ has_log = true;
+ }
+ 3 => {
+ input.varint()?;
+ has_next_file = true;
+ }
+ 4 => {
+ manifest.last_sequence = input.varint()?;
+ ensure!(
+ manifest.last_sequence <= MAX_SEQUENCE,
+ "invalid LevelDB sequence number"
+ );
+ has_last_sequence = true;
+ }
+ 5 => {
+ read_level(&mut input)?;
+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB compaction key");
+ }
+ 6 => deleted.push((read_level(&mut input)?, input.varint()?)),
+ 7 => {
+ let level = read_level(&mut input)?;
+ let number = input.varint()?;
+ let size = input.varint()?;
+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB smallest key");
+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB largest key");
+ added.push(((level, number), size));
+ }
+ 9 => manifest.previous_log_number = input.varint()?,
+ tag => bail!("unsupported LevelDB manifest tag {tag}"),
+ }
+ }
+ // A VersionEdit applies deletions before additions, including a
+ // table moved between levels without rewriting its contents.
+ for key in deleted {
+ manifest.tables.remove(&key);
+ }
+ manifest.tables.extend(added);
+ Ok(())
+ })
+ .with_context(|| format!("failed to read LevelDB manifest `{name}`"))?;
+ ensure!(
+ has_log && has_next_file && has_last_sequence,
+ "incomplete LevelDB manifest"
+ );
+ Ok(manifest)
+ }
+}
+
+fn read_level(input: &mut Decoder<'_>) -> Result<u32> {
+ let level = input.varint32()?;
+ ensure!(level < 7, "unsupported LevelDB level {level}");
+ Ok(level)
+}
diff --git a/moli-leveldb-parser/src/table.rs b/moli-leveldb-parser/src/table.rs
new file mode 100644
index 000000000..85f4b2147
--- /dev/null
+++ b/moli-leveldb-parser/src/table.rs
@@ -0,0 +1,168 @@
+use std::{
+ fs::File,
+ io::{Read, Seek, SeekFrom},
+ path::Path,
+};
+
+use anyhow::{Context, Result, anyhow, bail, ensure};
+use zerocopy::{FromBytes, byteorder::little_endian::U32};
+
+use super::{
+ coding::{Decoder, masked_crc},
+ layout::{BlockTrailer, TableFooter},
+};
+
+const FOOTER_SIZE: u64 = size_of::<TableFooter>() as u64;
+const TRAILER_SIZE: u64 = size_of::<BlockTrailer>() as u64;
+const TABLE_MAGIC: u64 = 0xdb47_7524_8b80_fb57;
+
+pub(super) fn read_table(
+ path: &Path,
+ expected_size: u64,
+ mut visit: impl FnMut(&[u8], &[u8]) -> Result<()>,
+) -> Result<()> {
+ let mut file = File::open(path)?;
+ let size = file.metadata()?.len();
+ ensure!(
+ size == expected_size && size >= FOOTER_SIZE,
+ "invalid LevelDB table size"
+ );
+ file.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;
+ let mut footer = [0; FOOTER_SIZE as usize];
+ file.read_exact(&mut footer)?;
+ let footer = Decoder::new(&footer).fixed::<TableFooter>()?;
+ ensure!(
+ footer.magic.get() == TABLE_MAGIC,
+ "invalid LevelDB table magic"
+ );
+ let mut footer = Decoder::new(&footer.block_handles);
+ // Full scans don't need the metaindex or Bloom filters.
+ read_handle(&mut footer)?;
+ let index_handle = read_handle(&mut footer)?;
+ let index = read_block(&mut file, size - FOOTER_SIZE, index_handle)?;
+ read_entries(&index, |_, handle| {
+ let mut input = Decoder::new(handle);
+ let handle = read_handle(&mut input)?;
+ let block = read_block(&mut file, index_handle.0, handle)?;
+ read_entries(&block, &mut visit)
+ })
+}
+
+fn read_handle(input: &mut Decoder<'_>) -> Result<(u64, u64)> {
+ Ok((input.varint()?, input.varint()?))
+}
+
+fn read_block(file: &mut File, end: u64, (offset, size): (u64, u64)) -> Result<Vec<u8>> {
+ ensure!(
+ offset <= end && end - offset >= TRAILER_SIZE && size <= end - offset - TRAILER_SIZE,
+ "LevelDB block lies outside its table"
+ );
+ let size = usize::try_from(size).context("LevelDB block is too large")?;
+ let length = size
+ .checked_add(TRAILER_SIZE as usize)
+ .context("LevelDB block length overflow")?;
+ let mut block = Vec::new();
+ block
+ .try_reserve_exact(length)
+ .context("LevelDB block is too large")?;
+ block.resize(length, 0);
+ file.seek(SeekFrom::Start(offset))?;
+ file.read_exact(&mut block)?;
+ let trailer = Decoder::new(&block[size..]).fixed::<BlockTrailer>()?;
+ let checksum_end = size + std::mem::offset_of!(BlockTrailer, checksum);
+ ensure!(
+ masked_crc(&block[..checksum_end]) == trailer.checksum.get(),
+ "LevelDB table block checksum mismatch"
+ );
+ let compression = trailer.compression;
+ block.truncate(size);
+ match compression {
+ 0 => Ok(block),
+ 1 => {
+ let length = snap::raw::decompress_len(&block)?;
+ // A Snappy copy encodes at most 64 output bytes. Validate the
+ // claimed size before letting the decoder allocate from its header.
+ ensure!(
+ length <= block.len().saturating_mul(64),
+ "invalid Snappy decoded length"
+ );
+ let mut decoded = Vec::new();
+ decoded
+ .try_reserve_exact(length)
+ .context("decoded LevelDB block is too large")?;
+ decoded.resize(length, 0);
+ let written = snap::raw::Decoder::new().decompress(&block, &mut decoded)?;
+ ensure!(written == length, "invalid Snappy decoded length");
+ Ok(decoded)
+ }
+ other => bail!("unsupported LevelDB compression type {other}"),
+ }
+}
+
+/// Borrow the entries and restart offsets directly from a block, including at
+/// unaligned byte offsets. Zerocopy checks the count and buffer size together.
+pub(super) fn split_restarts(block: &[u8]) -> Result<(&[u8], &[U32])> {
+ let (body, count) = U32::ref_from_suffix(block)
+ .map_err(|_| anyhow!("truncated LevelDB block restart array"))?;
+ let (entries, restarts) = <[U32]>::ref_from_suffix_with_elems(body, count.get() as usize)
+ .map_err(|_| anyhow!("invalid LevelDB restart count"))?;
+ // Java LevelDB can emit an empty block containing only a zero count.
+ if restarts.is_empty() {
+ ensure!(
+ entries.is_empty(),
+ "nonempty LevelDB block has no restart points"
+ );
+ }
+ Ok((entries, restarts))
+}
+
+pub(super) fn read_entries(
+ block: &[u8],
+ mut visit: impl FnMut(&[u8], &[u8]) -> Result<()>,
+) -> Result<()> {
+ let (entries, restarts) = split_restarts(block)?;
+ let entries_end = entries.len();
+ let count = restarts.len();
+ let mut previous = None;
+ for restart in restarts {
+ let offset = restart.get() as usize;
+ ensure!(
+ offset < entries_end || (entries_end == 0 && count == 1 && offset == 0),
+ "LevelDB restart offset out of bounds"
+ );
+ if let Some(previous) = previous {
+ ensure!(previous < offset, "unordered LevelDB restart offsets");
+ } else {
+ ensure!(offset == 0, "first LevelDB restart offset must be zero");
+ }
+ previous = Some(offset);
+ }
+ let mut next_restart = 0;
+ let mut input = Decoder::new(entries);
+ let mut key = Vec::new();
+ while !input.bytes.is_empty() {
+ let offset = entries_end - input.bytes.len();
+ let shared = input.varint32()? as usize;
+ let unshared = input.varint32()? as usize;
+ let value_length = input.varint32()? as usize;
+ ensure!(shared <= key.len(), "invalid LevelDB key prefix length");
+ if let Some(restart) = restarts.get(next_restart) {
+ let restart = restart.get() as usize;
+ ensure!(restart >= offset, "LevelDB restart points inside an entry");
+ if restart == offset {
+ ensure!(shared == 0, "LevelDB restart entry has a shared prefix");
+ next_restart += 1;
+ }
+ }
+ let suffix = input.take(unshared)?;
+ let value = input.take(value_length)?;
+ key.truncate(shared);
+ key.extend_from_slice(suffix);
+ visit(&key, value)?;
+ }
+ ensure!(
+ entries_end == 0 || next_restart == count,
+ "LevelDB restart points inside an entry"
+ );
+ Ok(())
+}
diff --git a/moli-leveldb-parser/src/tests/coding.rs b/moli-leveldb-parser/src/tests/coding.rs
new file mode 100644
index 000000000..b2c3d0984
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/coding.rs
@@ -0,0 +1,225 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Ports of util/coding_test.cc and util/crc32c_test.cc; see ../../TESTS.md,
+// ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.
+
+use zerocopy::byteorder::little_endian::{U32, U64};
+
+use super::support::{slice, varint};
+use crate::coding::{Decoder, masked_crc};
+
+#[test]
+fn fixed32() {
+ let bytes: Vec<_> = (0..100_000u32).flat_map(u32::to_le_bytes).collect();
+ let mut input = Decoder::new(&bytes);
+ for expected in 0..100_000 {
+ assert_eq!(input.fixed::<U32>().unwrap().get(), expected);
+ }
+ assert!(input.bytes.is_empty());
+}
+
+#[test]
+fn fixed64() {
+ let values: Vec<_> = (0..64)
+ .flat_map(|power| {
+ let v = 1u64 << power;
+ [v - 1, v, v + 1]
+ })
+ .collect();
+ let bytes: Vec<_> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
+ let mut input = Decoder::new(&bytes);
+ for expected in values {
+ assert_eq!(input.fixed::<U64>().unwrap().get(), expected);
+ }
+ assert!(input.bytes.is_empty());
+}
+
+#[test]
+fn encoding_output() {
+ assert_eq!(
+ Decoder::new(&[1, 2, 3, 4]).fixed::<U32>().unwrap().get(),
+ 0x0403_0201
+ );
+ assert_eq!(
+ Decoder::new(&[1, 2, 3, 4, 5, 6, 7, 8])
+ .fixed::<U64>()
+ .unwrap()
+ .get(),
+ 0x0807_0605_0403_0201
+ );
+}
+
+#[test]
+fn varint32() {
+ let values: Vec<_> = (0..32 * 32u32).map(|i| (i / 32) << (i % 32)).collect();
+ let mut bytes = Vec::new();
+ for value in &values {
+ varint(u64::from(*value), &mut bytes);
+ }
+ let mut input = Decoder::new(&bytes);
+ for value in values {
+ let before = input.bytes.len();
+ assert_eq!(input.varint32().unwrap(), value);
+ assert_eq!(
+ before - input.bytes.len(),
+ ((32 - value.leading_zeros()).max(1).div_ceil(7)) as usize
+ );
+ }
+ assert!(input.bytes.is_empty());
+}
+
+#[test]
+fn varint64() {
+ let mut values = vec![0, 100, u64::MAX, u64::MAX - 1];
+ values.extend((0..64).flat_map(|k| {
+ let v = 1u64 << k;
+ [v, v - 1, v + 1]
+ }));
+ let mut bytes = Vec::new();
+ for value in &values {
+ varint(*value, &mut bytes);
+ }
+ let mut input = Decoder::new(&bytes);
+ for value in values {
+ let before = input.bytes.len();
+ assert_eq!(input.varint().unwrap(), value);
+ assert_eq!(
+ before - input.bytes.len(),
+ ((64 - value.leading_zeros()).max(1).div_ceil(7)) as usize
+ );
+ }
+ assert!(input.bytes.is_empty());
+}
+
+#[test]
+fn varint32_overflow() {
+ assert!(
+ Decoder::new(&[0x81, 0x82, 0x83, 0x84, 0x85, 0x11])
+ .varint32()
+ .is_err()
+ );
+}
+
+#[test]
+fn varint32_truncation() {
+ let value = (1u32 << 31) + 100;
+ let mut bytes = Vec::new();
+ varint(u64::from(value), &mut bytes);
+ for end in 0..bytes.len() {
+ assert!(Decoder::new(&bytes[..end]).varint32().is_err());
+ }
+ assert_eq!(Decoder::new(&bytes).varint32().unwrap(), value);
+}
+
+#[test]
+fn varint64_overflow() {
+ assert!(
+ Decoder::new(&[
+ 0x81, 0x82, 0x83, 0x84, 0x85, 0x81, 0x82, 0x83, 0x84, 0x85, 0x11
+ ])
+ .varint()
+ .is_err()
+ );
+}
+
+#[test]
+fn varint64_truncation() {
+ let value = (1u64 << 63) + 100;
+ let mut bytes = Vec::new();
+ varint(value, &mut bytes);
+ for end in 0..bytes.len() {
+ assert!(Decoder::new(&bytes[..end]).varint().is_err());
+ }
+ assert_eq!(Decoder::new(&bytes).varint().unwrap(), value);
+}
+
+#[test]
+fn strings() {
+ let values = [vec![], b"foo".to_vec(), b"bar".to_vec(), vec![b'x'; 200]];
+ let mut bytes = Vec::new();
+ for value in &values {
+ slice(value, &mut bytes);
+ }
+ let mut input = Decoder::new(&bytes);
+ for value in values {
+ assert_eq!(input.slice().unwrap(), value);
+ }
+ assert!(input.bytes.is_empty());
+}
+
+#[test]
+fn crc_standard_results() {
+ let vectors = [
+ (vec![0; 32], 0x8a91_36aa_u32),
+ (vec![0xff; 32], 0x62a8_ab43),
+ ((0..32).collect(), 0x46dd_794e),
+ ((0..32).rev().collect(), 0x113f_db5c),
+ (
+ vec![
+ 0x01, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x14, 0, 0, 0, 0, 0, 0x04, 0,
+ 0, 0, 0, 0x14, 0, 0, 0, 0x18, 0x28, 0, 0, 0, 0, 0, 0, 0, 0x02, 0, 0, 0, 0, 0, 0, 0,
+ ],
+ 0xd996_3a56,
+ ),
+ ];
+ for (bytes, expected) in vectors {
+ // Check the actual parser wrapper, including its masking step.
+ assert_eq!(unmask(masked_crc(&bytes)), expected);
+ }
+}
+
+fn unmask(value: u32) -> u32 {
+ value.wrapping_sub(0xa282_ead8).rotate_left(15)
+}
+
+#[test]
+fn crc_values() {
+ assert_ne!(masked_crc(b"a"), masked_crc(b"foo"));
+}
+
+#[test]
+fn crc_mask() {
+ let crc = crc32c::crc32c(b"foo");
+ assert_ne!(crc, masked_crc(b"foo"));
+ assert_eq!(crc, unmask(masked_crc(b"foo")));
+}
+
+#[test]
+fn fixed_fields_cannot_read_past_end() {
+ for length in 0..8 {
+ assert!(Decoder::new(&[0; 8][..length]).fixed::<U64>().is_err());
+ }
+}
+
+#[test]
+fn length_prefixed_fields_are_bounded() {
+ for bytes in [
+ vec![0xff, 0xff, 0xff, 0xff, 0x0f],
+ vec![3, b'a', b'b'],
+ vec![0x80],
+ ] {
+ assert!(Decoder::new(&bytes).slice().is_err());
+ }
+}
+
+#[test]
+fn varint32_overlong_zero_is_rejected() {
+ assert!(
+ Decoder::new(&[0x80, 0x80, 0x80, 0x80, 0x80, 0])
+ .varint32()
+ .is_err()
+ );
+}
+
+#[test]
+fn varint_high_bits_cannot_wrap() {
+ assert!(
+ Decoder::new(&[0xff, 0xff, 0xff, 0xff, 0x10])
+ .varint32()
+ .is_err()
+ );
+ assert!(
+ Decoder::new(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2])
+ .varint()
+ .is_err()
+ );
+}
diff --git a/moli-leveldb-parser/src/tests/layout.rs b/moli-leveldb-parser/src/tests/layout.rs
new file mode 100644
index 000000000..aeac6e210
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/layout.rs
@@ -0,0 +1,98 @@
+use zerocopy::{FromBytes, Immutable, KnownLayout, Unaligned};
+
+use crate::{
+ coding::Decoder,
+ layout::{BlockTrailer, LogHeader, TableFooter, WriteBatchHeader},
+ table::{read_entries, split_restarts},
+};
+
+fn at_each_alignment(bytes: &[u8], mut visit: impl FnMut(&[u8])) {
+ let mut storage = vec![0xff; bytes.len() + 8];
+ for offset in 0..8 {
+ let input = &mut storage[offset..offset + bytes.len()];
+ input.copy_from_slice(bytes);
+ visit(input);
+ }
+}
+
+fn borrowed<T: FromBytes + KnownLayout + Immutable + Unaligned>(input: &[u8]) -> &T {
+ let mut decoder = Decoder::new(input);
+ let value = decoder.fixed::<T>().unwrap();
+ assert_eq!(std::ptr::from_ref(value).cast::<u8>(), input.as_ptr());
+ // The header stays borrowed while the cursor advances over its payload.
+ let payload = decoder.take(input.len() - size_of::<T>()).unwrap();
+ assert_eq!(payload.as_ptr(), input[size_of::<T>()..].as_ptr());
+ assert!(decoder.bytes.is_empty());
+ value
+}
+
+#[test]
+fn headers_borrow_unaligned_little_endian_fields() {
+ // Literal format bytes, independent of the fixture writers and derives.
+ at_each_alignment(&[1, 2, 3, 4, 5, 6, 0xfe, 0xaa], |input| {
+ let header = borrowed::<LogHeader>(input);
+ assert_eq!(header.checksum.get(), 0x0403_0201);
+ assert_eq!(header.length.get(), 0x0605);
+ // Unknown types remain representable for the reader to validate.
+ assert_eq!(header.record_type, 0xfe);
+ });
+ at_each_alignment(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0xaa], |input| {
+ let header = borrowed::<WriteBatchHeader>(input);
+ assert_eq!(header.sequence.get(), 0x0807_0605_0403_0201);
+ assert_eq!(header.count.get(), 0x0c0b_0a09);
+ });
+ at_each_alignment(&[0xfe, 1, 2, 3, 4], |input| {
+ let trailer = borrowed::<BlockTrailer>(input);
+ assert_eq!(trailer.compression, 0xfe);
+ assert_eq!(trailer.checksum.get(), 0x0403_0201);
+ });
+ let mut footer = [0x10; 48];
+ footer[40..].copy_from_slice(&[0x57, 0xfb, 0x80, 0x8b, 0x24, 0x75, 0x47, 0xdb]);
+ at_each_alignment(&footer, |input| {
+ let footer = borrowed::<TableFooter>(input);
+ assert_eq!(footer.block_handles, [0x10; 40]);
+ assert_eq!(footer.magic.get(), 0xdb47_7524_8b80_fb57);
+ });
+}
+
+fn check_truncations<T: FromBytes + KnownLayout + Immutable + Unaligned>() {
+ for length in 0..size_of::<T>() {
+ let bytes = vec![0; length];
+ let mut decoder = Decoder::new(&bytes);
+ assert!(decoder.fixed::<T>().is_err());
+ assert_eq!(decoder.bytes, bytes);
+ assert_eq!(decoder.bytes.as_ptr(), bytes.as_ptr());
+ }
+}
+
+#[test]
+fn truncated_headers_do_not_consume_input() {
+ check_truncations::<LogHeader>();
+ check_truncations::<WriteBatchHeader>();
+ check_truncations::<BlockTrailer>();
+ check_truncations::<TableFooter>();
+}
+
+#[test]
+fn restart_offsets_borrow_the_original_block_at_any_alignment() {
+ // Three body bytes, two little-endian restart offsets, and their count.
+ let bytes = [0xa1, 0xa2, 0xa3, 0, 0, 0, 0, 1, 2, 3, 4, 2, 0, 0, 0];
+ at_each_alignment(&bytes, |input| {
+ let (body, restarts) = split_restarts(input).unwrap();
+ assert_eq!(body, [0xa1, 0xa2, 0xa3]);
+ assert_eq!(body.as_ptr(), input.as_ptr());
+ assert_eq!(restarts.len(), 2);
+ assert_eq!(restarts.as_ptr().cast::<u8>(), input[3..].as_ptr());
+ assert_eq!(restarts[0].get(), 0);
+ assert_eq!(restarts[1].get(), 0x0403_0201);
+ });
+}
+
+#[test]
+fn nonempty_block_cannot_hide_entries_with_zero_restarts() {
+ for body in [b"x".as_slice(), &[0; 4], &[0, 1, 1, b'k', b'v']] {
+ let mut bytes = body.to_vec();
+ bytes.extend_from_slice(&[0; 4]);
+ assert!(read_entries(&bytes, |_, _| panic!("invalid block must be rejected")).is_err());
+ }
+}
diff --git a/moli-leveldb-parser/src/tests/log.rs b/moli-leveldb-parser/src/tests/log.rs
new file mode 100644
index 000000000..49b01b5fd
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/log.rs
@@ -0,0 +1,318 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Rust ports of db/log_test.cc. See ../../LICENSE-LevelDB, ../../AUTHORS-LevelDB
+// and ../../TESTS.md. Corruption cases assert Err instead of reporter/salvage.
+
+use std::io::{self, Cursor, Read};
+
+use anyhow::Result;
+
+use super::support::*;
+use crate::log::read_records_from;
+
+fn read(bytes: &[u8]) -> Result<Vec<Vec<u8>>> {
+ let mut records = Vec::new();
+ read_records_from(Cursor::new(bytes), |record| {
+ records.push(record.to_vec());
+ Ok(())
+ })?;
+ Ok(records)
+}
+
+fn roundtrip(records: &[Vec<u8>]) {
+ assert_eq!(read(&log_bytes(records)).unwrap(), records);
+}
+
+fn error(bytes: &[u8], expected: &str) {
+ let error = format!("{:#}", read(bytes).unwrap_err());
+ assert!(error.contains(expected), "{error}");
+}
+
+#[test]
+fn empty() {
+ roundtrip(&[]);
+}
+
+#[test]
+fn read_write() {
+ roundtrip(&[b"foo".to_vec(), b"bar".to_vec(), vec![], b"xxxx".to_vec()]);
+}
+
+#[test]
+fn many_blocks() {
+ let records: Vec<_> = (0..100_000).map(|i| format!("{i}.").into_bytes()).collect();
+ roundtrip(&records);
+}
+
+#[test]
+fn fragmentation() {
+ roundtrip(&[
+ b"small".to_vec(),
+ big_string(b"medium", 50_000),
+ big_string(b"large", 100_000),
+ ]);
+}
+
+#[test]
+fn marginal_trailer() {
+ roundtrip(&[
+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE),
+ vec![],
+ b"bar".to_vec(),
+ ]);
+}
+
+#[test]
+fn marginal_trailer2() {
+ roundtrip(&[
+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE),
+ b"bar".to_vec(),
+ ]);
+}
+
+#[test]
+fn short_trailer() {
+ roundtrip(&[
+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE + 4),
+ vec![],
+ b"bar".to_vec(),
+ ]);
+}
+
+#[test]
+fn aligned_eof() {
+ roundtrip(&[big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE + 4)]);
+}
+
+#[test]
+fn open_for_append() {
+ let mut bytes = log_bytes(&[b"hello".to_vec()]);
+ append_log(&mut bytes, b"world");
+ assert_eq!(read(&bytes).unwrap(), [b"hello", b"world"]);
+}
+
+#[test]
+fn random_read() {
+ let mut random = Random::new(301);
+ let records: Vec<_> = (0..500)
+ .map(|i| big_string(format!("{i}.").as_bytes(), random.skewed(17)))
+ .collect();
+ roundtrip(&records);
+}
+
+#[test]
+fn read_error() {
+ struct Broken;
+ impl Read for Broken {
+ fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {
+ Err(io::Error::other("injected read error"))
+ }
+ }
+ let result = read_records_from(Broken, |_| panic!("read error must propagate"));
+ assert!(format!("{:#}", result.unwrap_err()).contains("injected read error"));
+}
+
+#[test]
+fn bad_record_type() {
+ let mut bytes = log_bytes(&[b"foo".to_vec()]);
+ bytes[6] += 100;
+ fix_log_checksum(&mut bytes, 0);
+ error(&bytes, "record type");
+}
+
+#[test]
+fn truncated_trailing_record_is_ignored() {
+ let mut bytes = log_bytes(&[b"foo".to_vec()]);
+ bytes.truncate(bytes.len() - 4);
+ assert!(read(&bytes).unwrap().is_empty());
+}
+
+#[test]
+fn bad_length() {
+ let mut bytes = log_bytes(&[
+ big_string(b"bar", BLOCK_SIZE - HEADER_SIZE),
+ b"foo".to_vec(),
+ ]);
+ bytes[4] += 1;
+ error(&bytes, "block boundary");
+}
+
+#[test]
+fn bad_length_at_end_is_ignored() {
+ let mut bytes = log_bytes(&[b"foo".to_vec()]);
+ bytes.pop();
+ assert!(read(&bytes).unwrap().is_empty());
+}
+
+#[test]
+fn checksum_mismatch() {
+ let mut bytes = log_bytes(&[b"foo".to_vec()]);
+ bytes[0] = bytes[0].wrapping_add(10);
+ error(&bytes, "checksum mismatch");
+}
+
+#[test]
+fn unexpected_middle_type() {
+ error(&physical_record(3, b"foo"), "without FIRST");
+}
+
+#[test]
+fn unexpected_last_type() {
+ error(&physical_record(4, b"foo"), "without FIRST");
+}
+
+#[test]
+fn unexpected_full_type() {
+ let mut bytes = log_bytes(&[b"foo".to_vec(), b"bar".to_vec()]);
+ bytes[6] = 2;
+ fix_log_checksum(&mut bytes, 0);
+ error(&bytes, "before FULL");
+}
+
+#[test]
+fn unexpected_first_type() {
+ let mut bytes = log_bytes(&[b"foo".to_vec(), big_string(b"bar", 100_000)]);
+ bytes[6] = 2;
+ fix_log_checksum(&mut bytes, 0);
+ error(&bytes, "before FIRST");
+}
+
+#[test]
+fn missing_last_is_ignored() {
+ let mut bytes = log_bytes(&[big_string(b"bar", BLOCK_SIZE)]);
+ bytes.truncate(bytes.len() - 14);
+ assert!(read(&bytes).unwrap().is_empty());
+}
+
+#[test]
+fn partial_last_is_ignored() {
+ let mut bytes = log_bytes(&[big_string(b"bar", BLOCK_SIZE)]);
+ bytes.pop();
+ assert!(read(&bytes).unwrap().is_empty());
+}
+
+#[test]
+fn error_joins_records() {
+ let mut bytes = log_bytes(&[
+ big_string(b"foo", BLOCK_SIZE),
+ big_string(b"bar", BLOCK_SIZE),
+ b"correct".to_vec(),
+ ]);
+ bytes[BLOCK_SIZE..2 * BLOCK_SIZE].fill(b'x');
+ let mut seen = Vec::new();
+ assert!(
+ read_records_from(Cursor::new(bytes), |record| {
+ seen.push(record.to_vec());
+ Ok(())
+ })
+ .is_err()
+ );
+ assert!(
+ seen.is_empty(),
+ "fragments from different records must never be joined"
+ );
+}
+
+// Additional regressions for the compatibility behavior documented in the
+// pinned db/log_reader.cc and for the snapshot reader's strict error policy.
+#[test]
+fn legacy_empty_first_before_full_or_first() {
+ for record in [b"full".to_vec(), big_string(b"fragmented", 100_000)] {
+ let first = big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE);
+ let mut bytes = log_bytes(std::slice::from_ref(&first));
+ bytes.extend_from_slice(&physical_record(2, b""));
+ append_log(&mut bytes, &record);
+ assert_eq!(read(&bytes).unwrap(), [first, record]);
+ }
+}
+
+#[test]
+fn all_fragment_checksums_are_verified() {
+ let original = log_bytes(&[big_string(b"payload", 3 * BLOCK_SIZE)]);
+ for offset in [0, BLOCK_SIZE, 2 * BLOCK_SIZE, 3 * BLOCK_SIZE] {
+ let mut bytes = original.clone();
+ bytes[offset + HEADER_SIZE] ^= 1;
+ error(&bytes, "checksum mismatch");
+ }
+}
+
+#[test]
+fn truncated_tail_never_emits_a_partial_logical_record() {
+ let prefix = log_bytes(&[b"committed".to_vec()]);
+ let mut bytes = prefix.clone();
+ append_log(&mut bytes, &big_string(b"pending", 2 * BLOCK_SIZE));
+ let mut cuts: Vec<_> = (prefix.len()..prefix.len() + HEADER_SIZE + 2).collect();
+ for boundary in [BLOCK_SIZE, 2 * BLOCK_SIZE, bytes.len()] {
+ cuts.extend(boundary - HEADER_SIZE..boundary + HEADER_SIZE);
+ }
+ for cut in cuts.into_iter().filter(|cut| *cut < bytes.len()) {
+ assert_eq!(read(&bytes[..cut]).unwrap(), [b"committed"], "cut {cut}");
+ }
+}
+
+#[test]
+fn zero_filled_preallocation() {
+ let mut bytes = log_bytes(&[b"foo".to_vec()]);
+ bytes.resize(3 * BLOCK_SIZE, 0);
+ append_log(&mut bytes, b"bar");
+ assert_eq!(read(&bytes).unwrap(), [b"foo", b"bar"]);
+}
+
+#[test]
+fn padding_cannot_join_fragments() {
+ let mut bytes = physical_record(2, b"first");
+ bytes.resize(BLOCK_SIZE, 0);
+ bytes.extend_from_slice(&physical_record(4, b"unrelated"));
+ error(&bytes, "without FIRST");
+}
+
+#[test]
+fn nonzero_padding_is_corruption() {
+ let mut bytes = vec![0; BLOCK_SIZE];
+ bytes[HEADER_SIZE] = 1;
+ error(&bytes, "padding");
+ let mut bytes = log_bytes(&[big_string(b"foo", BLOCK_SIZE - HEADER_SIZE - 1)]);
+ bytes.push(1);
+ error(&bytes, "trailer");
+}
+
+#[test]
+fn short_reads_and_interrupted_io() {
+ struct Chunked {
+ inner: Cursor<Vec<u8>>,
+ calls: usize,
+ }
+ impl Read for Chunked {
+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+ self.calls += 1;
+ if self.calls.is_multiple_of(7) {
+ return Err(io::ErrorKind::Interrupted.into());
+ }
+ let length = buffer.len().min(13);
+ self.inner.read(&mut buffer[..length])
+ }
+ }
+ let records = [big_string(b"foo", 2 * BLOCK_SIZE), b"tail".to_vec()];
+ let source = Chunked {
+ inner: Cursor::new(log_bytes(&records)),
+ calls: 0,
+ };
+ let mut seen = Vec::new();
+ read_records_from(source, |record| {
+ seen.push(record.to_vec());
+ Ok(())
+ })
+ .unwrap();
+ assert_eq!(seen, records);
+}
+
+#[test]
+fn visitor_error_stops_the_read() {
+ let bytes = log_bytes(&[b"foo".to_vec(), b"bar".to_vec()]);
+ let mut calls = 0;
+ let result = read_records_from(Cursor::new(bytes), |_| {
+ calls += 1;
+ anyhow::bail!("visitor error")
+ });
+ assert!(format!("{:#}", result.unwrap_err()).contains("visitor error"));
+ assert_eq!(calls, 1);
+}
diff --git a/moli-leveldb-parser/src/tests/manifest.rs b/moli-leveldb-parser/src/tests/manifest.rs
new file mode 100644
index 000000000..597b3152e
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/manifest.rs
@@ -0,0 +1,267 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// VersionEditTest.EncodeDecode plus parser regressions for db/version_edit.cc.
+// See ../../TESTS.md, ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.
+
+use std::fs;
+
+use anyhow::Result;
+
+use super::support::*;
+use crate::{MAX_SEQUENCE, manifest::Manifest};
+
+fn read(edits: &[Vec<u8>]) -> Result<Manifest> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), edits)?;
+ Manifest::read(directory.path())
+}
+
+#[test]
+fn encode_decode() -> Result<()> {
+ const BIG: u64 = 1 << 50;
+ let mut records = vec![metadata(BIG + 100, BIG + 1000)];
+ let mut edit = Vec::new();
+ for i in 0..4 {
+ let parsed = read(&records)?;
+ assert_eq!(parsed.tables.len(), i as usize);
+ add_table(
+ &mut edit,
+ 3,
+ BIG + 300 + i,
+ BIG + 400 + i,
+ &internal_key(b"foo", BIG + 500 + i, 1),
+ &internal_key(b"zoo", BIG + 600 + i, 0),
+ );
+ delete_table(&mut edit, 4, BIG + 700 + i);
+ edit.push(5);
+ varint(i, &mut edit);
+ slice(&internal_key(b"x", BIG + 900 + i, 1), &mut edit);
+ records.truncate(1);
+ records.push(edit.clone());
+ }
+ let parsed = read(&records)?;
+ assert_eq!(parsed.log_number, BIG + 100);
+ assert_eq!(parsed.last_sequence, BIG + 1000);
+ for i in 0..4 {
+ assert_eq!(
+ parsed.tables.get(&(3, BIG + 300 + i)),
+ Some(&(BIG + 400 + i))
+ );
+ }
+ Ok(())
+}
+
+#[test]
+fn comparator_check() {
+ let mut edit = vec![1];
+ slice(b"foo", &mut edit);
+ assert!(format!("{:#}", read(&[metadata(3, 0), edit]).err().unwrap()).contains("comparator"));
+}
+
+#[test]
+fn required_metadata_can_span_records() -> Result<()> {
+ let parsed = read(&[vec![2, 3], vec![], vec![3, 4], vec![4, 5]])?;
+ assert_eq!((parsed.log_number, parsed.last_sequence), (3, 5));
+ Ok(())
+}
+
+#[test]
+fn missing_required_metadata() {
+ for mask in 0..7 {
+ let fields = [vec![2, 3], vec![3, 4], vec![4, 5]];
+ let edits: Vec<_> = fields
+ .into_iter()
+ .enumerate()
+ .filter_map(|(i, field)| ((mask & (1 << i)) != 0).then_some(field))
+ .collect();
+ assert!(read(&edits).is_err(), "mask {mask}");
+ }
+}
+
+#[test]
+fn later_metadata_and_previous_log_override_earlier_values() -> Result<()> {
+ let parsed = read(&[metadata(3, 5), vec![2, 8, 9, 3, 4, 12]])?;
+ assert_eq!(
+ (
+ parsed.log_number,
+ parsed.previous_log_number,
+ parsed.last_sequence
+ ),
+ (8, 3, 12)
+ );
+ Ok(())
+}
+
+#[test]
+fn unknown_tags() {
+ for tag in [0, 8, 10, 100, u32::MAX as u64, u64::MAX] {
+ let mut edit = Vec::new();
+ varint(tag, &mut edit);
+ assert!(read(&[metadata(3, 0), edit]).is_err(), "tag {tag}");
+ }
+}
+
+#[test]
+fn truncated_version_edit_fields() {
+ let mut fields = Vec::new();
+ let mut comparator = vec![1];
+ slice(b"leveldb.BytewiseComparator", &mut comparator);
+ fields.push(comparator);
+ for tag in [2, 3, 4, 9] {
+ let mut field = vec![tag];
+ varint(1 << 50, &mut field);
+ fields.push(field);
+ }
+ let mut compaction = vec![5, 1];
+ slice(&internal_key(b"x", 100, 1), &mut compaction);
+ fields.push(compaction);
+ let mut deleted = Vec::new();
+ delete_table(&mut deleted, 1, 1 << 50);
+ fields.push(deleted);
+ let mut added = Vec::new();
+ add_table(
+ &mut added,
+ 1,
+ 1 << 50,
+ 4096,
+ &internal_key(b"a", 1, 1),
+ &internal_key(b"z", 9, 1),
+ );
+ fields.push(added);
+ for field in fields {
+ for end in 1..field.len() {
+ assert!(
+ read(&[metadata(3, 0), field[..end].to_vec()]).is_err(),
+ "tag {}, end {end}",
+ field[0]
+ );
+ }
+ assert!(read(&[metadata(3, 0), field]).is_ok());
+ }
+}
+
+#[test]
+fn invalid_levels() {
+ for level in [7, u32::MAX as u64, u64::MAX] {
+ for tag in [5, 6, 7] {
+ let mut edit = vec![tag];
+ varint(level, &mut edit);
+ edit.extend_from_slice(&[0; 32]);
+ assert!(
+ format!("{:#}", read(&[metadata(3, 0), edit]).err().unwrap()).contains("LevelDB")
+ );
+ }
+ }
+}
+
+#[test]
+fn truncated_internal_keys() {
+ for length in 0..8 {
+ let mut compaction = vec![5, 1];
+ slice(&[0; 8][..length], &mut compaction);
+ assert!(read(&[metadata(3, 0), compaction]).is_err());
+ for short_smallest in [true, false] {
+ let good = internal_key(b"key", 1, 1);
+ let short = &[0; 8][..length];
+ let (smallest, largest) = if short_smallest {
+ (short, good.as_slice())
+ } else {
+ (good.as_slice(), short)
+ };
+ let mut edit = Vec::new();
+ add_table(&mut edit, 1, 4, 4096, smallest, largest);
+ assert!(read(&[metadata(3, 0), edit]).is_err());
+ }
+ }
+}
+
+#[test]
+fn deleted_files_and_level_moves() -> Result<()> {
+ let key = internal_key(b"key", 1, 1);
+ let mut first = metadata(3, 1);
+ add_table(&mut first, 0, 4, 100, &key, &key);
+ add_table(&mut first, 1, 5, 200, &key, &key);
+ let mut edit = Vec::new();
+ add_table(&mut edit, 2, 4, 100, &key, &key);
+ delete_table(&mut edit, 0, 4);
+ delete_table(&mut edit, 1, 5);
+ let parsed = read(&[first, edit])?;
+ assert_eq!(parsed.tables, [((2, 4), 100)].into());
+ Ok(())
+}
+
+#[test]
+fn one_edit_applies_deletions_before_additions() -> Result<()> {
+ let key = internal_key(b"key", 1, 1);
+ let mut edit = metadata(3, 1);
+ add_table(&mut edit, 1, 4, 100, &key, &key);
+ delete_table(&mut edit, 1, 4);
+ assert_eq!(read(&[edit])?.tables, [((1, 4), 100)].into());
+ Ok(())
+}
+
+#[test]
+fn sequence_overflow() {
+ assert!(read(&[metadata(3, MAX_SEQUENCE)]).is_ok());
+ assert!(read(&[metadata(3, MAX_SEQUENCE + 1)]).is_err());
+}
+
+#[test]
+fn current_must_name_one_manifest() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ for name in [
+ "",
+ "\n",
+ "MANIFEST-000001",
+ "MANIFEST-000001\n\n",
+ "../MANIFEST-000001\n",
+ "MANIFEST-../000001\n",
+ "MANIFEST-000001/child\n",
+ "MANIFEST-abc\n",
+ "/MANIFEST-000001\n",
+ ] {
+ fs::write(directory.path().join("CURRENT"), name)?;
+ assert!(Manifest::read(directory.path()).is_err(), "{name:?}");
+ }
+ Ok(())
+}
+
+#[test]
+fn current_selects_the_manifest_not_its_largest_number() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 5)])?;
+ fs::write(
+ directory.path().join("MANIFEST-999999"),
+ log_bytes(&[metadata(4, 100)]),
+ )?;
+ assert_eq!(Manifest::read(directory.path())?.last_sequence, 5);
+ Ok(())
+}
+
+#[test]
+fn incomplete_last_edit_is_ignored() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let first = metadata(3, 5);
+ let mut next = vec![4];
+ varint(1 << 50, &mut next);
+ let bytes = log_bytes(&[first, next]);
+ write_manifest(directory.path(), &[])?;
+ fs::write(
+ directory.path().join("MANIFEST-000001"),
+ &bytes[..bytes.len() - 1],
+ )?;
+ assert_eq!(Manifest::read(directory.path())?.last_sequence, 5);
+ Ok(())
+}
+
+#[test]
+fn complete_edit_checksum_failure_is_not_ignored() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 5)])?;
+ let path = directory.path().join("MANIFEST-000001");
+ let mut bytes = fs::read(&path)?;
+ bytes[0] ^= 1;
+ fs::write(path, bytes)?;
+ assert!(format!("{:#}", Manifest::read(directory.path()).err().unwrap()).contains("checksum"));
+ Ok(())
+}
diff --git a/moli-leveldb-parser/src/tests/mod.rs b/moli-leveldb-parser/src/tests/mod.rs
new file mode 100644
index 000000000..db3e9d5b8
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/mod.rs
@@ -0,0 +1,9 @@
+mod coding;
+mod layout;
+mod log;
+mod manifest;
+mod recovery;
+mod snapshot;
+mod support;
+mod table;
+mod write_batch;
diff --git a/moli-leveldb-parser/src/tests/recovery.rs b/moli-leveldb-parser/src/tests/recovery.rs
new file mode 100644
index 000000000..a72e0e2a7
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/recovery.rs
@@ -0,0 +1,509 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Read-only adaptations of db/recovery_test.cc, db/db_test.cc and
+// db/corruption_test.cc. See ../../TESTS.md, ../../LICENSE-LevelDB and
+// ../../AUTHORS-LevelDB for the exact scope and intentional differences.
+
+use std::{fs, path::Path};
+
+use anyhow::Result;
+
+use super::support::*;
+use crate::read_snapshot;
+
+fn files(directory: &Path) -> Result<Entries> {
+ fs::read_dir(directory)?
+ .map(|entry| {
+ let entry = entry?;
+ Ok((
+ entry.file_name().to_string_lossy().as_bytes().to_vec(),
+ fs::read(entry.path())?,
+ ))
+ })
+ .collect()
+}
+
+fn check(directory: &Path, expected: &Entries) -> Result<()> {
+ let before = files(directory)?;
+ assert_eq!(&read_snapshot(directory)?, expected);
+ assert_eq!(
+ files(directory)?,
+ before,
+ "recovery must not write, delete or create database files"
+ );
+ Ok(())
+}
+
+#[test]
+fn manifest_reused() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])]),
+ )?;
+ let expected = [(b"foo".to_vec(), b"bar".to_vec())].into();
+ check(directory.path(), &expected)?;
+ // A second read and an appended VersionEdit both preserve CURRENT and the
+ // original manifest, without requiring the engine's reuse_logs option.
+ let path = directory.path().join("MANIFEST-000001");
+ let mut bytes = fs::read(&path)?;
+ append_log(&mut bytes, &[4, 1]);
+ fs::write(path, bytes)?;
+ check(directory.path(), &expected)
+}
+
+#[test]
+fn large_manifest_compacted_read_without_compaction() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let path = directory.path().join("MANIFEST-000001");
+ let mut bytes = fs::read(&path)?;
+ bytes.resize(3 * 1048576, 0);
+ fs::write(path, bytes)?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])]),
+ )?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"bar".to_vec())].into(),
+ )
+}
+
+#[test]
+fn no_log_files() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ check(directory.path(), &Entries::new())
+}
+
+#[test]
+fn log_file_reuse() -> Result<()> {
+ for empty_log in [true, false] {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 1);
+ let bytes = if empty_log {
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[(internal_key(b"foo", 1, 1), b"bar".to_vec())],
+ )?;
+ vec![]
+ } else {
+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])])
+ };
+ write_manifest(directory.path(), &[edit])?;
+ fs::write(directory.path().join("000003.log"), bytes)?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"bar".to_vec())].into(),
+ )?;
+ }
+ Ok(())
+}
+
+#[test]
+fn multiple_mem_tables_recovered_in_memory() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let mut expected = Entries::new();
+ let mut records = Vec::new();
+ for i in 0..1000 {
+ let key = format!("{i:050}").into_bytes();
+ records.push(batch(i + 1, &[(&key, Some(&key))]));
+ expected.insert(key.clone(), key);
+ }
+ fs::write(directory.path().join("000003.log"), log_bytes(&records))?;
+ check(directory.path(), &expected)
+}
+
+#[test]
+fn multiple_log_files() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let writes: &[(u64, u64, &[u8], &[u8])] = &[
+ (3, 1, b"foo", b"bar"),
+ (4, 1000, b"hello", b"world"),
+ (5, 1001, b"hi", b"there"),
+ (6, 1002, b"foo", b"bar2"),
+ (2, 2000, b"hello", b"stale write"),
+ ];
+ for (number, sequence, key, value) in writes {
+ fs::write(
+ directory.path().join(format!("{number:06}.log")),
+ log_bytes(&[batch(*sequence, &[(key, Some(value))])]),
+ )?;
+ }
+ check(
+ directory.path(),
+ &[
+ (b"foo".to_vec(), b"bar2".to_vec()),
+ (b"hello".to_vec(), b"world".to_vec()),
+ (b"hi".to_vec(), b"there".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn manifest_missing() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ fs::remove_file(directory.path().join("MANIFEST-000001"))?;
+ let before = files(directory.path())?;
+ assert!(read_snapshot(directory.path()).is_err());
+ assert_eq!(files(directory.path())?, before);
+ Ok(())
+}
+
+#[test]
+fn get_level0_ordering() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 3);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 0,
+ 4,
+ &[
+ (internal_key(b"bar", 1, 1), b"b".to_vec()),
+ (internal_key(b"foo", 2, 1), b"v1".to_vec()),
+ ],
+ )?;
+ save_table(
+ directory.path(),
+ &mut edit,
+ 0,
+ 5,
+ &[(internal_key(b"foo", 3, 1), b"v2".to_vec())],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ check(
+ directory.path(),
+ &[
+ (b"bar".to_vec(), b"b".to_vec()),
+ (b"foo".to_vec(), b"v2".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn get_ordered_by_levels() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 2);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 2,
+ 4,
+ &[(internal_key(b"foo", 1, 1), b"v1".to_vec())],
+ )?;
+ save_table(
+ directory.path(),
+ &mut edit,
+ 0,
+ 5,
+ &[(internal_key(b"foo", 2, 1), b"v2".to_vec())],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"v2".to_vec())].into(),
+ )
+}
+
+#[test]
+fn recover() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let path = directory.path().join("000003.log");
+ let mut bytes = log_bytes(&[batch(1, &[(b"foo", Some(b"v1")), (b"baz", Some(b"v5"))])]);
+ fs::write(&path, &bytes)?;
+ check(
+ directory.path(),
+ &[
+ (b"foo".to_vec(), b"v1".to_vec()),
+ (b"baz".to_vec(), b"v5".to_vec()),
+ ]
+ .into(),
+ )?;
+ append_log(
+ &mut bytes,
+ &batch(3, &[(b"bar", Some(b"v2")), (b"foo", Some(b"v3"))]),
+ );
+ fs::write(&path, &bytes)?;
+ let mut expected: Entries = [
+ (b"foo".to_vec(), b"v3".to_vec()),
+ (b"baz".to_vec(), b"v5".to_vec()),
+ (b"bar".to_vec(), b"v2".to_vec()),
+ ]
+ .into();
+ check(directory.path(), &expected)?;
+ append_log(&mut bytes, &batch(5, &[(b"foo", Some(b"v4"))]));
+ fs::write(path, bytes)?;
+ expected.insert(b"foo".to_vec(), b"v4".to_vec());
+ check(directory.path(), &expected)
+}
+
+#[test]
+fn recovery_with_empty_log() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 2);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[
+ (internal_key(b"foo", 2, 1), b"v2".to_vec()),
+ (internal_key(b"foo", 1, 1), b"v1".to_vec()),
+ ],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ let path = directory.path().join("000003.log");
+ fs::write(&path, [])?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"v2".to_vec())].into(),
+ )?;
+ fs::write(path, log_bytes(&[batch(3, &[(b"foo", Some(b"v3"))])]))?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"v3".to_vec())].into(),
+ )
+}
+
+#[test]
+fn iter_multi_with_delete() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(
+ 1,
+ &[
+ (b"a", Some(b"va")),
+ (b"b", Some(b"vb")),
+ (b"c", Some(b"vc")),
+ (b"b", None),
+ ],
+ )]),
+ )?;
+ check(
+ directory.path(),
+ &[
+ (b"a".to_vec(), b"va".to_vec()),
+ (b"c".to_vec(), b"vc".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn iter_multi_with_delete_and_compaction() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 3);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[
+ (internal_key(b"a", 3, 1), b"va".to_vec()),
+ (internal_key(b"b", 1, 1), b"vb".to_vec()),
+ (internal_key(b"c", 2, 1), b"vc".to_vec()),
+ ],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(4, &[(b"b", None)])]),
+ )?;
+ check(
+ directory.path(),
+ &[
+ (b"a".to_vec(), b"va".to_vec()),
+ (b"c".to_vec(), b"vc".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn sequence_number_recovery() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 5);
+ let entries: Vec<_> = (1..=5)
+ .rev()
+ .map(|i| (internal_key(b"foo", i, 1), format!("v{i}").into_bytes()))
+ .collect();
+ save_table(directory.path(), &mut edit, 1, 4, &entries)?;
+ write_manifest(directory.path(), &[edit])?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"v5".to_vec())].into(),
+ )?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(6, &[(b"foo", Some(b"v6"))])]),
+ )?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"v6".to_vec())].into(),
+ )
+}
+
+#[test]
+fn corrupted_recovery_is_an_error_without_salvage() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let records: Vec<_> = (1..=100)
+ .map(|i| {
+ batch(
+ i,
+ &[(format!("{i:016}").as_bytes(), Some(&vec![b'x'; 1000]))],
+ )
+ })
+ .collect();
+ let mut bytes = log_bytes(&records);
+ bytes[19] ^= 1;
+ bytes[BLOCK_SIZE + 1000] ^= 1;
+ fs::write(directory.path().join("000003.log"), bytes)?;
+ let before = files(directory.path())?;
+ assert!(read_snapshot(directory.path()).is_err());
+ assert_eq!(files(directory.path())?, before);
+ Ok(())
+}
+
+#[test]
+fn sst_versions_newer_than_the_recovered_sequence_are_invisible() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 1);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[
+ (internal_key(b"foo", 5, 1), b"future".to_vec()),
+ (internal_key(b"foo", 1, 1), b"current".to_vec()),
+ ],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"current".to_vec())].into(),
+ )?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(5, &[(b"other", Some(b"value"))])]),
+ )?;
+ check(
+ directory.path(),
+ &[
+ (b"foo".to_vec(), b"future".to_vec()),
+ (b"other".to_vec(), b"value".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn table_tombstone_hides_an_older_wal_value() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 5);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[(internal_key(b"foo", 5, 0), vec![])],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(1, &[(b"foo", Some(b"old"))])]),
+ )?;
+ check(directory.path(), &Entries::new())
+}
+
+#[test]
+fn empty_batch_cannot_advance_the_snapshot_sequence() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 1);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 1,
+ 4,
+ &[
+ (internal_key(b"foo", 5, 1), b"future".to_vec()),
+ (internal_key(b"foo", 1, 1), b"current".to_vec()),
+ ],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(100, &[])]),
+ )?;
+ check(
+ directory.path(),
+ &[(b"foo".to_vec(), b"current".to_vec())].into(),
+ )
+}
+
+#[test]
+fn malformed_batch_does_not_return_a_partial_snapshot() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ let mut malformed = batch(2, &[(b"foo", Some(b"updated")), (b"bar", None)]);
+ malformed.pop();
+ fs::write(
+ directory.path().join("000003.log"),
+ log_bytes(&[batch(1, &[(b"foo", Some(b"old"))]), malformed]),
+ )?;
+ assert!(read_snapshot(directory.path()).is_err());
+ Ok(())
+}
+
+#[test]
+fn duplicate_log_number_is_rejected() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ write_manifest(directory.path(), &[metadata(3, 0)])?;
+ for name in ["000003.log", "3.log"] {
+ fs::write(directory.path().join(name), [])?;
+ }
+ assert!(
+ read_snapshot(directory.path())
+ .unwrap_err()
+ .to_string()
+ .contains("duplicate")
+ );
+ Ok(())
+}
+
+#[test]
+fn native_chromium_tables_and_fragmented_wal() -> Result<()> {
+ let directory = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/chromium/recovery");
+ let mut expected = Entries::new();
+ for index in 0..200 {
+ if index % 7 == 0 {
+ continue;
+ }
+ let key = format!("key-{index:04}").into_bytes();
+ let value = if index % 5 == 0 {
+ b"new".to_vec()
+ } else {
+ format!("value-{index}{}", "x".repeat(300)).into_bytes()
+ };
+ expected.insert(key, value);
+ }
+ expected.insert(vec![], vec![]);
+ expected.insert(vec![0, 0xff, b'k'], vec![0xff, 0]);
+ expected.insert(b"large".to_vec(), vec![b'y'; 120_000]);
+ assert_eq!(expected.len(), 174);
+ check(&directory, &expected)
+}
diff --git a/moli-leveldb-parser/src/tests/snapshot.rs b/moli-leveldb-parser/src/tests/snapshot.rs
new file mode 100644
index 000000000..f182c70e1
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/snapshot.rs
@@ -0,0 +1,389 @@
+use std::{
+ collections::BTreeMap,
+ fs,
+ path::{Path, PathBuf},
+ time::SystemTime,
+};
+
+use anyhow::Result;
+use rusty_leveldb::{DB, LdbIterator, Options};
+
+use super::support::{batch, log_bytes, metadata, write_manifest};
+
+use crate::{
+ Manifest,
+ coding::{Decoder, masked_crc},
+ log::read_records,
+ read_snapshot,
+};
+
+fn open_writer(path: &Path, compression: u8) -> Result<DB> {
+ Ok(DB::open(
+ path,
+ Options {
+ compressor: compression,
+ block_size: 128,
+ block_restart_interval: 2,
+ ..Options::default()
+ },
+ )?)
+}
+
+fn engine_entries(database: &mut DB) -> Result<BTreeMap<Vec<u8>, Vec<u8>>> {
+ let mut iterator = database.new_iter()?;
+ let mut entries = BTreeMap::new();
+ while let Some((key, value)) = iterator.next() {
+ entries.insert(key, value);
+ }
+ Ok(entries)
+}
+
+fn image(path: &Path) -> Result<BTreeMap<PathBuf, (Vec<u8>, SystemTime)>> {
+ fs::read_dir(path)?
+ .map(|entry| {
+ let entry = entry?;
+ Ok((
+ entry.file_name().into(),
+ (fs::read(entry.path())?, entry.metadata()?.modified()?),
+ ))
+ })
+ .collect()
+}
+
+#[test]
+fn matches_engine_for_tables_wal_versions_and_tombstones_without_writes() -> Result<()> {
+ for compression in [0, 1] {
+ let directory = tempfile::tempdir()?;
+ let mut database = open_writer(directory.path(), compression)?;
+ let mut expected = BTreeMap::new();
+ for index in 0..200 {
+ let key = format!("key-{index:04}").into_bytes();
+ let value = format!("value-{index:04}-{}世界", "x".repeat(300)).into_bytes();
+ database.put(&key, &value)?;
+ expected.insert(key, value);
+ }
+ let large = vec![b'x'; 100_000];
+ database.put(b"large", &large)?;
+ database.compact_range(b"", b"\xff")?;
+ assert!(!Manifest::read(directory.path())?.tables.is_empty());
+ for index in 0..200 {
+ let key = format!("key-{index:04}").into_bytes();
+ if index % 3 == 0 {
+ database.delete(&key)?;
+ expected.remove(&key);
+ } else if index % 5 == 0 {
+ database.put(&key, b"new-value")?;
+ expected.insert(key, b"new-value".to_vec());
+ }
+ }
+ let large = vec![b'y'; 110_000];
+ database.put(b"large", &large)?;
+ expected.insert(b"large".to_vec(), large);
+ database.put(b"", b"empty-key")?;
+ expected.insert(Vec::new(), b"empty-key".to_vec());
+ database.flush()?;
+ assert_eq!(engine_entries(&mut database)?, expected);
+
+ // The writer still holds LOCK, and its WAL has not been recovered into
+ // an SST. Reading must not acquire a lock or modify any snapshot file.
+ let before = image(directory.path())?;
+ assert_eq!(read_snapshot(directory.path())?, expected);
+ assert_eq!(image(directory.path())?, before);
+ }
+ Ok(())
+}
+
+#[test]
+fn ignores_obsolete_tables_logs_and_manifests_after_compaction() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut database = open_writer(directory.path(), 1)?;
+ database.put(b"deleted", b"must-not-resurrect")?;
+ database.flush()?;
+ let old_log_path = fs::read_dir(directory.path())?
+ .map(|entry| entry.map(|entry| entry.path()))
+ .collect::<std::io::Result<Vec<_>>>()?
+ .into_iter()
+ .find(|path| path.extension().is_some_and(|extension| extension == "log"))
+ .unwrap();
+ let old_log_number: u64 = old_log_path
+ .file_stem()
+ .unwrap()
+ .to_str()
+ .unwrap()
+ .parse()?;
+ let old_log = fs::read(&old_log_path)?;
+ database.compact_range(b"", b"\xff")?;
+ let tables = Manifest::read(directory.path())?.tables;
+ let old_tables = tables
+ .keys()
+ .map(|(_, number)| {
+ let name = format!("{number:06}.ldb");
+ Ok((name.clone(), fs::read(directory.path().join(name))?))
+ })
+ .collect::<Result<Vec<_>>>()?;
+ database.delete(b"deleted")?;
+ database.put(b"kept", b"current")?;
+ database.compact_range(b"", b"\xff")?;
+ let expected = engine_entries(&mut database)?;
+ database.close()?;
+ drop(database);
+
+ let live = Manifest::read(directory.path())?;
+ assert!(old_log_number < live.log_number);
+ assert_ne!(old_log_number, live.previous_log_number);
+ for ((_, number), _) in tables {
+ assert!(
+ !live
+ .tables
+ .keys()
+ .any(|(_, live_number)| *live_number == number)
+ );
+ }
+ for (name, bytes) in old_tables {
+ fs::write(directory.path().join(name), bytes)?;
+ }
+ fs::write(old_log_path, old_log)?;
+ fs::write(
+ directory.path().join("999999.ldb"),
+ b"unreferenced incomplete table",
+ )?;
+ fs::write(
+ directory.path().join("MANIFEST-999999"),
+ b"unselected manifest",
+ )?;
+ assert_eq!(read_snapshot(directory.path())?, expected);
+ assert!(!expected.contains_key(b"deleted".as_slice()));
+ Ok(())
+}
+
+#[test]
+fn reads_legacy_sst_extension_and_rejects_corrupt_or_missing_live_tables() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut database = open_writer(directory.path(), 1)?;
+ database.put(b"key", b"value")?;
+ database.compact_range(b"", b"\xff")?;
+ database.close()?;
+ drop(database);
+ let manifest = Manifest::read(directory.path())?;
+ let (_, number) = manifest.tables.keys().next().unwrap();
+ let original = directory.path().join(format!("{number:06}.ldb"));
+ let legacy = original.with_extension("sst");
+ fs::rename(&original, &legacy)?;
+ assert_eq!(
+ read_snapshot(directory.path())?.get(b"key".as_slice()),
+ Some(&b"value".to_vec())
+ );
+ let mut bytes = fs::read(&legacy)?;
+ bytes[0] ^= 1;
+ fs::write(&legacy, &bytes)?;
+ let error = read_snapshot(directory.path()).unwrap_err();
+ assert!(format!("{error:#}").contains("checksum"));
+ fs::remove_file(legacy)?;
+ assert!(read_snapshot(directory.path()).is_err());
+ Ok(())
+}
+
+#[test]
+fn rejects_malformed_table_blocks_even_with_valid_checksums() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut database = open_writer(directory.path(), 0)?;
+ database.put(b"key", b"value")?;
+ database.compact_range(b"", b"\xff")?;
+ database.close()?;
+ drop(database);
+ let manifest = Manifest::read(directory.path())?;
+ let (_, number) = manifest.tables.keys().next().unwrap();
+ let path = directory.path().join(format!("{number:06}.ldb"));
+ let original = fs::read(&path)?;
+
+ let mut footer = Decoder::new(&original[original.len() - 48..original.len() - 8]);
+ footer.varint()?;
+ footer.varint()?;
+ let index_offset = footer.varint()? as usize;
+ let index_size = footer.varint()? as usize;
+ assert_eq!(original[index_offset + index_size], 0);
+ let mut index = Decoder::new(&original[index_offset..index_offset + index_size]);
+ assert_eq!(index.varint32()?, 0);
+ let key_length = index.varint32()? as usize;
+ let value_length = index.varint32()? as usize;
+ index.take(key_length)?;
+ let mut handle = Decoder::new(index.take(value_length)?);
+ let offset = handle.varint()? as usize;
+ let size = handle.varint()? as usize;
+
+ for case in 0..7 {
+ let mut bytes = original.clone();
+ match case {
+ 0 => bytes[offset] = 1, // First entry cannot share a key prefix.
+ 1 => bytes[offset + 1] = 127, // Key extends outside the block.
+ 2 => bytes[offset + 2] = 127, // Value extends outside the block.
+ 3 => bytes[offset + size - 4..offset + size].fill(0xff), // Restart count overflow.
+ 4 => bytes[offset + size - 8] = 1, // First restart must start at zero.
+ 5 => bytes[offset + size] = 42, // Unknown compression type.
+ 6 => {
+ bytes[offset..offset + 5].copy_from_slice(&[0xff, 0xff, 0xff, 0xff, 0x0f]);
+ bytes[offset + size] = 1; // Snappy header claims a 4 GiB block.
+ }
+ _ => unreachable!(),
+ }
+ let checksum = masked_crc(&bytes[offset..offset + size + 1]);
+ bytes[offset + size + 1..offset + size + 5].copy_from_slice(&checksum.to_le_bytes());
+ fs::write(&path, bytes)?;
+ assert!(read_snapshot(directory.path()).is_err(), "case {case}");
+ }
+ fs::write(path, original)?;
+ assert_eq!(
+ read_snapshot(directory.path())?.get(b"key".as_slice()),
+ Some(&b"value".to_vec())
+ );
+ Ok(())
+}
+
+fn empty_snapshot(path: &Path) -> Result<PathBuf> {
+ write_manifest(path, &[metadata(3, 0)])?;
+ Ok(path.join("000003.log"))
+}
+
+fn put_batch(sequence: u64, key: &[u8], value: &[u8]) -> Vec<u8> {
+ batch(sequence, &[(key, Some(value))])
+}
+
+#[test]
+fn replays_new_and_previous_wals_and_uses_sequence_order() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let log = empty_snapshot(directory.path())?;
+ // Include the legacy previous WAL as well as a newer, unregistered WAL.
+ let manifest = directory.path().join("MANIFEST-000001");
+ let mut bytes = fs::read(&manifest)?;
+ bytes.extend_from_slice(&log_bytes(&[vec![9, 2]]));
+ fs::write(&manifest, bytes)?;
+ fs::write(
+ directory.path().join("000002.log"),
+ log_bytes(&[put_batch(1, b"previous", b"value")]),
+ )?;
+ fs::write(&log, log_bytes(&[put_batch(8, b"key", b"newer")]))?;
+ fs::write(
+ directory.path().join("000004.log"),
+ log_bytes(&[put_batch(3, b"key", b"older")]),
+ )?;
+ let values = read_snapshot(directory.path())?;
+ assert_eq!(values.get(b"key".as_slice()), Some(&b"newer".to_vec()));
+ assert_eq!(values.get(b"previous".as_slice()), Some(&b"value".to_vec()));
+ Ok(())
+}
+
+#[test]
+fn ignores_only_incomplete_wal_tail_and_rejects_corrupt_complete_records() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let log = empty_snapshot(directory.path())?;
+ let complete = put_batch(1, b"committed", b"value");
+ let unfinished = put_batch(2, b"unfinished", &vec![b'x'; 100_000]);
+ let valid = log_bytes(&[complete, unfinished]);
+ for missing in [1, 1000, 40_000] {
+ fs::write(&log, &valid[..valid.len() - missing])?;
+ let values = read_snapshot(directory.path())?;
+ assert_eq!(values.len(), 1);
+ assert!(values.contains_key(b"committed".as_slice()));
+ }
+ let mut corrupt = valid;
+ corrupt[8] ^= 1;
+ fs::write(&log, corrupt)?;
+ assert!(format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("checksum"));
+ Ok(())
+}
+
+#[test]
+fn handles_log_block_padding_and_empty_first_fragments() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let path = directory.path().join("log");
+ for first_length in [32754, 32755, 32760, 32761] {
+ let records = vec![
+ vec![b'a'; first_length],
+ vec![b'b'; 70_000],
+ b"end".to_vec(),
+ ];
+ fs::write(&path, log_bytes(&records))?;
+ let mut decoded = Vec::new();
+ read_records(&path, |record| {
+ decoded.push(record.to_vec());
+ Ok(())
+ })?;
+ assert_eq!(decoded, records);
+ }
+ Ok(())
+}
+
+#[test]
+fn rejects_malformed_batches_and_current_paths_without_panicking() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let log = empty_snapshot(directory.path())?;
+ let valid = put_batch(1, b"key", b"value");
+ let mut wrong_count = valid.clone();
+ wrong_count[8] = 2;
+ let mut unknown_type = valid.clone();
+ unknown_type[12] = 2;
+ let mut length_overflow = valid.clone();
+ length_overflow.splice(13..14, [0xff; 10]);
+ let mut extra_bytes = valid.clone();
+ extra_bytes.push(0);
+ for malformed in [
+ valid[..11].to_vec(),
+ wrong_count,
+ unknown_type,
+ length_overflow,
+ extra_bytes,
+ put_batch(u64::MAX, b"key", b"value"),
+ ] {
+ fs::write(&log, log_bytes(&[malformed]))?;
+ assert!(read_snapshot(directory.path()).is_err());
+ }
+ fs::write(&log, [])?;
+ for current in [
+ "../MANIFEST-000001\n",
+ "MANIFEST-../000001\n",
+ "MANIFEST-000001",
+ "\n",
+ ] {
+ fs::write(directory.path().join("CURRENT"), current)?;
+ assert!(read_snapshot(directory.path()).is_err());
+ }
+ Ok(())
+}
+
+#[cfg(unix)]
+#[test]
+fn reads_a_directory_with_no_write_permissions() -> Result<()> {
+ use std::os::unix::fs::PermissionsExt;
+
+ struct RestorePermissions(Vec<(PathBuf, fs::Permissions)>);
+ impl Drop for RestorePermissions {
+ fn drop(&mut self) {
+ for (path, permissions) in &self.0 {
+ fs::set_permissions(path, permissions.clone()).unwrap();
+ }
+ }
+ }
+
+ let directory = tempfile::tempdir()?;
+ let log = empty_snapshot(directory.path())?;
+ fs::write(log, log_bytes(&[put_batch(1, b"key", b"value")]))?;
+ let before = image(directory.path())?;
+ let mut permissions = RestorePermissions(vec![(
+ directory.path().to_owned(),
+ fs::metadata(directory.path())?.permissions(),
+ )]);
+ for entry in fs::read_dir(directory.path())? {
+ let entry = entry?;
+ permissions
+ .0
+ .push((entry.path(), entry.metadata()?.permissions()));
+ fs::set_permissions(entry.path(), fs::Permissions::from_mode(0o444))?;
+ }
+ fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o555))?;
+ assert_eq!(
+ read_snapshot(directory.path())?.get(b"key".as_slice()),
+ Some(&b"value".to_vec())
+ );
+ assert_eq!(image(directory.path())?, before);
+ Ok(())
+}
diff --git a/moli-leveldb-parser/src/tests/support.rs b/moli-leveldb-parser/src/tests/support.rs
new file mode 100644
index 000000000..258a813dd
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/support.rs
@@ -0,0 +1,265 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Test encoders and Random adapted from LevelDB; see ../../LICENSE-LevelDB,
+// ../../AUTHORS-LevelDB and ../../TESTS.md for the pinned sources and scope.
+
+use std::{collections::BTreeMap, fs, path::Path};
+
+use anyhow::Result;
+
+pub const BLOCK_SIZE: usize = 32768;
+pub const HEADER_SIZE: usize = 7;
+pub type Entries = BTreeMap<Vec<u8>, Vec<u8>>;
+
+// Fixture construction deliberately does not call the production Decoder or
+// masked_crc. The CRC wrapper is checked against upstream's published vectors.
+pub fn crc(bytes: &[u8]) -> u32 {
+ let value = crc32c::crc32c(bytes);
+ value.rotate_right(15).wrapping_add(0xa282_ead8)
+}
+
+pub fn varint(mut value: u64, output: &mut Vec<u8>) {
+ while value >= 128 {
+ output.push((value as u8) | 128);
+ value >>= 7;
+ }
+ output.push(value as u8);
+}
+
+pub fn slice(bytes: &[u8], output: &mut Vec<u8>) {
+ varint(bytes.len() as u64, output);
+ output.extend_from_slice(bytes);
+}
+
+pub fn internal_key(key: &[u8], sequence: u64, kind: u8) -> Vec<u8> {
+ let mut result = key.to_vec();
+ result.extend_from_slice(&((sequence << 8) | u64::from(kind)).to_le_bytes());
+ result
+}
+
+pub fn batch(sequence: u64, changes: &[(&[u8], Option<&[u8]>)]) -> Vec<u8> {
+ let mut result = sequence.to_le_bytes().to_vec();
+ result.extend_from_slice(&(changes.len() as u32).to_le_bytes());
+ for (key, value) in changes {
+ result.push(u8::from(value.is_some()));
+ slice(key, &mut result);
+ if let Some(value) = value {
+ slice(value, &mut result);
+ }
+ }
+ result
+}
+
+pub fn physical_record(kind: u8, data: &[u8]) -> Vec<u8> {
+ let length = u16::try_from(data.len()).unwrap();
+ let mut checked = vec![kind];
+ checked.extend_from_slice(data);
+ let mut result = crc(&checked).to_le_bytes().to_vec();
+ result.extend_from_slice(&length.to_le_bytes());
+ result.extend_from_slice(&checked);
+ result
+}
+
+pub fn append_log(output: &mut Vec<u8>, record: &[u8]) {
+ let mut offset = 0;
+ let mut first = true;
+ loop {
+ let leftover = BLOCK_SIZE - output.len() % BLOCK_SIZE;
+ if leftover < HEADER_SIZE {
+ output.resize(output.len() + leftover, 0);
+ }
+ let length =
+ (record.len() - offset).min(BLOCK_SIZE - output.len() % BLOCK_SIZE - HEADER_SIZE);
+ let last = offset + length == record.len();
+ let kind = match (first, last) {
+ (true, true) => 1,
+ (true, false) => 2,
+ (false, false) => 3,
+ (false, true) => 4,
+ };
+ output.extend_from_slice(&physical_record(kind, &record[offset..offset + length]));
+ offset += length;
+ first = false;
+ if last {
+ break;
+ }
+ }
+}
+
+pub fn log_bytes(records: &[Vec<u8>]) -> Vec<u8> {
+ let mut result = Vec::new();
+ for record in records {
+ append_log(&mut result, record);
+ }
+ result
+}
+
+pub fn fix_log_checksum(bytes: &mut [u8], offset: usize) {
+ let length = u16::from_le_bytes(bytes[offset + 4..offset + 6].try_into().unwrap()) as usize;
+ let checksum = crc(&bytes[offset + 6..offset + HEADER_SIZE + length]);
+ bytes[offset..offset + 4].copy_from_slice(&checksum.to_le_bytes());
+}
+
+pub fn metadata(log_number: u64, sequence: u64) -> Vec<u8> {
+ let mut result = vec![1];
+ slice(b"leveldb.BytewiseComparator", &mut result);
+ for (tag, value) in [(2, log_number), (3, 100), (4, sequence)] {
+ result.push(tag);
+ varint(value, &mut result);
+ }
+ result
+}
+
+pub fn write_manifest(directory: &Path, records: &[Vec<u8>]) -> Result<()> {
+ fs::write(directory.join("CURRENT"), b"MANIFEST-000001\n")?;
+ fs::write(directory.join("MANIFEST-000001"), log_bytes(records))?;
+ Ok(())
+}
+
+pub fn add_table(
+ edit: &mut Vec<u8>,
+ level: u64,
+ number: u64,
+ size: u64,
+ smallest: &[u8],
+ largest: &[u8],
+) {
+ for value in [7, level, number, size] {
+ varint(value, edit);
+ }
+ slice(smallest, edit);
+ slice(largest, edit);
+}
+
+pub fn delete_table(edit: &mut Vec<u8>, level: u64, number: u64) {
+ for value in [6, level, number] {
+ varint(value, edit);
+ }
+}
+
+// Test-only BlockBuilder/TableBuilder equivalents. They also allow constructing
+// states that normal writers cannot emit, to exercise corruption validation.
+pub fn block(entries: &[(Vec<u8>, Vec<u8>)], restart_interval: usize) -> Vec<u8> {
+ assert!(restart_interval > 0);
+ let mut output = Vec::new();
+ let mut restarts = vec![0u32];
+ let mut previous: &[u8] = b"";
+ for (index, (key, value)) in entries.iter().enumerate() {
+ let shared = if index % restart_interval == 0 {
+ if index != 0 {
+ restarts.push(output.len() as u32);
+ }
+ 0
+ } else {
+ previous.iter().zip(key).take_while(|(a, b)| a == b).count()
+ };
+ for length in [shared, key.len() - shared, value.len()] {
+ varint(length as u64, &mut output);
+ }
+ output.extend_from_slice(&key[shared..]);
+ output.extend_from_slice(value);
+ previous = key;
+ }
+ for offset in &restarts {
+ output.extend_from_slice(&offset.to_le_bytes());
+ }
+ output.extend_from_slice(&(restarts.len() as u32).to_le_bytes());
+ output
+}
+
+fn append_block(output: &mut Vec<u8>, data: &[u8], compression: u8) -> Vec<u8> {
+ let mut encoded = match compression {
+ 0 => data.to_vec(),
+ 1 => snap::raw::Encoder::new().compress_vec(data).unwrap(),
+ _ => panic!("unsupported fixture compression"),
+ };
+ let mut handle = Vec::new();
+ varint(output.len() as u64, &mut handle);
+ varint(encoded.len() as u64, &mut handle);
+ encoded.push(compression);
+ let checksum = crc(&encoded);
+ output.extend_from_slice(&encoded);
+ output.extend_from_slice(&checksum.to_le_bytes());
+ handle
+}
+
+pub fn table(
+ entries: &[(Vec<u8>, Vec<u8>)],
+ restart_interval: usize,
+ entries_per_block: usize,
+ compression: u8,
+) -> Vec<u8> {
+ let mut output = Vec::new();
+ let mut index = Vec::new();
+ for chunk in entries.chunks(entries_per_block) {
+ let handle = append_block(&mut output, &block(chunk, restart_interval), compression);
+ index.push((chunk.last().unwrap().0.clone(), handle));
+ }
+ let metaindex = append_block(&mut output, &block(&[], 1), 0);
+ let index = append_block(&mut output, &block(&index, 1), compression);
+ let mut footer = metaindex;
+ footer.extend_from_slice(&index);
+ footer.resize(40, 0);
+ footer.extend_from_slice(&0xdb47_7524_8b80_fb57_u64.to_le_bytes());
+ output.extend_from_slice(&footer);
+ output
+}
+
+pub fn save_table(
+ directory: &Path,
+ edit: &mut Vec<u8>,
+ level: u64,
+ number: u64,
+ entries: &[(Vec<u8>, Vec<u8>)],
+) -> Result<()> {
+ let bytes = table(entries, 2, 2, 0);
+ fs::write(directory.join(format!("{number:06}.ldb")), &bytes)?;
+ add_table(
+ edit,
+ level,
+ number,
+ bytes.len() as u64,
+ &entries.first().unwrap().0,
+ &entries.last().unwrap().0,
+ );
+ Ok(())
+}
+
+// LevelDB util/random.h's deterministic Park-Miller generator. Fixed seeds make
+// the upstream stress cases reproducible without adding a rand dependency.
+pub struct Random(u32);
+
+impl Random {
+ pub fn new(seed: u32) -> Self {
+ let seed = seed & 0x7fff_ffff;
+ Self(if seed == 0 || seed == 0x7fff_ffff {
+ 1
+ } else {
+ seed
+ })
+ }
+
+ pub fn uniform(&mut self, limit: u32) -> u32 {
+ self.0 = (u64::from(self.0) * 16807 % 0x7fff_ffff) as u32;
+ self.0 % limit
+ }
+
+ pub fn skewed(&mut self, max_log: u32) -> usize {
+ let bits = self.uniform(max_log + 1);
+ self.uniform(1 << bits) as usize
+ }
+
+ pub fn key(&mut self, length: usize) -> Vec<u8> {
+ const CHARS: &[u8] = &[0, 1, b'a', b'b', b'c', b'd', b'e', 0xfd, 0xfe, 0xff];
+ (0..length)
+ .map(|_| CHARS[self.uniform(CHARS.len() as u32) as usize])
+ .collect()
+ }
+
+ pub fn string(&mut self, length: usize) -> Vec<u8> {
+ (0..length).map(|_| b' ' + self.uniform(95) as u8).collect()
+ }
+}
+
+pub fn big_string(partial: &[u8], length: usize) -> Vec<u8> {
+ partial.iter().copied().cycle().take(length).collect()
+}
diff --git a/moli-leveldb-parser/src/tests/table.rs b/moli-leveldb-parser/src/tests/table.rs
new file mode 100644
index 000000000..d4a8cfb72
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/table.rs
@@ -0,0 +1,384 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Ports of table/table_test.cc and db/dbformat_test.cc, plus corruption cases.
+// See ../../TESTS.md, ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.
+
+use std::fs;
+
+use anyhow::Result;
+
+use super::support::*;
+use crate::{
+ MAX_SEQUENCE,
+ coding::Decoder,
+ read_snapshot,
+ table::{read_entries, read_table},
+};
+
+fn scan(bytes: &[u8]) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
+ let directory = tempfile::tempdir()?;
+ let path = directory.path().join("table.ldb");
+ fs::write(&path, bytes)?;
+ let mut result = Vec::new();
+ read_table(&path, bytes.len() as u64, |key, value| {
+ result.push((key.to_vec(), value.to_vec()));
+ Ok(())
+ })?;
+ Ok(result)
+}
+
+fn check(entries: &Entries) -> Result<()> {
+ let expected: Vec<_> = entries
+ .iter()
+ .map(|(k, v)| (k.clone(), v.clone()))
+ .collect();
+ for restart_interval in [1, 16, 1024] {
+ let mut actual = Vec::new();
+ read_entries(&block(&expected, restart_interval), |k, v| {
+ actual.push((k.to_vec(), v.to_vec()));
+ Ok(())
+ })?;
+ assert_eq!(actual, expected, "restart interval {restart_interval}");
+ for compression in [0, 1] {
+ let bytes = table(&expected, restart_interval, 13, compression);
+ assert_eq!(
+ scan(&bytes)?,
+ expected,
+ "restart interval {restart_interval}, compression {compression}"
+ );
+ }
+ }
+ Ok(())
+}
+
+#[test]
+fn empty() -> Result<()> {
+ check(&Entries::new())
+}
+
+#[test]
+fn zero_restart_points_in_block() -> Result<()> {
+ // Upstream explicitly accepts this four-byte Java LevelDB empty block.
+ read_entries(&[0; 4], |_, _| panic!("zero-restart block must be empty"))
+}
+
+#[test]
+fn simple_empty_key() -> Result<()> {
+ check(&[(vec![], b"v".to_vec())].into())
+}
+
+#[test]
+fn simple_single() -> Result<()> {
+ check(&[(b"abc".to_vec(), b"v".to_vec())].into())
+}
+
+#[test]
+fn simple_multi() -> Result<()> {
+ check(
+ &[
+ (b"abc".to_vec(), b"v".to_vec()),
+ (b"abcd".to_vec(), b"v".to_vec()),
+ (b"ac".to_vec(), b"v2".to_vec()),
+ ]
+ .into(),
+ )
+}
+
+#[test]
+fn simple_special_key() -> Result<()> {
+ check(&[(vec![0xff, 0xff], b"v3".to_vec())].into())
+}
+
+#[test]
+fn randomized() -> Result<()> {
+ // Same entry-count schedule, key alphabet and skewed lengths as Harness.
+ let mut random = Random::new(306);
+ for count in (0..50).chain((50..2000).step_by(200)) {
+ let mut entries = Entries::new();
+ for _ in 0..count {
+ let key_length = random.skewed(4);
+ let key = random.key(key_length);
+ let value_length = random.skewed(5);
+ entries.insert(key, random.string(value_length));
+ }
+ check(&entries)?;
+ }
+ Ok(())
+}
+
+#[test]
+fn randomized_long_db() -> Result<()> {
+ // Generated by Chromium's pinned C++ engine. The Rust fixture engine panics
+ // in its shortest-separator routine for empty keys during this workload.
+ let directory = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("tests/fixtures/chromium/randomized-long-db");
+ let mut random = Random::new(301);
+ let mut expected = Entries::new();
+ for _ in 0..100_000 {
+ let key_length = random.skewed(4);
+ let key = random.key(key_length);
+ let value_length = random.skewed(5);
+ let value = random.string(value_length);
+ expected.insert(key, value);
+ }
+ assert!(
+ !crate::manifest::Manifest::read(&directory)?
+ .tables
+ .is_empty()
+ );
+ assert_eq!(read_snapshot(&directory)?, expected);
+ Ok(())
+}
+
+#[test]
+fn approximate_offset_of_plain_read_contents() -> Result<()> {
+ // Keep upstream's large-value/block-boundary workload; this parser exposes
+ // a full scan, so validate contents instead of a writer's offset estimate.
+ let entries = vec![
+ (b"k01".to_vec(), b"hello".to_vec()),
+ (b"k02".to_vec(), b"hello2".to_vec()),
+ (b"k03".to_vec(), vec![b'x'; 10_000]),
+ (b"k04".to_vec(), vec![b'x'; 200_000]),
+ (b"k05".to_vec(), vec![b'x'; 300_000]),
+ (b"k06".to_vec(), b"hello3".to_vec()),
+ (b"k07".to_vec(), vec![b'x'; 100_000]),
+ ];
+ assert_eq!(scan(&table(&entries, 16, 2, 0))?, entries);
+ Ok(())
+}
+
+#[test]
+fn approximate_offset_of_compressed_read_contents() -> Result<()> {
+ let mut random = Random::new(301);
+ let first = big_string(&random.string(2500), 10_000);
+ let second = big_string(&random.string(2500), 10_000);
+ let entries = vec![
+ (b"k01".to_vec(), b"hello".to_vec()),
+ (b"k02".to_vec(), first),
+ (b"k03".to_vec(), b"hello3".to_vec()),
+ (b"k04".to_vec(), second),
+ ];
+ let compressed = table(&entries, 16, 2, 1);
+ assert!(compressed.len() < 10_000);
+ assert_eq!(scan(&compressed)?, entries);
+ Ok(())
+}
+
+#[test]
+fn internal_key_encode_decode() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let sequences = [
+ 1,
+ 2,
+ 3,
+ (1 << 8) - 1,
+ 1 << 8,
+ (1 << 8) + 1,
+ (1 << 16) - 1,
+ 1 << 16,
+ (1 << 16) + 1,
+ (1 << 32) - 1,
+ 1 << 32,
+ (1 << 32) + 1,
+ MAX_SEQUENCE,
+ ];
+ for sequence in sequences {
+ let mut edit = metadata(3, sequence);
+ let entries: Vec<_> = [b"".as_slice(), b"k", b"hello", b"longggggggggggggggggggggg"]
+ .into_iter()
+ .map(|key| (internal_key(key, sequence, 1), b"value".to_vec()))
+ .collect();
+ // The data reader consumes the on-disk order; public output is bytewise.
+ let mut entries = entries;
+ entries.sort_by(|a, b| a.0[..a.0.len() - 8].cmp(&b.0[..b.0.len() - 8]));
+ save_table(directory.path(), &mut edit, 0, 4, &entries)?;
+ write_manifest(directory.path(), &[edit])?;
+ let expected: Entries = [b"".as_slice(), b"k", b"hello", b"longggggggggggggggggggggg"]
+ .into_iter()
+ .map(|key| (key.to_vec(), b"value".to_vec()))
+ .collect();
+ assert_eq!(read_snapshot(directory.path())?, expected);
+ let mut edit = metadata(3, sequence);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 0,
+ 4,
+ &[(internal_key(b"hello", sequence, 0), vec![])],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ assert!(read_snapshot(directory.path())?.is_empty());
+ }
+ Ok(())
+}
+
+#[test]
+fn internal_key_decode_from_empty() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ for length in 0..8 {
+ let bytes = table(&[(vec![b'k'; length], b"value".to_vec())], 1, 1, 0);
+ fs::write(directory.path().join("000004.ldb"), &bytes)?;
+ let mut edit = metadata(3, 1);
+ let key = internal_key(b"key", 1, 1);
+ add_table(&mut edit, 0, 4, bytes.len() as u64, &key, &key);
+ write_manifest(directory.path(), &[edit])?;
+ assert!(
+ format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("internal key")
+ );
+ }
+ Ok(())
+}
+
+#[test]
+fn unknown_internal_value_type() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let mut edit = metadata(3, 1);
+ save_table(
+ directory.path(),
+ &mut edit,
+ 0,
+ 4,
+ &[(internal_key(b"key", 1, 2), vec![])],
+ )?;
+ write_manifest(directory.path(), &[edit])?;
+ assert!(format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("value type"));
+ Ok(())
+}
+
+#[test]
+fn truncated_footer_and_bad_magic() {
+ for length in 0..48 {
+ assert!(scan(&[0; 48][..length]).is_err());
+ }
+ let mut bytes = table(&[], 1, 1, 0);
+ let length = bytes.len();
+ bytes[length - 1] ^= 1;
+ assert!(scan(&bytes).unwrap_err().to_string().contains("magic"));
+}
+
+#[test]
+fn manifest_table_size_must_match() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let path = directory.path().join("table.ldb");
+ let bytes = table(&[], 1, 1, 0);
+ fs::write(&path, &bytes)?;
+ for size in [0, bytes.len() as u64 - 1, bytes.len() as u64 + 1, u64::MAX] {
+ assert!(read_table(&path, size, |_, _| Ok(())).is_err());
+ }
+ Ok(())
+}
+
+#[test]
+fn index_handles_cannot_point_outside_the_table() {
+ for (offset, size) in [(u64::MAX, 1), (0, u64::MAX), (10_000, 1), (0, 10_000)] {
+ let mut bytes = table(&[], 1, 1, 0);
+ let mut footer = vec![0, 0];
+ varint(offset, &mut footer);
+ varint(size, &mut footer);
+ footer.resize(40, 0);
+ let start = bytes.len() - 48;
+ bytes[start..start + 40].copy_from_slice(&footer);
+ assert!(scan(&bytes).unwrap_err().to_string().contains("outside"));
+ }
+}
+
+#[test]
+fn data_and_index_checksums() {
+ let original = table(&[(b"key".to_vec(), b"value".to_vec())], 1, 1, 0);
+ let mut footer = Decoder::new(&original[original.len() - 48..original.len() - 8]);
+ footer.varint().unwrap();
+ footer.varint().unwrap();
+ let index_offset = footer.varint().unwrap() as usize;
+ for offset in [0, index_offset] {
+ let mut bytes = original.clone();
+ bytes[offset] ^= 1;
+ assert!(scan(&bytes).unwrap_err().to_string().contains("checksum"));
+ }
+}
+
+#[test]
+fn corrupt_compression_with_valid_checksum() {
+ let entries = [(b"key".to_vec(), vec![b'x'; 100])];
+ let original = table(&entries, 1, 1, 0);
+ let size = block(&entries, 1).len();
+ for compression in [1, 2, 255] {
+ let mut bytes = original.clone();
+ bytes[..size].fill(0xff);
+ bytes[size] = compression;
+ let checksum = crc(&bytes[..size + 1]);
+ bytes[size + 1..size + 5].copy_from_slice(&checksum.to_le_bytes());
+ assert!(scan(&bytes).is_err(), "compression {compression}");
+ }
+}
+
+#[test]
+fn truncated_and_excessive_restart_arrays() {
+ for length in 0..4 {
+ assert!(read_entries(&[0; 4][..length], |_, _| Ok(())).is_err());
+ }
+ for count in [2u32, u32::MAX] {
+ let mut bytes = vec![0; 4];
+ bytes.extend_from_slice(&count.to_le_bytes());
+ assert!(read_entries(&bytes, |_, _| Ok(())).is_err());
+ }
+}
+
+#[test]
+fn restart_offsets_must_be_ordered_entry_boundaries() {
+ let original = block(
+ &[
+ (b"abc".to_vec(), vec![1]),
+ (b"abcd".to_vec(), vec![2]),
+ (b"abd".to_vec(), vec![3]),
+ ],
+ 1,
+ );
+ let restarts = original.len() - 16;
+ for (index, offset) in [(0, 1u32), (1, 0), (1, 1), (1, u32::MAX), (2, 0)] {
+ let mut bytes = original.clone();
+ let start = restarts + index * 4;
+ bytes[start..start + 4].copy_from_slice(&offset.to_le_bytes());
+ assert!(
+ read_entries(&bytes, |_, _| Ok(())).is_err(),
+ "index {index}, offset {offset}"
+ );
+ }
+}
+
+#[test]
+fn restart_entry_cannot_share_a_prefix() {
+ let mut bytes = block(&[(b"abc".to_vec(), vec![]), (b"abcd".to_vec(), vec![])], 1);
+ bytes[6] = 1;
+ assert!(
+ read_entries(&bytes, |_, _| Ok(()))
+ .unwrap_err()
+ .to_string()
+ .contains("shared prefix")
+ );
+}
+
+#[test]
+fn entries_cannot_extend_into_the_restart_array() {
+ let original = block(&[(b"key".to_vec(), b"value".to_vec())], 1);
+ for (index, value) in [(0, 1), (1, 100), (2, 100)] {
+ let mut bytes = original.clone();
+ bytes[index] = value;
+ assert!(read_entries(&bytes, |_, _| Ok(())).is_err());
+ }
+}
+
+#[test]
+fn table_visitor_error_is_propagated() -> Result<()> {
+ let directory = tempfile::tempdir()?;
+ let path = directory.path().join("table.ldb");
+ let bytes = table(&[(b"a".to_vec(), vec![]), (b"b".to_vec(), vec![])], 1, 1, 0);
+ fs::write(&path, &bytes)?;
+ let mut calls = 0;
+ let error = read_table(&path, bytes.len() as u64, |_, _| {
+ calls += 1;
+ anyhow::bail!("visitor failed")
+ })
+ .unwrap_err();
+ assert_eq!(calls, 1);
+ assert!(error.to_string().contains("visitor failed"));
+ Ok(())
+}
diff --git a/moli-leveldb-parser/src/tests/write_batch.rs b/moli-leveldb-parser/src/tests/write_batch.rs
new file mode 100644
index 000000000..2bd6e47f0
--- /dev/null
+++ b/moli-leveldb-parser/src/tests/write_batch.rs
@@ -0,0 +1,163 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Ports of db/write_batch_test.cc; see ../../TESTS.md, ../../LICENSE-LevelDB
+// and ../../AUTHORS-LevelDB. Assertions observe the latest sequence per key.
+
+use super::support::*;
+use crate::{MAX_SEQUENCE, Versions, read_batch};
+
+#[test]
+fn empty() {
+ let mut versions = Versions::new();
+ assert_eq!(read_batch(&batch(0, &[]), &mut versions).unwrap(), None);
+ assert!(versions.is_empty());
+}
+
+#[test]
+fn multiple() {
+ let mut versions = Versions::new();
+ let bytes = batch(
+ 100,
+ &[
+ (b"foo", Some(b"bar")),
+ (b"box", None),
+ (b"baz", Some(b"boo")),
+ ],
+ );
+ assert_eq!(read_batch(&bytes, &mut versions).unwrap(), Some(102));
+ assert_eq!(
+ versions,
+ Versions::from([
+ (b"baz".to_vec(), (102, Some(b"boo".to_vec()))),
+ (b"box".to_vec(), (101, None)),
+ (b"foo".to_vec(), (100, Some(b"bar".to_vec()))),
+ ])
+ );
+}
+
+#[test]
+fn corruption() {
+ let mut bytes = batch(200, &[(b"foo", Some(b"bar")), (b"box", None)]);
+ bytes.pop();
+ assert!(read_batch(&bytes, &mut Versions::new()).is_err());
+}
+
+#[test]
+fn append() {
+ // Upstream appends batches with a different original sequence. On disk the
+ // combined header supplies the sequence, and each appended operation adds 1.
+ let mut bytes = batch(200, &[(b"a", Some(b"va"))]);
+ let appended = batch(300, &[(b"b", Some(b"vb"))]);
+ bytes.extend_from_slice(&appended[12..]);
+ let appended = batch(300, &[(b"b", Some(b"vb")), (b"foo", None)]);
+ bytes.extend_from_slice(&appended[12..]);
+ bytes[8..12].copy_from_slice(&4u32.to_le_bytes());
+ let mut versions = Versions::new();
+ assert_eq!(read_batch(&bytes, &mut versions).unwrap(), Some(203));
+ assert_eq!(
+ versions,
+ Versions::from([
+ (b"a".to_vec(), (200, Some(b"va".to_vec()))),
+ (b"b".to_vec(), (202, Some(b"vb".to_vec()))),
+ (b"foo".to_vec(), (203, None)),
+ ])
+ );
+}
+
+#[test]
+fn truncated_header() {
+ for end in 0..12 {
+ assert!(read_batch(&[0; 12][..end], &mut Versions::new()).is_err());
+ }
+}
+
+#[test]
+fn every_truncated_key_and_value_is_rejected() {
+ let key = vec![b'k'; 130];
+ let value = vec![b'v'; 300];
+ let bytes = batch(12, &[(&key, Some(&value)), (b"deleted", None)]);
+ for end in 12..bytes.len() {
+ assert!(
+ read_batch(&bytes[..end], &mut Versions::new()).is_err(),
+ "end {end}"
+ );
+ }
+ assert_eq!(read_batch(&bytes, &mut Versions::new()).unwrap(), Some(13));
+}
+
+#[test]
+fn count_must_match_in_both_directions() {
+ for count in [0u32, 2, u32::MAX] {
+ let mut bytes = batch(100, &[(b"foo", Some(b"bar"))]);
+ bytes[8..12].copy_from_slice(&count.to_le_bytes());
+ assert!(
+ read_batch(&bytes, &mut Versions::new()).is_err(),
+ "count {count}"
+ );
+ }
+}
+
+#[test]
+fn unknown_value_type() {
+ for tag in [2, 7, 0xff] {
+ let mut bytes = batch(100, &[(b"foo", Some(b"bar"))]);
+ bytes[12] = tag;
+ let error = read_batch(&bytes, &mut Versions::new()).unwrap_err();
+ assert!(error.to_string().contains("value type"));
+ }
+}
+
+#[test]
+fn sequence_limits() {
+ let changes: &[(&[u8], Option<&[u8]>)] = &[(b"a", Some(b"v")), (b"b", None)];
+ assert_eq!(
+ read_batch(&batch(MAX_SEQUENCE, &changes[..1]), &mut Versions::new()).unwrap(),
+ Some(MAX_SEQUENCE)
+ );
+ assert_eq!(
+ read_batch(&batch(MAX_SEQUENCE - 1, changes), &mut Versions::new()).unwrap(),
+ Some(MAX_SEQUENCE)
+ );
+ for (sequence, changes) in [
+ (MAX_SEQUENCE, changes),
+ (MAX_SEQUENCE + 1, &changes[..1]),
+ (u64::MAX, &changes[..0]),
+ ] {
+ assert!(read_batch(&batch(sequence, changes), &mut Versions::new()).is_err());
+ }
+}
+
+#[test]
+fn empty_key_and_value_are_distinct_from_delete() {
+ let mut versions = Versions::new();
+ read_batch(
+ &batch(0, &[(b"", Some(b"")), (b"deleted", None)]),
+ &mut versions,
+ )
+ .unwrap();
+ assert_eq!(versions.get(b"".as_slice()), Some(&(0, Some(vec![]))));
+ assert_eq!(versions.get(b"deleted".as_slice()), Some(&(1, None)));
+}
+
+#[test]
+fn later_delete_cannot_be_resurrected_by_an_older_batch() {
+ let mut versions = Versions::new();
+ read_batch(&batch(20, &[(b"key", None)]), &mut versions).unwrap();
+ read_batch(&batch(10, &[(b"key", Some(b"old"))]), &mut versions).unwrap();
+ assert_eq!(versions.get(b"key".as_slice()), Some(&(20, None)));
+ read_batch(&batch(30, &[(b"key", Some(b"new"))]), &mut versions).unwrap();
+ assert_eq!(
+ versions.get(b"key".as_slice()),
+ Some(&(30, Some(b"new".to_vec())))
+ );
+}
+
+#[test]
+fn duplicate_sequence_requires_identical_contents() {
+ let mut versions = Versions::new();
+ let bytes = batch(10, &[(b"key", Some(b"value"))]);
+ read_batch(&bytes, &mut versions).unwrap();
+ read_batch(&bytes, &mut versions).unwrap();
+ for value in [Some(b"other".as_slice()), None] {
+ assert!(read_batch(&batch(10, &[(b"key", value)]), &mut versions).is_err());
+ }
+}
diff --git a/moli-leveldb-parser/tests/fixtures/.gitattributes b/moli-leveldb-parser/tests/fixtures/.gitattributes
new file mode 100644
index 000000000..671dbfff4
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/.gitattributes
@@ -0,0 +1 @@
+chromium/** -text
diff --git a/moli-leveldb-parser/tests/fixtures/.gitignore b/moli-leveldb-parser/tests/fixtures/.gitignore
new file mode 100644
index 000000000..15498bbfb
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/.gitignore
@@ -0,0 +1 @@
+!*.log
diff --git a/moli-leveldb-parser/tests/fixtures/README.md b/moli-leveldb-parser/tests/fixtures/README.md
new file mode 100644
index 000000000..a9d518566
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/README.md
@@ -0,0 +1,42 @@
+# Chromium LevelDB fixtures
+
+These closed databases were generated by `generate.cc`, linked against
+Chromium's LevelDB revision `7ee830d02b623e8ffe0b95d59a74db1e58da04c5`.
+The generator checks every entry against the native engine's iterator before
+closing each database. Rust tests independently reconstruct the expected data.
+
+- `chromium/randomized-long-db`: the 100,000-operation `Harness.RandomizedLongDB`
+ workload, fixed seed 301, binary and empty keys, duplicate-key updates,
+ multiple live SSTs, and the final WAL not yet converted into an SST.
+- `chromium/recovery`: SSTs plus a fragmented WAL with updates, deletes, binary
+ and empty keys/values, and a 120,000-byte replacement value.
+
+The native fixtures use uncompressed tables. The Rust ports exercise both
+uncompressed and Snappy-compressed data/index blocks, independently of whether
+the host has a C++ Snappy library installed. Cargo tests read the checked-in
+fixtures directly; they need no C++ compiler, downloads, or fixture generation.
+
+To regenerate, run from the repository root with an empty scratch directory:
+
+```sh
+git clone https://chromium.googlesource.com/external/leveldb /tmp/moli-leveldb-fixture-source
+git -C /tmp/moli-leveldb-fixture-source checkout 7ee830d02b623e8ffe0b95d59a74db1e58da04c5
+cmake -S /tmp/moli-leveldb-fixture-source -B /tmp/moli-leveldb-fixture-build \
+ -DCMAKE_BUILD_TYPE=Release -DLEVELDB_BUILD_TESTS=OFF \
+ -DLEVELDB_BUILD_BENCHMARKS=OFF -DLEVELDB_INSTALL=OFF
+cmake --build /tmp/moli-leveldb-fixture-build
+c++ -std=c++17 -O2 moli-leveldb-parser/tests/fixtures/generate.cc \
+ -I/tmp/moli-leveldb-fixture-source/include -I/tmp/moli-leveldb-fixture-source \
+ /tmp/moli-leveldb-fixture-build/libleveldb.a -pthread \
+ -o /tmp/moli-leveldb-fixture-generator
+/tmp/moli-leveldb-fixture-generator /tmp/moli-leveldb-fixtures
+```
+
+If CMake detects optional native Snappy, CRC32C or Zstd libraries, link those as
+well. Inspect the generated contents before replacing `chromium/`. Background
+compaction can change file numbers/layout between runs; the expected logical
+contents are fixed. `SHA256SUMS` records the checked-in files, not an assertion
+that regenerated database files must be byte-for-byte identical.
+
+See [TESTS.md](../../TESTS.md) for upstream sources and port coverage and
+[LICENSE-LevelDB](../../LICENSE-LevelDB) for the upstream license.
diff --git a/moli-leveldb-parser/tests/fixtures/SHA256SUMS b/moli-leveldb-parser/tests/fixtures/SHA256SUMS
new file mode 100644
index 000000000..c0f401a88
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/SHA256SUMS
@@ -0,0 +1,12 @@
+de6ebc200dfc05ac59f36b97ec5e839b9b677265b8ca771f842daa185f43d33b chromium/randomized-long-db/000005.ldb
+6be74a84a3a6b50b5d2417572f5833d3e531afb05a4b6b90ee2083289d25a32c chromium/randomized-long-db/000142.ldb
+07e309106e9e6636b0f8dff4b065f4d4053ee9b87b010079da0c07ba5b1d2b85 chromium/randomized-long-db/000144.ldb
+ea05f11a8a054ae8b475480ad119da17f8e8f751483a981a6d64e4abeb4b6413 chromium/randomized-long-db/000145.log
+57fb102f8f75cd2586fbc5ef4a4b2a9784b841e4ef52e8cc4568dabb694dbd36 chromium/randomized-long-db/000146.ldb
+1005a525006f148c86efcbfb36c6eac091b311532448010f70f7de9a68007167 chromium/randomized-long-db/CURRENT
+0fda46e0ab68223da33a71e967ddfc8d7b85a9b11caf1c877c764aa57f79b983 chromium/randomized-long-db/MANIFEST-000002
+47d6efcc0fe417cf7642d09b6804f28f7dcd979d0d55e3d51c54a4edceb892a3 chromium/recovery/000008.log
+d4471cfe94c0d4d8e7a9843e09da3b4f07bfcf62f69c51e09ffa3dadfe774c15 chromium/recovery/000009.ldb
+9535df112704ffae2a304b10a30fad7ef8dd084a34cbe4df48e45a9d887c2b6e chromium/recovery/000010.ldb
+1005a525006f148c86efcbfb36c6eac091b311532448010f70f7de9a68007167 chromium/recovery/CURRENT
+a0ed0bb5358d12e3dc82265a281ff274800ae7922cc43bea82ff56e44a4dbdad chromium/recovery/MANIFEST-000002
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldb
new file mode 100644
index 000000000..9e7212d30
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldb
new file mode 100644
index 000000000..2f4a4f0f0
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldb
new file mode 100644
index 000000000..cac084a76
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.log b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.log
new file mode 100644
index 000000000..06d11f7c2
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.log differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldb
new file mode 100644
index 000000000..04f950cdb
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENT b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENT
new file mode 100644
index 000000000..1a8485221
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENT
@@ -0,0 +1 @@
+MANIFEST-000002
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002 b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002
new file mode 100644
index 000000000..436049cdf
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002 differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.log b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.log
new file mode 100644
index 000000000..fac922d28
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.log differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldb b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldb
new file mode 100644
index 000000000..93b650b2e
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldb b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldb
new file mode 100644
index 000000000..85e473d02
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldb differ
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENT b/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENT
new file mode 100644
index 000000000..1a8485221
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENT
@@ -0,0 +1 @@
+MANIFEST-000002
diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002 b/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002
new file mode 100644
index 000000000..e832f65a1
Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002 differ
diff --git a/moli-leveldb-parser/tests/fixtures/generate.cc b/moli-leveldb-parser/tests/fixtures/generate.cc
new file mode 100644
index 000000000..b1fa39744
--- /dev/null
+++ b/moli-leveldb-parser/tests/fixtures/generate.cc
@@ -0,0 +1,135 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// RandomKey/RandomString and the stress workload are adapted from LevelDB's
+// util/testutil.cc and table/table_test.cc. See ../../LICENSE-LevelDB,
+// ../../AUTHORS-LevelDB and ../../TESTS.md.
+//
+// Build against the pinned LevelDB source as documented in README.md here.
+// This generator is not compiled or run by Cargo.
+
+#include <cstdio>
+#include <filesystem>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+#include "leveldb/db.h"
+#include "leveldb/write_batch.h"
+#include "util/random.h"
+
+using Entries = std::map<std::string, std::string>;
+
+static void Check(const leveldb::Status& status) {
+ if (!status.ok()) throw std::runtime_error(status.ToString());
+}
+
+static std::string RandomKey(leveldb::Random& random, int length) {
+ static const char chars[] = {'\0', '\1', 'a', 'b', 'c', 'd', 'e',
+ '\xfd', '\xfe', '\xff'};
+ std::string key;
+ for (int i = 0; i < length; ++i) key += chars[random.Uniform(sizeof(chars))];
+ return key;
+}
+
+static std::string RandomString(leveldb::Random& random, int length) {
+ std::string value;
+ for (int i = 0; i < length; ++i) value += char(' ' + random.Uniform(95));
+ return value;
+}
+
+static std::unique_ptr<leveldb::DB> Open(const std::filesystem::path& path) {
+ leveldb::Options options;
+ options.create_if_missing = true;
+ options.error_if_exists = true;
+ options.write_buffer_size = 10000;
+ options.block_size = 256;
+ options.compression = leveldb::kNoCompression;
+ leveldb::DB* db = nullptr;
+ Check(leveldb::DB::Open(options, path.string(), &db));
+ return std::unique_ptr<leveldb::DB>(db);
+}
+
+static void Verify(leveldb::DB& db, const Entries& expected) {
+ auto iter = std::unique_ptr<leveldb::Iterator>(db.NewIterator(leveldb::ReadOptions()));
+ iter->SeekToFirst();
+ for (const auto& [key, value] : expected) {
+ if (!iter->Valid() || iter->key().ToString() != key || iter->value().ToString() != value)
+ throw std::runtime_error("native iterator differs from expected contents");
+ iter->Next();
+ }
+ Check(iter->status());
+ if (iter->Valid()) throw std::runtime_error("native iterator has extra entries");
+}
+
+static void RandomizedLongDB(const std::filesystem::path& path) {
+ auto db = Open(path);
+ leveldb::Random random(301);
+ Entries expected;
+ for (int i = 0; i < 100000; ++i) {
+ // Explicit evaluation order matches the Rust port on every C++ compiler.
+ const int key_length = random.Skewed(4);
+ const auto key = RandomKey(random, key_length);
+ const int value_length = random.Skewed(5);
+ const auto value = RandomString(random, value_length);
+ Check(db->Put(leveldb::WriteOptions(), key, value));
+ expected[key] = value;
+ }
+ Verify(*db, expected);
+ std::cout << path.filename() << ": " << expected.size() << " entries\n";
+}
+
+static void Recovery(const std::filesystem::path& path) {
+ auto db = Open(path);
+ Entries expected;
+ for (int i = 0; i < 200; ++i) {
+ char key[16]; std::snprintf(key, sizeof(key), "key-%04d", i);
+ const std::string value = "value-" + std::to_string(i) + std::string(300, 'x');
+ Check(db->Put(leveldb::WriteOptions(), key, value));
+ expected[key] = value;
+ }
+ Check(db->Put(leveldb::WriteOptions(), "deleted", "must-not-return"));
+ Check(db->Put(leveldb::WriteOptions(), "large", std::string(100000, 'x')));
+ Check(db->Put(leveldb::WriteOptions(), "", "")); expected[""] = "";
+ const std::string binary_key("\0\xffk", 3), binary_value("\xff\0", 2);
+ Check(db->Put(leveldb::WriteOptions(), binary_key, binary_value));
+ expected[binary_key] = binary_value;
+ db->CompactRange(nullptr, nullptr);
+
+ leveldb::WriteBatch batch;
+ for (int i = 0; i < 200; ++i) {
+ char key[16]; std::snprintf(key, sizeof(key), "key-%04d", i);
+ if (i % 7 == 0) { batch.Delete(key); expected.erase(key); }
+ else if (i % 5 == 0) { batch.Put(key, "new"); expected[key] = "new"; }
+ }
+ batch.Delete("deleted");
+ batch.Put("large", std::string(120000, 'y'));
+ expected["large"] = std::string(120000, 'y');
+ leveldb::WriteOptions write; write.sync = true;
+ Check(db->Write(write, &batch));
+ Verify(*db, expected);
+ std::cout << path.filename() << ": " << expected.size() << " entries\n";
+}
+
+int main(int argc, char** argv) {
+ try {
+ if (argc != 2) throw std::runtime_error("usage: generate OUTPUT_DIRECTORY");
+ const std::filesystem::path root(argv[1]);
+ if (!std::filesystem::create_directory(root))
+ throw std::runtime_error("output directory must not already exist");
+ RandomizedLongDB(root / "randomized-long-db");
+ Recovery(root / "recovery");
+ // Keep only the closed database's format-bearing files. Removing LOCK/LOG
+ // here does not change snapshot contents; the Rust tests never open writers.
+ for (const auto& directory : std::filesystem::directory_iterator(root)) {
+ for (const auto& entry : std::filesystem::directory_iterator(directory)) {
+ const auto name = entry.path().filename().string();
+ if (name == "LOCK" || name == "LOG" || name == "LOG.old")
+ std::filesystem::remove(entry);
+ }
+ }
+ } catch (const std::exception& error) {
+ std::cerr << error.what() << '\n';
+ return 1;
+ }
+}