Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
refactor(profile): simplify imports with a read-only LevelDB parser
ldm04e3f174ab13e
diff --git a/Cargo.lock b/Cargo.lockindex 2bbb32e20..683d0081e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2498,8 +2498,10 @@ dependencies = ["moli-cookie-cache","moli-cookie-jar","moli-crypto",+ "moli-leveldb-parser","rusqlite","rusty-leveldb",+ "serde","serde_json","snap","tempfile",@@ -2771,6 +2773,18 @@ dependencies = ["wuff",]+[[package]]+name = "moli-leveldb-parser"+version = "0.1.0"+dependencies = [+ "anyhow",+ "crc32c",+ "rusty-leveldb",+ "snap",+ "tempfile",+ "zerocopy",+]+[[package]]name = "moli-local-executor"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex 9aa15d2e6..8934278c7 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -27,6 +27,7 @@ members = ["moli-indexeddb","moli-local-executor","moli-layout",+ "moli-leveldb-parser","moli-message-port","moli-multipart","moli-xpath",diff --git a/moli-cookie-import/Cargo.toml b/moli-cookie-import/Cargo.tomlindex 4c2f52933..e50613725 100644--- a/moli-cookie-import/Cargo.toml+++ b/moli-cookie-import/Cargo.toml@@ -11,13 +11,18 @@ moli-browser-profile = { path = "../moli-browser-profile" }moli-cookie-cache = { path = "../moli-cookie-cache" }moli-cookie-jar = { path = "../moli-cookie-jar" }moli-crypto = { path = "../moli-crypto" }+moli-leveldb-parser = { path = "../moli-leveldb-parser" }# Keep imports self-contained; .cargo/config.toml trims unused SQLite C extensions.rusqlite = { version = "0.37.0", features = ["bundled"] }-rusty-leveldb = "4.0.1"+serde = { version = "1.0.228", features = ["derive"] }serde_json = "1.0.145"snap = "1.1.1"tempfile = "3.23.0"time = "0.3.44"+[dev-dependencies]+# Generate real LevelDB fixtures and compare the read-only importer with the engine.+rusty-leveldb = "4.0.1"+[lints]workspace = truediff --git a/moli-cookie-import/src/chrome.rs b/moli-cookie-import/src/chrome.rsindex a91a2bd16..98c88d2dd 100644--- a/moli-cookie-import/src/chrome.rs+++ b/moli-cookie-import/src/chrome.rs@@ -1,4 +1,4 @@-use std::{collections::BTreeMap, fs, path::Path};+use std::{fs, path::Path};#[cfg(target_os = "macos")]use std::process::Command;@@ -13,24 +13,13 @@ use moli_cookie_jar::{use moli_crypto::derive_pbkdf2_hmac_sha1;use moli_crypto::{aes_128_cbc_pkcs7_decrypt, aes_256_gcm_decrypt, sha256_digest};use rusqlite::Connection;-use rusty_leveldb::{DB, LdbIterator, Options};use tempfile::TempDir;use time::OffsetDateTime;-use crate::ChromeCryptoKey;+use crate::{ChromeCryptoKey, sqlite::SqliteSnapshot, storage::LocalStorage};const CHROME_EPOCH_OFFSET_MICROS: i64 = 11_644_473_600_000_000;-pub(crate) fn validate_source(path: &Path) -> Result<()> {- if !path.is_dir() {- bail!(- "Chrome profile directory `{}` does not exist",- path.display()- );- }- Ok(())-}-pub(crate) fn import_cookies(profile: &Path,key_source: &ChromeCryptoKey,@@ -44,15 +33,8 @@ pub(crate) fn import_cookies(profile.display())})?;- let temp = tempfile::NamedTempFile::new().context("failed to create Cookies snapshot")?;- fs::copy(&source, temp.path()).with_context(|| {- format!(- "failed to snapshot Chrome Cookies database `{}`",- source.display()- )- })?;- let connection =- Connection::open(temp.path()).context("failed to open Chrome Cookies snapshot")?;+ let snapshot = SqliteSnapshot::open(&source, "Chrome Cookies")?;+ let connection = &snapshot.connection;let schema_version = connection.query_row("SELECT value FROM meta WHERE key = 'version'", [], |row| {row.get::<_, String>(0)@@ -60,18 +42,7 @@ pub(crate) fn import_cookies(.ok().and_then(|value| value.parse::<i64>().ok()).unwrap_or(0);- let has_partition_key = cookie_column_exists(&connection, "top_frame_site_key")?;- let has_cross_site_ancestor = cookie_column_exists(&connection, "has_cross_site_ancestor")?;- let partition_key_column = if has_partition_key {- "top_frame_site_key"- } else {- "NULL"- };- let cross_site_column = if has_cross_site_ancestor {- "has_cross_site_ancestor"- } else {- "0"- };+ let (partition_key_column, cross_site_column) = cookie_partition_columns(connection)?;let sql = format!("SELECT host_key,name,value,encrypted_value,path,expires_utc,is_secure,is_httponly,samesite,priority,source_scheme,source_port,{partition_key_column},{cross_site_column} FROM cookies");@@ -179,15 +150,19 @@ impl ChromeCookieRow {}}-fn cookie_column_exists(connection: &Connection, name: &str) -> Result<bool> {+fn cookie_partition_columns(connection: &Connection) -> Result<(&'static str, &'static str)> {let mut statement = connection.prepare("PRAGMA table_info(cookies)")?;let columns = statement.query_map([], |row| row.get::<_, String>(1))?;+ let mut partition_key = "NULL";+ let mut cross_site_ancestor = "0";for column in columns {- if column? == name {- return Ok(true);+ match column?.as_str() {+ "top_frame_site_key" => partition_key = "top_frame_site_key",+ "has_cross_site_ancestor" => cross_site_ancestor = "has_cross_site_ancestor",+ _ => {}}}- Ok(false)+ Ok((partition_key, cross_site_ancestor))}fn chrome_expiry(micros: i64) -> Option<OffsetDateTime> {@@ -265,26 +240,17 @@ fn decrypt_cookie_value(String::from_utf8(plaintext.to_vec()).context("decrypted cookie value is not UTF-8")}-pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result<usize> {+pub(crate) fn read_local_storage(profile: &Path) -> Result<LocalStorage> {let source = profile.join("Local Storage/leveldb");if !source.is_dir() {- return Ok(0);+ return Ok(LocalStorage::new());}let snapshot = TempDir::new().context("failed to create localStorage snapshot directory")?;copy_dir(&source, snapshot.path())?;- let options = Options {- create_if_missing: false,- ..Options::default()- };- let mut database = DB::open(snapshot.path(), options)- .map_err(|error| anyhow!("failed to open Chrome localStorage LevelDB snapshot: {error}"))?;- let mut origins = BTreeMap::<String, BTreeMap<Vec<u16>, Vec<u16>>>::new();- let mut iterator = database- .new_iter()- .map_err(|error| anyhow!("failed to iterate Chrome localStorage: {error}"))?;- iterator.seek_to_first();- let mut imported = 0;- while let Some((key, value)) = iterator.next() {+ let entries = moli_leveldb_parser::read_snapshot(snapshot.path())+ .context("failed to read Chrome localStorage LevelDB snapshot")?;+ let mut origins = LocalStorage::new();+ for (key, value) in entries {let Some((origin, item_key)) = decode_local_storage_key(&key) else {continue;};@@ -294,37 +260,9 @@ pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Resultorigins.entry(origin).or_default()- .insert(item_key, item_value);- imported += 1;- }- let origins = origins- .into_iter()- .map(|(origin, entries)| {- let entries = entries- .into_iter()- .map(|(key, value)| {- serde_json::json!({- "key": json_dom_string(key),- "value": json_dom_string(value),- })- })- .collect::<Vec<_>>();- (origin, serde_json::json!({ "entries": entries }))- })- .collect::<serde_json::Map<_, _>>();- let bytes = serde_json::to_vec_pretty(&serde_json::json!({- "version": 1,- "origins": origins,- }))?;- moli_browser_profile::write_file_atomically(destination, &bytes, "Chrome localStorage import")?;- Ok(imported)-}--fn json_dom_string(units: Vec<u16>) -> serde_json::Value {- match String::from_utf16(&units) {- Ok(text) => serde_json::Value::String(text),- Err(_) => serde_json::json!({ "utf16": units }),+ .insert(item_key, item_value.into());}+ Ok(origins)}fn decode_local_storage_key(key: &[u8]) -> Option<(String, Vec<u16>)> {@@ -374,13 +312,88 @@ fn copy_dir(source: &Path, destination: &Path) -> Result<()> {mod tests {use super::*;+ #[test]+ fn imports_chrome_local_storage_from_tables_and_wal_without_resurrecting_deleted_keys()+ -> Result<()> {+ let profile = tempfile::tempdir()?;+ let source = profile.path().join("Local Storage/leveldb");+ fs::create_dir_all(&source)?;+ let mut database = rusty_leveldb::DB::open(+ &source,+ rusty_leveldb::Options {+ compressor: 1,+ ..rusty_leveldb::Options::default()+ },+ )?;+ let token = b"_https://example.com\0\x01token";+ let deleted = b"_https://example.com\0\x01deleted";+ let surrogate = b"_https://example.com\0\x00\x00\xd8";+ database.put(b"VERSION", b"1")?;+ database.put(token, b"\x01old")?;+ database.put(deleted, b"\x01removed")?;+ database.put(surrogate, b"\x00\x00\xdc")?;+ database.compact_range(b"", b"\xff")?;+ database.put(token, b"\x01v\xff")?;+ database.delete(deleted)?;+ database.flush()?;++ let destination = profile.path().join("moli-profile");+ let request = crate::ImportRequest {+ profile_dir: &destination,+ includes: "storage".parse().unwrap(),+ chrome_profile_dir: Some(profile.path()),+ chrome_crypto_key: None,+ firefox_profile_dir: None,+ cookie_jars: &[],+ };+ assert_eq!(crate::import_session_state(&request)?.storage, 2);+ let storage_path = destination.join("partitions/default/localstorage.json");+ let bytes = fs::read(&storage_path)?;+ let json: serde_json::Value = serde_json::from_slice(&bytes)?;+ assert_eq!(+ json["origins"]["https://example.com"]["entries"],+ serde_json::json!([+ { "key": "token", "value": "vÿ" },+ { "key": { "utf16": [0xd800] }, "value": { "utf16": [0xdc00] } },+ ])+ );++ // An unreadable live SST must fail the import before replacing a+ // previously imported profile with partial data.+ let table = fs::read_dir(&source)?+ .map(|entry| entry.map(|entry| entry.path()))+ .collect::<std::io::Result<Vec<_>>>()?+ .into_iter()+ .find(|path| path.extension().is_some_and(|extension| extension == "ldb"))+ .unwrap();+ let mut corrupt = fs::read(&table)?;+ corrupt[0] ^= 1;+ fs::write(table, corrupt)?;+ assert!(crate::import_session_state(&request).is_err());+ assert_eq!(fs::read(storage_path)?, bytes);+ Ok(())+ }+#[test]fn imports_chrome_cookies_with_and_without_partition_columns() -> Result<()> {- for partitioned in [false, true] {+ check_chrome_cookie_imports("DELETE")+ }++ #[test]+ fn imports_chrome_cookies_from_wal() -> Result<()> {+ check_chrome_cookie_imports("WAL")+ }++ fn check_chrome_cookie_imports(journal_mode: &str) -> Result<()> {+ for (partitioned, cross_site) in+ [(false, false), (true, false), (false, true), (true, true)]+ {let profile = tempfile::tempdir()?;let network = profile.path().join("Network");fs::create_dir(&network)?;let connection = Connection::open(network.join("Cookies"))?;+ connection.pragma_update(None, "journal_mode", journal_mode)?;+ connection.pragma_update(None, "wal_autocheckpoint", 0)?;connection.execute_batch("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);INSERT INTO meta VALUES ('version', '24');@@ -398,12 +411,19 @@ mod tests {if partitioned {connection.execute_batch("ALTER TABLE cookies ADD COLUMN top_frame_site_key TEXT;- ALTER TABLE cookies ADD COLUMN has_cross_site_ancestor INTEGER;- UPDATE cookies SET top_frame_site_key = 'https://example.org',- has_cross_site_ancestor = 1;",+ UPDATE cookies SET top_frame_site_key = 'https://example.org';",)?;}- drop(connection);+ if cross_site {+ connection.execute_batch(+ "ALTER TABLE cookies ADD COLUMN has_cross_site_ancestor INTEGER;+ UPDATE cookies SET has_cross_site_ancestor = 1;",+ )?;+ }+ // Keep the writer open so uncheckpointed data remains in the WAL.+ if journal_mode == "WAL" {+ assert!(network.join("Cookies-wal").is_file());+ }let cookies = import_cookies(profile.path(), &ChromeCryptoKey::System)?;assert_eq!(cookies.len(), 1);@@ -423,7 +443,7 @@ mod tests {cookie.partition_key,partitioned.then(|| StoredCookiePartitionKey::site("https://example.org".to_owned(),- true,+ cross_site,)),);}diff --git a/moli-cookie-import/src/firefox.rs b/moli-cookie-import/src/firefox.rsindex a0a9f844c..acbe2481f 100644--- a/moli-cookie-import/src/firefox.rs+++ b/moli-cookie-import/src/firefox.rs@@ -1,31 +1,20 @@-use std::{collections::BTreeMap, fs, path::Path};+use std::{fs, path::Path};use anyhow::{Context, Result, bail};use moli_cookie_jar::{CookiePriority, StoredCookie, StoredCookieSameSite, StoredCookieSourceScheme,};-use rusqlite::Connection;use time::OffsetDateTime;-pub(crate) fn validate_source(path: &Path) -> Result<()> {- if !path.is_dir() {- bail!(- "Firefox profile directory `{}` does not exist",- path.display()- );- }- Ok(())-}+use crate::{sqlite::SqliteSnapshot, storage::LocalStorage};pub(crate) fn import_cookies(profile: &Path) -> Result<Vec<StoredCookie>> {let source = profile.join("cookies.sqlite");if !source.is_file() {return Ok(Vec::new());}- let snapshot = snapshot_sqlite(&source, "Firefox cookies")?;- let connection = Connection::open(snapshot.path().join("database.sqlite"))- .context("failed to open Firefox cookies snapshot")?;- let mut statement = connection+ let snapshot = SqliteSnapshot::open(&source, "Firefox cookies")?;+ let mut statement = snapshot.connection.prepare("SELECT name,value,host,path,expiry,isSecure,isHttpOnly,sameSite,schemeMap,originAttributes FROM moz_cookies").context("Firefox cookies database has an unsupported schema")?;let rows = statement.query_map([], |row| {@@ -90,18 +79,18 @@ pub(crate) fn import_cookies(profile: &Path) -> Result<Vec<StoredCookie>> {Ok(cookies)}-pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Result<usize> {+pub(crate) fn read_local_storage(profile: &Path) -> Result<LocalStorage> {let root = profile.join("storage/default");if !root.is_dir() {- return Ok(0);+ return Ok(LocalStorage::new());}let mut databases = Vec::new();collect_local_storage_databases(&root, &mut databases)?;- let mut origins = BTreeMap::<String, BTreeMap<String, String>>::new();- let mut imported = 0;+ databases.sort();+ let mut origins = LocalStorage::new();for source in databases {- let snapshot = snapshot_sqlite(&source, "Firefox localStorage")?;- let connection = Connection::open(snapshot.path().join("database.sqlite"))?;+ let snapshot = SqliteSnapshot::open(&source, "Firefox localStorage")?;+ let connection = &snapshot.connection;let origin: String = connection.query_row("SELECT origin FROM database LIMIT 1", [], |row| row.get(0)).with_context(|| format!("failed to read origin from `{}`", source.display()))?;@@ -115,19 +104,15 @@ pub(crate) fn import_local_storage(profile: &Path, destination: &Path) -> Resultrow.get::<_, Vec<u8>>(3)?,))})?;+ let entries = origins.entry(origin).or_default();for row in rows {let (key, conversion, compression, value) = row?;- let value = decode_local_storage_value(&value, conversion, compression)+ let value = decode_local_storage_value(value, conversion, compression).with_context(|| format!("failed to decode Firefox localStorage `{key}`"))?;- origins- .entry(origin.clone())- .or_default()- .insert(key, value);- imported += 1;+ entries.insert(key.encode_utf16().collect(), value.into());}}- write_local_storage(destination, origins)?;- Ok(imported)+ Ok(origins)}struct FirefoxCookieRow {@@ -169,11 +154,11 @@ fn collect_local_storage_databases(Ok(())}-fn decode_local_storage_value(value: &[u8], conversion: i64, compression: i64) -> Result<String> {+fn decode_local_storage_value(value: Vec<u8>, conversion: i64, compression: i64) -> Result<String> {let value = match compression {- 0 => value.to_vec(),+ 0 => value,1 => snap::raw::Decoder::new()- .decompress_vec(value)+ .decompress_vec(&value).context("invalid Snappy payload")?,other => bail!("unsupported Firefox localStorage compression type {other}"),};@@ -193,67 +178,19 @@ fn decode_local_storage_value(value: &[u8], conversion: i64, compression: i64) -}}-fn write_local_storage(- destination: &Path,- origins: BTreeMap<String, BTreeMap<String, String>>,-) -> Result<()> {- let mut persisted_origins = if destination.is_file() {- let existing: serde_json::Value = serde_json::from_slice(&fs::read(destination)?)- .with_context(|| {- format!(- "failed to parse existing Moli localStorage `{}`",- destination.display()- )- })?;- existing- .get("origins")- .and_then(serde_json::Value::as_object)- .cloned()- .unwrap_or_default()- } else {- serde_json::Map::new()- };- for (origin, entries) in origins {- let entries = entries- .into_iter()- .map(|(key, value)| serde_json::json!({ "key": key, "value": value }))- .collect::<Vec<_>>();- persisted_origins.insert(origin, serde_json::json!({ "entries": entries }));- }- let bytes = serde_json::to_vec_pretty(&serde_json::json!({- "version": 1,- "origins": persisted_origins,- }))?;- moli_browser_profile::write_file_atomically(destination, &bytes, "Firefox localStorage import")-}--fn snapshot_sqlite(source: &Path, label: &str) -> Result<tempfile::TempDir> {- let snapshot =- tempfile::tempdir().with_context(|| format!("failed to create {label} snapshot"))?;- fs::copy(source, snapshot.path().join("database.sqlite"))- .with_context(|| format!("failed to snapshot {label} `{}`", source.display()))?;- for suffix in ["-wal", "-shm"] {- let companion = std::path::PathBuf::from(format!("{}{suffix}", source.display()));- if companion.is_file() {- fs::copy(- &companion,- snapshot.path().join(format!("database.sqlite{suffix}")),- )?;- }- }- Ok(snapshot)-}-#[cfg(test)]mod tests {use super::*;- use rusqlite::params;+ use rusqlite::{Connection, params};#[test]fn decodes_utf8_and_snappy_values() -> Result<()> {- assert_eq!(decode_local_storage_value(b"hello", 1, 0)?, "hello");+ assert_eq!(+ decode_local_storage_value(b"hello".to_vec(), 1, 0)?,+ "hello"+ );let compressed = snap::raw::Encoder::new().compress_vec("世界".as_bytes())?;- assert_eq!(decode_local_storage_value(&compressed, 1, 1)?, "世界");+ assert_eq!(decode_local_storage_value(compressed, 1, 1)?, "世界");Ok(())}@@ -263,7 +200,7 @@ mod tests {.encode_utf16().flat_map(u16::to_ne_bytes).collect::<Vec<_>>();- assert_eq!(decode_local_storage_value(&bytes, 0, 0)?, "hello");+ assert_eq!(decode_local_storage_value(bytes, 0, 0)?, "hello");Ok(())}@@ -338,8 +275,10 @@ mod tests {assert_eq!(cookies[0].value, "value");assert_eq!(cookies[0].same_site, StoredCookieSameSite::Lax);let destination = temp.path().join("localstorage.json");- assert_eq!(import_local_storage(&profile, &destination)?, 1);- let json: serde_json::Value = serde_json::from_slice(&fs::read(destination)?)?;+ let origins = read_local_storage(&profile)?;+ assert_eq!(origins["https://example.com"].len(), 1);+ let bytes = crate::storage::prepare(&destination, origins)?.unwrap();+ let json: serde_json::Value = serde_json::from_slice(&bytes)?;assert_eq!(json["origins"]["https://example.com"]["entries"][0]["value"],"stored-value"diff --git a/moli-cookie-import/src/lib.rs b/moli-cookie-import/src/lib.rsindex 301f4b4fc..5f6b769c5 100644--- a/moli-cookie-import/src/lib.rs+++ b/moli-cookie-import/src/lib.rs@@ -2,6 +2,8 @@mod chrome;mod firefox;+mod sqlite;+mod storage;use std::{fmt,@@ -109,11 +111,10 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary)})?;let partition = profile.default_partition();- let mut summary = ImportSummary::default();let mut imported_cookies = Vec::new();+ let mut imported_storage = storage::LocalStorage::new();if let Some(source) = request.chrome_profile_dir {- chrome::validate_source(source)?;if request.includes.contains(ImportIncludes::COOKIES) {imported_cookies.extend(chrome::import_cookies(source,@@ -123,19 +124,16 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary)?);}if request.includes.contains(ImportIncludes::STORAGE) {- summary.storage +=- chrome::import_local_storage(source, partition.local_storage_path())?;+ storage::merge(&mut imported_storage, chrome::read_local_storage(source)?);}}if let Some(source) = request.firefox_profile_dir {- firefox::validate_source(source)?;if request.includes.contains(ImportIncludes::COOKIES) {imported_cookies.extend(firefox::import_cookies(source)?);}if request.includes.contains(ImportIncludes::STORAGE) {- summary.storage +=- firefox::import_local_storage(source, partition.local_storage_path())?;+ storage::merge(&mut imported_storage, firefox::read_local_storage(source)?);}}@@ -146,12 +144,31 @@ pub fn import_session_state(request: &ImportRequest<'_>) -> Result<ImportSummary.with_context(|| format!("failed to import cookie jar `{}`", jar.display()))?,);}- summary.cookies = imported_cookies.len();- if summary.cookies > 0 {- let mut cookies = moli_cookie_cache::load_cookie_cache(partition.cookies_path())?;- cookies.extend(imported_cookies);- moli_cookie_cache::save_cookie_cache(partition.cookies_path(), cookies)?;- }+ }++ let summary = ImportSummary {+ cookies: imported_cookies.len(),+ storage: imported_storage.values().map(|entries| entries.len()).sum(),+ };+ // Read and merge every source and existing destination before writing.+ // Each file is replaced atomically; the two files are not a transaction.+ let cookies = if imported_cookies.is_empty() {+ None+ } else {+ let mut cookies = moli_cookie_cache::load_cookie_cache(partition.cookies_path())?;+ cookies.extend(imported_cookies);+ Some(cookies)+ };+ let storage = storage::prepare(partition.local_storage_path(), imported_storage)?;+ if let Some(bytes) = storage {+ moli_browser_profile::write_file_atomically(+ partition.local_storage_path(),+ &bytes,+ "localStorage import",+ )?;+ }+ if let Some(cookies) = cookies {+ moli_cookie_cache::save_cookie_cache(partition.cookies_path(), cookies)?;}Ok(summary)@@ -166,72 +183,21 @@ fn validate_request(request: &ImportRequest<'_>) -> Result<()> {if !request.cookie_jars.is_empty() && !request.includes.contains(ImportIncludes::COOKIES) {bail!("cookie jars can only import cookies; include `cookies` in the import selection");}+ for (browser, source) in [+ ("Chrome", request.chrome_profile_dir),+ ("Firefox", request.firefox_profile_dir),+ ] {+ if let Some(source) = source+ && !source.is_dir()+ {+ bail!(+ "{browser} profile directory `{}` does not exist",+ source.display()+ );+ }+ }Ok(())}#[cfg(test)]-mod tests {- use std::fs;-- use anyhow::Result;-- use super::*;-- #[test]- fn bundled_sqlite_omits_unused_extensions() -> Result<()> {- // Inspect the linked engine so dependency upgrades cannot silently- // reintroduce extensions excluded by .cargo/config.toml.- let connection = rusqlite::Connection::open_in_memory()?;- for option in [- "ENABLE_FTS3",- "ENABLE_FTS3_PARENTHESIS",- "ENABLE_FTS4",- "ENABLE_FTS5",- "ENABLE_RTREE",- "ENABLE_DBSTAT_VTAB",- "ENABLE_STAT4",- "ENABLE_LOAD_EXTENSION",- ] {- let enabled: bool =- connection.query_row("SELECT sqlite_compileoption_used(?1)", [option], |row| {- row.get(0)- })?;- assert!(!enabled, "bundled SQLite unexpectedly enables {option}");- }- let omits_extension_loading: bool = connection.query_row(- "SELECT sqlite_compileoption_used('OMIT_LOAD_EXTENSION')",- [],- |row| row.get(0),- )?;- assert!(omits_extension_loading);- Ok(())- }-- #[test]- fn imports_a_netscape_cookie_jar() -> Result<()> {- let temp = tempfile::tempdir()?;- let jar = temp.path().join("cookies.txt");- let destination = temp.path().join("moli-profile");- fs::write(- &jar,- "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tFALSE\t0\tsession\tvalue\n",- )?;-- let summary = import_session_state(&ImportRequest {- profile_dir: &destination,- includes: "all".parse().expect("valid import selection"),- chrome_profile_dir: None,- chrome_crypto_key: None,- firefox_profile_dir: None,- cookie_jars: &[jar],- })?;-- let profile = BrowserProfile::open(&destination)?;- let cookies =- moli_cookie_cache::load_cookie_cache(profile.default_partition().cookies_path())?;- assert_eq!(summary.cookies, 1);- assert_eq!(cookies.len(), 1);- assert_eq!(cookies[0].name, "session");- Ok(())- }-}+mod tests;diff --git a/moli-cookie-import/src/sqlite.rs b/moli-cookie-import/src/sqlite.rsnew file mode 100644index 000000000..ef202d31a--- /dev/null+++ b/moli-cookie-import/src/sqlite.rs@@ -0,0 +1,106 @@+use std::{fs, io::ErrorKind, path::Path};++use anyhow::{Context, Result};+use rusqlite::{Connection, OpenFlags};+use tempfile::TempDir;++pub(crate) struct SqliteSnapshot {+ // Close the connection before removing its files (fields drop in order).+ pub connection: Connection,+ _directory: TempDir,+}++impl SqliteSnapshot {+ /// Copy a database and its WAL companions. The source files must stay+ /// stable during copying; this is not an online-backup transaction.+ pub fn open(source: &Path, label: &str) -> Result<Self> {+ let directory =+ tempfile::tempdir().with_context(|| format!("failed to create {label} snapshot"))?;+ let database = directory.path().join("database.sqlite");+ fs::copy(source, &database)+ .with_context(|| format!("failed to snapshot {label} `{}`", source.display()))?;+ for suffix in ["-wal", "-shm"] {+ let mut companion = source.as_os_str().to_owned();+ companion.push(suffix);+ match fs::copy(+ &companion,+ directory.path().join(format!("database.sqlite{suffix}")),+ ) {+ Ok(_) => {}+ Err(error) if error.kind() == ErrorKind::NotFound => {}+ Err(error) => {+ return Err(error).with_context(|| {+ format!(+ "failed to snapshot {label} `{}`",+ Path::new(&companion).display()+ )+ });+ }+ }+ }+ let connection = Connection::open_with_flags(&database, OpenFlags::SQLITE_OPEN_READ_ONLY)+ .with_context(|| format!("failed to open {label} snapshot"))?;+ Ok(Self {+ connection,+ _directory: directory,+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn snapshot_reads_wal_without_changing_source_and_rejects_writes() -> Result<()> {+ check_wal_snapshot(Path::new("cookies.sqlite"))+ }++ #[cfg(unix)]+ #[test]+ fn snapshot_preserves_non_utf8_paths_when_finding_wal_files() -> Result<()> {+ use std::os::unix::ffi::OsStrExt;+ check_wal_snapshot(Path::new(std::ffi::OsStr::from_bytes(+ b"cookies-\xff.sqlite",+ )))+ }++ fn check_wal_snapshot(name: &Path) -> Result<()> {+ let directory = tempfile::tempdir()?;+ let source = directory.path().join(name);+ let writer = Connection::open(&source)?;+ writer.execute_batch(+ "PRAGMA journal_mode=WAL;+ PRAGMA wal_autocheckpoint=0;+ CREATE TABLE entries (value TEXT);+ PRAGMA wal_checkpoint(TRUNCATE);+ INSERT INTO entries VALUES ('only in WAL');",+ )?;+ let files = fs::read_dir(directory.path())?+ .map(|entry| {+ let path = entry?.path();+ let bytes = fs::read(&path)?;+ Ok((path, bytes))+ })+ .collect::<std::io::Result<Vec<_>>>()?;+ let snapshot = SqliteSnapshot::open(&source, "test database")?;+ let value: String =+ snapshot+ .connection+ .query_row("SELECT value FROM entries", [], |row| row.get(0))?;+ assert_eq!(value, "only in WAL");+ let error = snapshot+ .connection+ .execute("INSERT INTO entries VALUES ('forbidden')", [])+ .unwrap_err();+ assert_eq!(+ error.sqlite_error_code(),+ Some(rusqlite::ErrorCode::ReadOnly)+ );+ drop(snapshot);+ for (path, bytes) in files {+ assert_eq!(fs::read(path)?, bytes);+ }+ Ok(())+ }+}diff --git a/moli-cookie-import/src/storage.rs b/moli-cookie-import/src/storage.rsnew file mode 100644index 000000000..8a28354d5--- /dev/null+++ b/moli-cookie-import/src/storage.rs@@ -0,0 +1,126 @@+use std::{collections::BTreeMap, fs, io::ErrorKind, path::Path};++use anyhow::{Context, Result, ensure};+use serde::{Deserialize, Serialize};++// Keys use UTF-16 units so JSON text and explicit UTF-16 keys merge identically.+// Values keep owned UTF-8 strings when possible, including Firefox's SQL values.+pub(crate) type LocalStorage = BTreeMap<String, BTreeMap<Vec<u16>, DomString>>;++#[derive(Debug, Serialize, Deserialize)]+#[serde(untagged)]+pub(crate) enum DomString {+ Text(String),+ Utf16 { utf16: Vec<u16> },+}++impl From<String> for DomString {+ fn from(value: String) -> Self {+ Self::Text(value)+ }+}++impl From<Vec<u16>> for DomString {+ fn from(units: Vec<u16>) -> Self {+ match String::from_utf16(&units) {+ Ok(text) => Self::Text(text),+ Err(_) => Self::Utf16 { utf16: units },+ }+ }+}++impl DomString {+ fn into_units(self) -> Vec<u16> {+ match self {+ Self::Text(text) => text.encode_utf16().collect(),+ Self::Utf16 { utf16 } => utf16,+ }+ }+}++#[derive(Serialize, Deserialize)]+struct StorageFile {+ version: u8,+ origins: BTreeMap<String, StorageArea>,+}++#[derive(Serialize, Deserialize)]+struct StorageArea {+ entries: Vec<StorageEntry>,+}++#[derive(Serialize, Deserialize)]+struct StorageEntry {+ key: DomString,+ value: DomString,+}++pub(crate) fn merge(target: &mut LocalStorage, source: LocalStorage) {+ for (origin, entries) in source {+ if !entries.is_empty() {+ target.entry(origin).or_default().extend(entries);+ }+ }+}++/// Prepare the merged file without writing it, so other inputs can be checked+/// before any destination data is replaced. Later sources win on matching keys.+pub(crate) fn prepare(destination: &Path, imported: LocalStorage) -> Result<Option<Vec<u8>>> {+ if imported.values().all(BTreeMap::is_empty) {+ return Ok(None);+ }+ let mut origins = load(destination)?;+ merge(&mut origins, imported);+ let file = StorageFile {+ version: 1,+ origins: origins+ .into_iter()+ .map(|(origin, entries)| {+ let entries = entries+ .into_iter()+ .map(|(key, value)| StorageEntry {+ key: key.into(),+ value,+ })+ .collect();+ (origin, StorageArea { entries })+ })+ .collect(),+ };+ serde_json::to_vec_pretty(&file)+ .context("failed to serialize imported localStorage")+ .map(Some)+}++fn load(path: &Path) -> Result<LocalStorage> {+ let bytes = match fs::read(path) {+ Ok(bytes) => bytes,+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(LocalStorage::new()),+ Err(error) => {+ return Err(error)+ .with_context(|| format!("failed to read Moli localStorage `{}`", path.display()));+ }+ };+ if bytes.is_empty() {+ return Ok(LocalStorage::new());+ }+ let file: StorageFile = serde_json::from_slice(&bytes)+ .with_context(|| format!("failed to parse Moli localStorage `{}`", path.display()))?;+ ensure!(+ file.version == 1,+ "unsupported Moli localStorage version {}",+ file.version+ );+ Ok(file+ .origins+ .into_iter()+ .map(|(origin, area)| {+ let entries = area+ .entries+ .into_iter()+ .map(|entry| (entry.key.into_units(), entry.value))+ .collect();+ (origin, entries)+ })+ .collect())+}diff --git a/moli-cookie-import/src/tests.rs b/moli-cookie-import/src/tests.rsnew file mode 100644index 000000000..c7838e206--- /dev/null+++ b/moli-cookie-import/src/tests.rs@@ -0,0 +1,307 @@+use std::fs;++use anyhow::Result;+use rusqlite::{Connection, params};++use super::*;++fn storage_request(profile_dir: &Path) -> ImportRequest<'_> {+ ImportRequest {+ profile_dir,+ includes: "storage".parse().unwrap(),+ chrome_profile_dir: None,+ chrome_crypto_key: None,+ firefox_profile_dir: None,+ cookie_jars: &[],+ }+}++fn chrome_storage(profile: &Path, entries: &[(&[u8], &[u8])]) -> Result<()> {+ let path = profile.join("Local Storage/leveldb");+ fs::create_dir_all(&path)?;+ let mut database = rusty_leveldb::DB::open(path, rusty_leveldb::Options::default())?;+ database.put(b"VERSION", b"1")?;+ for (key, value) in entries {+ database.put(key, value)?;+ }+ database.flush()?;+ Ok(())+}++fn firefox_storage(profile: &Path, entries: &[(&str, &str)]) -> Result<PathBuf> {+ let directory = profile.join("storage/default/https+++example.com/ls");+ fs::create_dir_all(&directory)?;+ let path = directory.join("data.sqlite");+ let connection = Connection::open(&path)?;+ connection.execute_batch(+ "CREATE TABLE database (origin TEXT);+ INSERT INTO database VALUES ('https://example.com');+ CREATE TABLE data (key TEXT, conversion_type INTEGER, compression_type INTEGER, value BLOB);",+ )?;+ for (key, value) in entries {+ connection.execute(+ "INSERT INTO data VALUES (?1, 1, 0, ?2)",+ params![key, value.as_bytes()],+ )?;+ }+ Ok(path)+}++fn existing_storage(destination: &Path) -> Result<PathBuf> {+ let path = destination.join("partitions/default/localstorage.json");+ fs::create_dir_all(path.parent().unwrap())?;+ fs::write(+ &path,+ serde_json::to_vec_pretty(&serde_json::json!({+ "version": 1,+ "origins": {+ "https://keep.example": {"entries": [{"key": "keep", "value": "untouched"}]},+ "https://example.com": {"entries": [+ {"key": "preserved", "value": "keep"},+ {"key": {"utf16": [115, 104, 97, 114, 101, 100]}, "value": "old shared"},+ {"key": "raw-value", "value": {"utf16": [55296]}},+ {"key": {"utf16": [55296]}, "value": "old surrogate"}+ ]}+ }+ }))?,+ )?;+ Ok(path)+}++#[test]+fn merges_both_browsers_with_existing_storage_by_utf16_key() -> Result<()> {+ let temp = tempfile::tempdir()?;+ let destination = temp.path().join("moli");+ let chrome = temp.path().join("chrome");+ let firefox = temp.path().join("firefox");+ let storage = existing_storage(&destination)?;+ chrome_storage(+ &chrome,+ &[+ (b"_https://example.com\0\x01chrome-only", b"\x01chrome"),+ (b"_https://example.com\0\x01shared", b"\x01chrome"),+ (b"_https://example.com\0\x00\x00\xd8", b"\x00\x00\xdc"),+ (b"_https://chrome.example\0\x01token", b"\x01chrome"),+ ],+ )?;+ firefox_storage(+ &firefox,+ &[("shared", "firefox"), ("firefox-only", "firefox")],+ )?;+ let mut request = storage_request(&destination);+ request.chrome_profile_dir = Some(&chrome);+ request.firefox_profile_dir = Some(&firefox);+ let summary = import_session_state(&request)?;+ assert_eq!(+ summary,+ ImportSummary {+ cookies: 0,+ storage: 5+ }+ );+ let bytes = fs::read(&storage)?;+ let actual: serde_json::Value = serde_json::from_slice(&bytes)?;+ assert_eq!(+ actual,+ serde_json::json!({+ "version": 1,+ "origins": {+ "https://chrome.example": {"entries": [{"key": "token", "value": "chrome"}]},+ "https://keep.example": {"entries": [{"key": "keep", "value": "untouched"}]},+ "https://example.com": {"entries": [+ {"key": "chrome-only", "value": "chrome"},+ {"key": "firefox-only", "value": "firefox"},+ {"key": "preserved", "value": "keep"},+ {"key": "raw-value", "value": {"utf16": [55296]}},+ {"key": "shared", "value": "firefox"},+ {"key": {"utf16": [55296]}, "value": {"utf16": [56320]}}+ ]}+ }+ })+ );+ assert_eq!(import_session_state(&request)?, summary);+ assert_eq!(fs::read(storage)?, bytes);+ Ok(())+}++#[test]+fn empty_browser_storage_does_not_create_or_replace_a_file() -> Result<()> {+ for existing in [false, true] {+ let temp = tempfile::tempdir()?;+ let destination = temp.path().join("moli");+ let chrome = temp.path().join("chrome");+ let firefox = temp.path().join("firefox");+ chrome_storage(&chrome, &[])?;+ firefox_storage(&firefox, &[])?;+ let storage = destination.join("partitions/default/localstorage.json");+ let before = if existing {+ Some(fs::read(existing_storage(&destination)?)?)+ } else {+ None+ };+ let mut request = storage_request(&destination);+ request.chrome_profile_dir = Some(&chrome);+ request.firefox_profile_dir = Some(&firefox);+ assert_eq!(import_session_state(&request)?, ImportSummary::default());+ assert_eq!(fs::read(storage).ok(), before);+ }+ Ok(())+}++#[test]+fn rejects_missing_source_directories_before_creating_a_profile() -> Result<()> {+ for browser in ["Chrome", "Firefox"] {+ let temp = tempfile::tempdir()?;+ let destination = temp.path().join("moli");+ let missing = temp.path().join("missing");+ let mut request = storage_request(&destination);+ if browser == "Chrome" {+ request.chrome_profile_dir = Some(&missing);+ } else {+ request.firefox_profile_dir = Some(&missing);+ }+ let error = import_session_state(&request).unwrap_err();+ assert!(error.to_string().contains(browser));+ assert!(!destination.exists());+ }+ Ok(())+}++#[test]+fn later_source_failures_leave_existing_storage_and_cookies_unchanged() -> Result<()> {+ for corrupt_firefox in [true, false] {+ let temp = tempfile::tempdir()?;+ let destination = temp.path().join("moli");+ let chrome = temp.path().join("chrome");+ let firefox = temp.path().join("firefox");+ let storage = existing_storage(&destination)?;+ let cookies = storage.with_file_name("cookies.json");+ moli_cookie_cache::save_cookie_cache(&cookies, [])?;+ let before_storage = fs::read(&storage)?;+ let before_cookies = fs::read(&cookies)?;+ let database = firefox_storage(&firefox, &[("new", "firefox")])?;+ let jars = [temp.path().join("missing-cookie-jar")];+ let mut request = storage_request(&destination);+ request.firefox_profile_dir = Some(&firefox);+ if corrupt_firefox {+ chrome_storage(+ &chrome,+ &[(b"_https://example.com\0\x01new", b"\x01chrome")],+ )?;+ request.chrome_profile_dir = Some(&chrome);+ fs::write(database, b"corrupt SQLite file")?;+ } else {+ request.includes = "all".parse().unwrap();+ request.cookie_jars = &jars;+ }+ let error = format!("{:#}", import_session_state(&request).unwrap_err());+ assert!(+ error.contains(if corrupt_firefox {+ "data.sqlite"+ } else {+ "missing-cookie-jar"+ }),+ "{error}"+ );+ assert_eq!(fs::read(storage)?, before_storage);+ assert_eq!(fs::read(cookies)?, before_cookies);+ }+ Ok(())+}++#[test]+fn invalid_destination_files_are_rejected_before_either_file_is_written() -> Result<()> {+ for (target, invalid) in [+ ("cookies.json", "{"),+ ("localstorage.json", "{"),+ ("localstorage.json", r#"{"version":2,"origins":{}}"#),+ (+ "localstorage.json",+ r#"{"version":1,"origins":{"https://example.com":{"entries":[{"key":{},"value":"x"}]}}}"#,+ ),+ ] {+ let temp = tempfile::tempdir()?;+ let destination = temp.path().join("moli");+ let firefox = temp.path().join("firefox");+ let storage = existing_storage(&destination)?;+ let cookies = storage.with_file_name("cookies.json");+ moli_cookie_cache::save_cookie_cache(&cookies, [])?;+ fs::write(storage.with_file_name(target), invalid)?;+ let before_storage = fs::read(&storage)?;+ let before_cookies = fs::read(&cookies)?;+ firefox_storage(&firefox, &[("new", "firefox")])?;+ let jars = [temp.path().join("cookies.txt")];+ fs::write(+ &jars[0],+ "example.com\tFALSE\t/\tFALSE\t0\tsession\tnew-value\n",+ )?;+ let mut request = storage_request(&destination);+ request.includes = "all".parse().unwrap();+ request.firefox_profile_dir = Some(&firefox);+ request.cookie_jars = &jars;+ assert!(+ import_session_state(&request).is_err(),+ "accepted {target}: {invalid}"+ );+ assert_eq!(fs::read(storage)?, before_storage);+ assert_eq!(fs::read(cookies)?, before_cookies);+ }+ Ok(())+}++#[test]+fn bundled_sqlite_omits_unused_extensions() -> Result<()> {+ // Inspect the linked engine so dependency upgrades cannot silently+ // reintroduce extensions excluded by .cargo/config.toml.+ let connection = rusqlite::Connection::open_in_memory()?;+ for option in [+ "ENABLE_FTS3",+ "ENABLE_FTS3_PARENTHESIS",+ "ENABLE_FTS4",+ "ENABLE_FTS5",+ "ENABLE_RTREE",+ "ENABLE_DBSTAT_VTAB",+ "ENABLE_STAT4",+ "ENABLE_LOAD_EXTENSION",+ ] {+ let enabled: bool =+ connection.query_row("SELECT sqlite_compileoption_used(?1)", [option], |row| {+ row.get(0)+ })?;+ assert!(!enabled, "bundled SQLite unexpectedly enables {option}");+ }+ let omits_extension_loading: bool = connection.query_row(+ "SELECT sqlite_compileoption_used('OMIT_LOAD_EXTENSION')",+ [],+ |row| row.get(0),+ )?;+ assert!(omits_extension_loading);+ Ok(())+}++#[test]+fn imports_a_netscape_cookie_jar() -> Result<()> {+ let temp = tempfile::tempdir()?;+ let jar = temp.path().join("cookies.txt");+ let destination = temp.path().join("moli-profile");+ fs::write(+ &jar,+ "# Netscape HTTP Cookie File\nexample.com\tFALSE\t/\tFALSE\t0\tsession\tvalue\n",+ )?;++ let summary = import_session_state(&ImportRequest {+ profile_dir: &destination,+ includes: "all".parse().expect("valid import selection"),+ chrome_profile_dir: None,+ chrome_crypto_key: None,+ firefox_profile_dir: None,+ cookie_jars: &[jar],+ })?;++ let profile = BrowserProfile::open(&destination)?;+ let cookies = moli_cookie_cache::load_cookie_cache(profile.default_partition().cookies_path())?;+ assert_eq!(summary.cookies, 1);+ assert_eq!(cookies.len(), 1);+ assert_eq!(cookies[0].name, "session");+ Ok(())+}diff --git a/moli-leveldb-parser/AUTHORS-LevelDB b/moli-leveldb-parser/AUTHORS-LevelDBnew file mode 100644index 000000000..2439d7a45--- /dev/null+++ b/moli-leveldb-parser/AUTHORS-LevelDB@@ -0,0 +1,12 @@+# Names should be added to this file like so:+# Name or Organization <email address>++Google Inc.++# Initial version authors:+Jeffrey Dean <[email protected]>+Sanjay Ghemawat <[email protected]>++# Partial list of contributors:+Kevin Regan <[email protected]>+Johan Bilien <[email protected]>diff --git a/moli-leveldb-parser/Cargo.toml b/moli-leveldb-parser/Cargo.tomlnew file mode 100644index 000000000..687dc7bc1--- /dev/null+++ b/moli-leveldb-parser/Cargo.toml@@ -0,0 +1,20 @@+[package]+license = "(MIT OR Apache-2.0) AND BSD-3-Clause"+name = "moli-leveldb-parser"+version = "0.1.0"+edition = "2024"+description = "Read-only LevelDB snapshot reader"++[dependencies]+anyhow = "1.0.100"+crc32c = "0.6.8"+snap = "1.1.1"+zerocopy = { version = "0.8.46", default-features = false, features = ["derive"] }++[dev-dependencies]+# Generate database fixtures and compare snapshot contents with an independent engine.+rusty-leveldb = "4.0.1"+tempfile = "3.23.0"++[lints]+workspace = truediff --git a/moli-leveldb-parser/LICENSE-LevelDB b/moli-leveldb-parser/LICENSE-LevelDBnew file mode 100644index 000000000..8e80208cd--- /dev/null+++ b/moli-leveldb-parser/LICENSE-LevelDB@@ -0,0 +1,27 @@+Copyright (c) 2011 The LevelDB Authors. All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are+met:++ * Redistributions of source code must retain the above copyright+notice, this list of conditions and the following disclaimer.+ * Redistributions in binary form must reproduce the above+copyright notice, this list of conditions and the following disclaimer+in the documentation and/or other materials provided with the+distribution.+ * Neither the name of Google Inc. nor the names of its+contributors may be used to endorse or promote products derived from+this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.diff --git a/moli-leveldb-parser/TESTS.md b/moli-leveldb-parser/TESTS.mdnew file mode 100644index 000000000..660bd4932--- /dev/null+++ b/moli-leveldb-parser/TESTS.md@@ -0,0 +1,166 @@+# Chromium LevelDB test ports++The source is Chromium's actual LevelDB dependency, pinned to+[`7ee830d02b623e8ffe0b95d59a74db1e58da04c5`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/).+Chromium revision+[`d9fa5a15858b7189aa1a70cebe811169154ebcf0`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/)+selects that gitlink and identifies it as LevelDB 1.23 in+[`README.chromium`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/README.chromium).+The tests retain the upstream [license](LICENSE-LevelDB) and+[authors](AUTHORS-LevelDB).++The suite has 133 tests: 68 ports/adaptations of the scenarios listed below,+56 additional parser regressions, and the 9 original snapshot tests. This is+coverage of the read-only parser, not the complete Chromium LevelDB engine+suite. Each original scenario keeps its snake_case name, with a suffix where+its observable behavior was adapted. Supporting fixtures are test-only code.++```sh+cargo test -p moli-leveldb-parser -p moli-cookie-import --lib+cargo fmt --all+cargo clippy --workspace --all-targets --all-features -- -D warnings+cargo nextest run --no-fail-fast+```++## WAL physical records++Source: [`db/log_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/log_test.cc).+Port: [`src/tests/log.rs`](src/tests/log.rs), 31 tests, including these 23 upstream cases:++| Upstream cases | What is checked |+| --- | --- |+| `Empty`, `ReadWrite`, `ManyBlocks` | Empty records and 100,000 records over many physical blocks |+| `Fragmentation`, `RandomRead` | 50/100 KB values and 500 skewed-size records, fixed seed 301 |+| `MarginalTrailer`, `MarginalTrailer2`, `ShortTrailer`, `AlignedEof` | Exact block ends, short padding and empty FIRST fragments |+| `OpenForAppend` | Reading a log whose writer resumed at a nonzero block offset |+| `ReadError` | An injected I/O error propagates |+| `BadRecordType`, `BadLength`, `ChecksumMismatch` | Unknown types, records crossing a block, and bad CRC32C |+| `UnexpectedMiddleType`, `UnexpectedLastType`, `UnexpectedFullType`, `UnexpectedFirstType` | Invalid fragment ordering |+| `TruncatedTrailingRecordIsIgnored`, `BadLengthAtEndIsIgnored`, `MissingLastIsIgnored`, `PartialLastIsIgnored` | Torn final appends never emit partial records |+| `ErrorJoinsRecords` | Corruption must not join fragments belonging to different logical records |++Upstream's reader reports corruption and can continue salvaging records. The+parser returns `Err` at complete corruption; the public snapshot API exposes+no partial result. Tests assert errors instead of upstream dropped-byte counts+and salvaged data. Incomplete final appends retain upstream EOF behavior.++The remaining 15 upstream cases (`SkipIntoMultiRecord`, `ReadStart`,+`ReadSecond*`, `ReadThird*`, `ReadFourth*`, `ReadInitialOffsetIntoBlockPadding`,+`ReadEnd`, `ReadPastEnd`) exercise an initial-offset/seek API. This parser always+starts at byte zero, so those API-specific cases are outside its scope.++Additional tests cover the empty-FIRST compatibility exception documented in+[`db/log_reader.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/log_reader.cc),+checksums of every fragment kind, zero-filled preallocation, corrupt padding,+short/interrupted reads, callback errors, and cuts around every fragment boundary.++## Coding, CRC and WriteBatch++The four additional [layout tests](src/tests/layout.rs) use literal wire bytes+at every alignment modulo 8 to check borrowed headers, trailers and restart+arrays, little-endian values, truncation without cursor advancement, and+nonempty blocks with zero restarts. Pointer checks ensure views refer to the+original input; the existing corruption tests continue to validate semantics.++| Source | Ported cases | Local tests |+| --- | --- | --- |+| [`util/coding_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/util/coding_test.cc) | All 10: `Fixed32`, `Fixed64`, `EncodingOutput`, `Varint32`, `Varint64`, both overflow cases, both truncation cases, `Strings` | [coding.rs](src/tests/coding.rs) |+| [`util/crc32c_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/util/crc32c_test.cc) | `StandardResults`, `Values`, `Mask` | [coding.rs](src/tests/coding.rs) |+| [`db/write_batch_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/write_batch_test.cc) | `Empty`, `Multiple`, `Corruption`, `Append` | [write_batch.rs](src/tests/write_batch.rs) |++`EncodingOutput` checks little-endian decoding against literal bytes; the+parser has no encoder. `CRC.Mask` checks the production masking wrapper+against its unmasked checksum. `CRC.Extend` and `WriteBatch.ApproximateSize`+test APIs absent from this crate. `Append` checks the combined on-disk batch;+the parser retains the latest version per key rather than all memtable history.++Added regressions cover truncated fixed fields and length prefixes, overlong+varint32 encodings, high bits that would overflow, every batch truncation point,+counts too small/large, unsupported tags, 56-bit sequence limits, empty keys and+values, ordered deletes, and conflicting values at the same sequence.++## MANIFEST / VersionEdit++Source: [`db/version_edit_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/version_edit_test.cc).+Port: [`src/tests/manifest.rs`](src/tests/manifest.rs), 16 tests.++`VersionEditTest.EncodeDecode` preserves the large `(1 << 50)` file, size and+sequence values, additions, removals and compaction pointers. Assertions check+decoded manifest state instead of encoding it again. Its arbitrary `foo`+comparator is tested as an explicit unsupported-comparator error; supported+snapshots use `leveldb.BytewiseComparator`.++Additional cases cover metadata split across records, missing metadata,+previous-log updates, every VersionEdit field's truncation, unknown tags,+invalid levels/internal keys, sequence overflow, file deletion and level moves,+deletions-before-additions in one edit, CURRENT validation/selection, incomplete+final edits, and complete edits with invalid checksums. `ComparatorCheck` also+corresponds to the custom-comparator mismatch scenario in `db/db_test.cc`.++## SST blocks and internal keys++Sources: [`table/table_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/table/table_test.cc),+[`db/dbformat_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/dbformat_test.cc).+Port: [`src/tests/table.rs`](src/tests/table.rs), 23 tests.++| Upstream cases | Adaptation |+| --- | --- |+| `Harness.Empty`, `SimpleEmptyKey`, `SimpleSingle`, `SimpleMulti`, `SimpleSpecialKey` | Block and table forward scans; restart intervals 1/16/1024; raw and Snappy data/index blocks |+| `Harness.ZeroRestartPointsInBlock` | The four-byte empty block emitted by Java LevelDB |+| `Harness.Randomized` | Original 0..2000 entry-count schedule, binary-key alphabet and skewed lengths; fixed seed 306 |+| `Harness.RandomizedLongDB` | Checked-in native fixture from 100,000 writes, fixed seed 301; Rust independently reconstructs all 25,479 final entries |+| `TableTest.ApproximateOffsetOfPlain` | Original values up to 300 KB; verifies full contents instead of an offset-estimation API |+| `CompressionTableTest.ApproximateOffsetOfCompressed` | Original 25%-compressible workload; verifies Snappy-decoded contents |+| `FormatTest.InternalKey_EncodeDecode`, `InternalKey_DecodeFromEmpty` | Snapshot decoding of empty/long keys and sequence boundaries; rejection of short internal keys |++The read-only API exposes a full scan. Reverse/custom comparators, backward+iterators, seeks, memtable constructors, key-shortening/debug formatting, and+approximate offsets have no corresponding API here. The pinned Chromium+[`port_chromium.h`](https://chromium.googlesource.com/chromium/src/+/d9fa5a15858b7189aa1a70cebe811169154ebcf0/third_party/leveldatabase/port/port_chromium.h)+returns false for Zstd support, so the Zstd parameter is not ported; compression+types other than raw/Snappy are explicitly rejected.++Added regressions cover footer size/magic, file-size disagreement with MANIFEST,+out-of-bounds block handles, data/index CRCs, invalid compression with valid+CRCs, restart counts/offsets/order, illegal shared prefixes, entries crossing+restart arrays, unknown internal value types, and callback errors. Existing+snapshot tests also cover malicious Snappy lengths and missing/legacy tables.++## Recovery, versions and tombstones++Sources: [`db/recovery_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/recovery_test.cc),+[`db/db_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/db_test.cc),+[`db/corruption_test.cc`](https://chromium.googlesource.com/external/leveldb/+/7ee830d02b623e8ffe0b95d59a74db1e58da04c5/db/corruption_test.cc).+Port: [`src/tests/recovery.rs`](src/tests/recovery.rs), 21 tests.++| Upstream cases | Read-only expectation |+| --- | --- |+| `RecoveryTest.ManifestReused`, `LargeManifestCompacted` | Appended edits and a 3 MB zero-padded MANIFEST read successfully; source files stay unchanged |+| `RecoveryTest.NoLogFiles`, `LogFileReuse` | Missing logs on a valid snapshot and empty/nonempty current WALs |+| `RecoveryTest.MultipleMemTables` | Original 1,000 padded keys recovered entirely in memory |+| `RecoveryTest.MultipleLogFiles` | Recover newer unregistered WALs while ignoring obsolete logs, even with larger sequences |+| `RecoveryTest.ManifestMissing` | Fail without creating/replacing files |+| `DBTest.GetLevel0Ordering`, `GetOrderedByLevels` | Newest visible version wins across overlapping L0 files and different levels |+| `DBTest.Recover`, `RecoveryWithEmptyLog` | Repeated updates across WAL/SST transitions |+| `DBTest.IterMultiWithDelete`, `IterMultiWithDeleteAndCompaction` | Deleted entries absent from output with both WAL-only and SST+WAL data |+| `CorruptionTest.SequenceNumberRecovery` | Recovered sequence ordering keeps subsequent updates visible; repair itself is outside scope |+| `CorruptionTest.Recovery` | Corruption in the original two WAL locations returns an error instead of salvaged partial data |++These are 15 upstream scenarios. Added regressions cover native Chromium+SST/WAL compatibility, future SST versions, tombstones newer than WAL values,+empty batches, malformed batches without partial output, and duplicate numeric+WAL filenames. Recovery checks compare the input files before and after reads.++The nine existing [snapshot tests](src/tests/snapshot.rs) moved from the Chrome+importer. They retain engine comparison, mtime/content checks, a held writer+lock, read-only filesystem permissions, obsolete-file resurrection checks,+`.sst` fallback/missing live tables (`DBTest.StillReadSST`/`MissingSSTFile`+scenarios), corruption, previous WAL selection, and UTF-agnostic byte handling.+Chrome string/JSON import tests remain in `moli-cookie-import`.++Native fixtures, their C++ generator, checksums and regeneration instructions+are in [tests/fixtures](tests/fixtures/README.md). Normal tests require no C+++compiler or network. Compaction scheduling, writable recovery/repair,+concurrency, caches/Bloom-filter performance, locking policy, and write-failure+injection for a database engine are outside the parser's API and are not+claimed as ported.diff --git a/moli-leveldb-parser/src/coding.rs b/moli-leveldb-parser/src/coding.rsnew file mode 100644index 000000000..9d9e26f0f--- /dev/null+++ b/moli-leveldb-parser/src/coding.rs@@ -0,0 +1,63 @@+use anyhow::{Result, anyhow, bail, ensure};+use zerocopy::{FromBytes, Immutable, KnownLayout, Unaligned};++pub(super) struct Decoder<'a> {+ pub bytes: &'a [u8],+}++impl<'a> Decoder<'a> {+ pub fn new(bytes: &'a [u8]) -> Self {+ Self { bytes }+ }++ pub fn take(&mut self, length: usize) -> Result<&'a [u8]> {+ ensure!(length <= self.bytes.len(), "truncated LevelDB field");+ let (value, rest) = self.bytes.split_at(length);+ self.bytes = rest;+ Ok(value)+ }++ pub fn fixed<T: FromBytes + KnownLayout + Immutable + Unaligned>(&mut self) -> Result<&'a T> {+ let (value, rest) =+ T::ref_from_prefix(self.bytes).map_err(|_| anyhow!("truncated LevelDB field"))?;+ self.bytes = rest;+ Ok(value)+ }++ pub fn varint(&mut self) -> Result<u64> {+ let mut value = 0;+ for shift in (0..70).step_by(7) {+ let byte = self.take(1)?[0];+ ensure!(shift != 63 || byte <= 1, "LevelDB varint overflow");+ value |= u64::from(byte & 0x7f) << shift;+ if byte < 0x80 {+ return Ok(value);+ }+ }+ bail!("invalid LevelDB varint")+ }++ pub fn varint32(&mut self) -> Result<u32> {+ let mut value = 0;+ for shift in (0..35).step_by(7) {+ let byte = self.take(1)?[0];+ ensure!(shift != 28 || byte <= 0x0f, "LevelDB varint32 overflow");+ value |= u32::from(byte & 0x7f) << shift;+ if byte < 0x80 {+ return Ok(value);+ }+ }+ bail!("invalid LevelDB varint32")+ }++ pub fn slice(&mut self) -> Result<&'a [u8]> {+ let length = self.varint32()? as usize;+ self.take(length)+ }+}++pub(super) fn masked_crc(bytes: &[u8]) -> u32 {+ crc32c::crc32c(bytes)+ .rotate_right(15)+ .wrapping_add(0xa282_ead8)+}diff --git a/moli-leveldb-parser/src/layout.rs b/moli-leveldb-parser/src/layout.rsnew file mode 100644index 000000000..555217892--- /dev/null+++ b/moli-leveldb-parser/src/layout.rs@@ -0,0 +1,49 @@+//! Fixed on-disk layouts shared by the log, batch and table readers.+//!+//! Little-endian numeric wrappers have alignment 1, so these views work at+//! arbitrary byte offsets without packed fields or unsafe code. All bit+//! patterns are representable; readers validate tags, lengths and checksums.++use zerocopy::{+ FromBytes, Immutable, KnownLayout, Unaligned,+ byteorder::little_endian::{U16, U32, U64},+};++/// Physical record header used by both WAL and MANIFEST files.+#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]+#[repr(C)]+pub(super) struct LogHeader {+ pub checksum: U32,+ pub length: U16,+ pub record_type: u8,+}++#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]+#[repr(C)]+pub(super) struct WriteBatchHeader {+ pub sequence: U64,+ pub count: U32,+}++#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]+#[repr(C)]+pub(super) struct BlockTrailer {+ pub compression: u8,+ pub checksum: U32,+}++#[derive(FromBytes, KnownLayout, Immutable, Unaligned)]+#[repr(C)]+pub(super) struct TableFooter {+ /// Two varint-encoded block handles followed by padding to 40 bytes.+ pub block_handles: [u8; 40],+ pub magic: U64,+}++// Keep the structs tied to the file format even if fields are edited later.+const _: () = {+ assert!(size_of::<LogHeader>() == 7);+ assert!(size_of::<WriteBatchHeader>() == 12);+ assert!(size_of::<BlockTrailer>() == 5);+ assert!(size_of::<TableFooter>() == 48);+};diff --git a/moli-leveldb-parser/src/lib.rs b/moli-leveldb-parser/src/lib.rsnew file mode 100644index 000000000..c124fbf11--- /dev/null+++ b/moli-leveldb-parser/src/lib.rs@@ -0,0 +1,151 @@+//! Read the current contents of a stable LevelDB snapshot without opening+//! a database engine. Only CURRENT, its MANIFEST, live tables and recoverable WAL+//! files are read. Recovery happens in memory; no locks, logs, tables, manifests+//! or compactions are created, including when the reader is dropped.+//!+//! Formats: https://github.com/google/leveldb/tree/main/doc+//! Manifest tags: https://github.com/google/leveldb/blob/main/db/version_edit.cc++mod coding;+mod layout;+mod log;+mod manifest;+mod table;++use std::{+ collections::{BTreeMap, btree_map::Entry},+ fs,+ path::Path,+};++use anyhow::{Context, Result, anyhow, bail, ensure};+use zerocopy::{FromBytes, byteorder::little_endian::U64};++use coding::Decoder;+use layout::WriteBatchHeader;+use manifest::Manifest;++const MAX_SEQUENCE: u64 = (1 << 56) - 1;+type Versions = BTreeMap<Vec<u8>, (u64, Option<Vec<u8>>)>;++/// Read the latest key/value pairs from an existing, stable LevelDB directory.+///+/// The caller must keep the directory unchanged for the duration of this call+/// (for example, by copying a closed database). This function acquires no locks+/// and never modifies the directory. It supports the bytewise comparator and+/// uncompressed or Snappy-compressed tables, including legacy `.sst` filenames.+/// Deleted keys and obsolete files are excluded; complete WAL records are+/// replayed in memory. An unfinished append at EOF is ignored.+///+/// # Errors+///+/// Returns an error for I/O failures, missing live tables, corrupt complete+/// records, or unsupported formats. No partial result is returned on failure.+pub fn read_snapshot(directory: &Path) -> Result<BTreeMap<Vec<u8>, Vec<u8>>> {+ let manifest = Manifest::read(directory)?;+ let mut versions = Versions::new();+ let mut logs = BTreeMap::new();+ for entry in fs::read_dir(directory)? {+ let entry = entry?;+ let name = entry.file_name();+ let Some(name) = name.to_str().and_then(|name| name.strip_suffix(".log")) else {+ continue;+ };+ if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) {+ continue;+ }+ let number = name.parse::<u64>().context("invalid LevelDB log number")?;+ if number >= manifest.log_number || number == manifest.previous_log_number {+ ensure!(+ logs.insert(number, entry.path()).is_none(),+ "duplicate LevelDB log number"+ );+ }+ }+ let mut last_sequence = manifest.last_sequence;+ for path in logs.values() {+ log::read_records(path, |record| {+ if let Some(sequence) = read_batch(record, &mut versions)? {+ last_sequence = last_sequence.max(sequence);+ }+ Ok(())+ })+ .with_context(|| format!("failed to read LevelDB WAL `{}`", path.display()))?;+ }+ // WAL recovery establishes the latest visible sequence, just as a normal+ // database open does. SST versions newer than that snapshot are invisible.+ for ((_, number), size) in manifest.tables {+ let mut path = directory.join(format!("{number:06}.ldb"));+ if !path.try_exists()? {+ path = directory.join(format!("{number:06}.sst"));+ }+ table::read_table(&path, size, |key, value| {+ let (key, tag) =+ U64::ref_from_suffix(key).map_err(|_| anyhow!("truncated LevelDB internal key"))?;+ let tag = tag.get();+ let value = match tag & 0xff {+ 0 => None,+ 1 => Some(value),+ other => bail!("unsupported LevelDB value type {other}"),+ };+ if tag >> 8 <= last_sequence {+ apply(&mut versions, key, tag >> 8, value)?;+ }+ Ok(())+ })+ .with_context(|| format!("failed to read LevelDB table `{}`", path.display()))?;+ }+ Ok(versions+ .into_iter()+ .filter_map(|(key, (_, value))| value.map(|value| (key, value)))+ .collect())+}++fn read_batch(record: &[u8], versions: &mut Versions) -> Result<Option<u64>> {+ let mut input = Decoder::new(record);+ let header = input.fixed::<WriteBatchHeader>()?;+ let sequence = header.sequence.get();+ let count = header.count.get();+ ensure!(+ sequence <= MAX_SEQUENCE && u64::from(count.saturating_sub(1)) <= MAX_SEQUENCE - sequence,+ "LevelDB batch sequence overflow"+ );+ for index in 0..count {+ let kind = input.take(1)?[0];+ let key = input.slice()?;+ let value = match kind {+ 0 => None,+ 1 => Some(input.slice()?),+ other => bail!("unsupported LevelDB batch value type {other}"),+ };+ apply(versions, key, sequence + u64::from(index), value)?;+ }+ ensure!(+ input.bytes.is_empty(),+ "LevelDB batch record count mismatch"+ );+ Ok((count > 0).then_some(sequence + u64::from(count.saturating_sub(1))))+}++fn apply(versions: &mut Versions, key: &[u8], sequence: u64, value: Option<&[u8]>) -> Result<()> {+ match versions.entry(key.to_vec()) {+ Entry::Vacant(entry) => {+ entry.insert((sequence, value.map(<[u8]>::to_vec)));+ }+ Entry::Occupied(mut entry) => {+ let (previous_sequence, previous_value) = entry.get();+ if *previous_sequence == sequence {+ ensure!(+ previous_value.as_deref() == value,+ "conflicting LevelDB values at the same sequence"+ );+ } else if *previous_sequence < sequence {+ entry.insert((sequence, value.map(<[u8]>::to_vec)));+ }+ }+ }+ Ok(())+}++#[cfg(test)]+mod tests;diff --git a/moli-leveldb-parser/src/log.rs b/moli-leveldb-parser/src/log.rsnew file mode 100644index 000000000..3bf4bbff4--- /dev/null+++ b/moli-leveldb-parser/src/log.rs@@ -0,0 +1,114 @@+use std::{fs::File, io::Read, path::Path};++use anyhow::{Context, Result, bail, ensure};++use super::{+ coding::{Decoder, masked_crc},+ layout::LogHeader,+};++const BLOCK_SIZE: usize = 32 * 1024;+const HEADER_SIZE: usize = size_of::<LogHeader>();++/// Read complete logical records, including records spanning physical blocks.+/// Like LevelDB recovery, ignore an unfinished append at EOF. A complete record+/// with a bad checksum is an error, rather than a silently incomplete import.+pub(super) fn read_records(path: &Path, visit: impl FnMut(&[u8]) -> Result<()>) -> Result<()> {+ read_records_from(File::open(path)?, visit)+}++pub(super) fn read_records_from(+ mut file: impl Read,+ mut visit: impl FnMut(&[u8]) -> Result<()>,+) -> Result<()> {+ let mut block = Vec::with_capacity(BLOCK_SIZE);+ let mut fragment = Vec::new();+ let mut fragmented = false;+ loop {+ block.clear();+ file.by_ref()+ .take(BLOCK_SIZE as u64)+ .read_to_end(&mut block)?;+ if block.is_empty() {+ return Ok(());+ }+ let mut offset = 0;+ while offset < block.len() {+ let remaining = &block[offset..];+ if remaining.len() < HEADER_SIZE {+ ensure!(+ block.len() < BLOCK_SIZE || remaining.iter().all(|byte| *byte == 0),+ "nonzero LevelDB log block trailer"+ );+ break;+ }+ let header = Decoder::new(remaining).fixed::<LogHeader>()?;+ let length = usize::from(header.length.get());+ let kind = header.record_type;+ if kind == 0 && length == 0 {+ // Writers may preallocate zero-filled blocks.+ ensure!(+ remaining.iter().all(|byte| *byte == 0),+ "invalid LevelDB log padding"+ );+ fragment.clear();+ fragmented = false;+ break;+ }+ if HEADER_SIZE + length > remaining.len() {+ ensure!(+ block.len() < BLOCK_SIZE,+ "LevelDB log record crosses a block boundary"+ );+ return Ok(());+ }+ let payload = &remaining[HEADER_SIZE..HEADER_SIZE + length];+ ensure!(+ // The CRC covers the record type and payload, excluding length.+ masked_crc(+ &remaining[std::mem::offset_of!(LogHeader, record_type)..HEADER_SIZE + length]+ ) == header.checksum.get(),+ "LevelDB log checksum mismatch"+ );+ match kind {+ 1 => {+ // Older writers could leave an empty FIRST at a block's+ // end, then start a new record in the next block. Match+ // LevelDB's compatibility exception only for empty data.+ ensure!(+ fragment.is_empty(),+ "unfinished LevelDB log record before FULL"+ );+ fragmented = false;+ visit(payload)?;+ }+ 2 => {+ ensure!(+ fragment.is_empty(),+ "unfinished LevelDB log record before FIRST"+ );+ fragment.clear();+ fragment+ .try_reserve(payload.len())+ .context("LevelDB log record is too large")?;+ fragment.extend_from_slice(payload);+ fragmented = true;+ }+ 3 | 4 => {+ ensure!(fragmented, "LevelDB log fragment without FIRST");+ fragment+ .try_reserve(payload.len())+ .context("LevelDB log record is too large")?;+ fragment.extend_from_slice(payload);+ if kind == 4 {+ visit(&fragment)?;+ fragment.clear();+ fragmented = false;+ }+ }+ _ => bail!("unsupported LevelDB log record type {kind}"),+ }+ offset += HEADER_SIZE + length;+ }+ }+}diff --git a/moli-leveldb-parser/src/manifest.rs b/moli-leveldb-parser/src/manifest.rsnew file mode 100644index 000000000..4d1fa4b80--- /dev/null+++ b/moli-leveldb-parser/src/manifest.rs@@ -0,0 +1,97 @@+use std::{collections::BTreeMap, fs, path::Path};++use anyhow::{Context, Result, bail, ensure};++use super::{MAX_SEQUENCE, coding::Decoder, log::read_records};++#[derive(Default)]+pub(super) struct Manifest {+ pub tables: BTreeMap<(u32, u64), u64>,+ pub log_number: u64,+ pub previous_log_number: u64,+ pub last_sequence: u64,+}++impl Manifest {+ pub fn read(directory: &Path) -> Result<Self> {+ let current = fs::read_to_string(directory.join("CURRENT"))+ .context("failed to read LevelDB CURRENT")?;+ let name = current+ .strip_suffix('\n')+ .context("LevelDB CURRENT must end with a newline")?;+ let number = name+ .strip_prefix("MANIFEST-")+ .context("invalid LevelDB CURRENT manifest name")?;+ ensure!(+ !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()),+ "invalid LevelDB CURRENT manifest name"+ );+ let mut manifest = Self::default();+ let mut has_log = false;+ let mut has_next_file = false;+ let mut has_last_sequence = false;+ read_records(&directory.join(name), |record| {+ let mut input = Decoder::new(record);+ let mut deleted = Vec::new();+ let mut added = Vec::new();+ while !input.bytes.is_empty() {+ match input.varint32()? {+ 1 => ensure!(+ input.slice()? == b"leveldb.BytewiseComparator",+ "unsupported LevelDB comparator"+ ),+ 2 => {+ manifest.log_number = input.varint()?;+ has_log = true;+ }+ 3 => {+ input.varint()?;+ has_next_file = true;+ }+ 4 => {+ manifest.last_sequence = input.varint()?;+ ensure!(+ manifest.last_sequence <= MAX_SEQUENCE,+ "invalid LevelDB sequence number"+ );+ has_last_sequence = true;+ }+ 5 => {+ read_level(&mut input)?;+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB compaction key");+ }+ 6 => deleted.push((read_level(&mut input)?, input.varint()?)),+ 7 => {+ let level = read_level(&mut input)?;+ let number = input.varint()?;+ let size = input.varint()?;+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB smallest key");+ ensure!(input.slice()?.len() >= 8, "invalid LevelDB largest key");+ added.push(((level, number), size));+ }+ 9 => manifest.previous_log_number = input.varint()?,+ tag => bail!("unsupported LevelDB manifest tag {tag}"),+ }+ }+ // A VersionEdit applies deletions before additions, including a+ // table moved between levels without rewriting its contents.+ for key in deleted {+ manifest.tables.remove(&key);+ }+ manifest.tables.extend(added);+ Ok(())+ })+ .with_context(|| format!("failed to read LevelDB manifest `{name}`"))?;+ ensure!(+ has_log && has_next_file && has_last_sequence,+ "incomplete LevelDB manifest"+ );+ Ok(manifest)+ }+}++fn read_level(input: &mut Decoder<'_>) -> Result<u32> {+ let level = input.varint32()?;+ ensure!(level < 7, "unsupported LevelDB level {level}");+ Ok(level)+}diff --git a/moli-leveldb-parser/src/table.rs b/moli-leveldb-parser/src/table.rsnew file mode 100644index 000000000..85f4b2147--- /dev/null+++ b/moli-leveldb-parser/src/table.rs@@ -0,0 +1,168 @@+use std::{+ fs::File,+ io::{Read, Seek, SeekFrom},+ path::Path,+};++use anyhow::{Context, Result, anyhow, bail, ensure};+use zerocopy::{FromBytes, byteorder::little_endian::U32};++use super::{+ coding::{Decoder, masked_crc},+ layout::{BlockTrailer, TableFooter},+};++const FOOTER_SIZE: u64 = size_of::<TableFooter>() as u64;+const TRAILER_SIZE: u64 = size_of::<BlockTrailer>() as u64;+const TABLE_MAGIC: u64 = 0xdb47_7524_8b80_fb57;++pub(super) fn read_table(+ path: &Path,+ expected_size: u64,+ mut visit: impl FnMut(&[u8], &[u8]) -> Result<()>,+) -> Result<()> {+ let mut file = File::open(path)?;+ let size = file.metadata()?.len();+ ensure!(+ size == expected_size && size >= FOOTER_SIZE,+ "invalid LevelDB table size"+ );+ file.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?;+ let mut footer = [0; FOOTER_SIZE as usize];+ file.read_exact(&mut footer)?;+ let footer = Decoder::new(&footer).fixed::<TableFooter>()?;+ ensure!(+ footer.magic.get() == TABLE_MAGIC,+ "invalid LevelDB table magic"+ );+ let mut footer = Decoder::new(&footer.block_handles);+ // Full scans don't need the metaindex or Bloom filters.+ read_handle(&mut footer)?;+ let index_handle = read_handle(&mut footer)?;+ let index = read_block(&mut file, size - FOOTER_SIZE, index_handle)?;+ read_entries(&index, |_, handle| {+ let mut input = Decoder::new(handle);+ let handle = read_handle(&mut input)?;+ let block = read_block(&mut file, index_handle.0, handle)?;+ read_entries(&block, &mut visit)+ })+}++fn read_handle(input: &mut Decoder<'_>) -> Result<(u64, u64)> {+ Ok((input.varint()?, input.varint()?))+}++fn read_block(file: &mut File, end: u64, (offset, size): (u64, u64)) -> Result<Vec<u8>> {+ ensure!(+ offset <= end && end - offset >= TRAILER_SIZE && size <= end - offset - TRAILER_SIZE,+ "LevelDB block lies outside its table"+ );+ let size = usize::try_from(size).context("LevelDB block is too large")?;+ let length = size+ .checked_add(TRAILER_SIZE as usize)+ .context("LevelDB block length overflow")?;+ let mut block = Vec::new();+ block+ .try_reserve_exact(length)+ .context("LevelDB block is too large")?;+ block.resize(length, 0);+ file.seek(SeekFrom::Start(offset))?;+ file.read_exact(&mut block)?;+ let trailer = Decoder::new(&block[size..]).fixed::<BlockTrailer>()?;+ let checksum_end = size + std::mem::offset_of!(BlockTrailer, checksum);+ ensure!(+ masked_crc(&block[..checksum_end]) == trailer.checksum.get(),+ "LevelDB table block checksum mismatch"+ );+ let compression = trailer.compression;+ block.truncate(size);+ match compression {+ 0 => Ok(block),+ 1 => {+ let length = snap::raw::decompress_len(&block)?;+ // A Snappy copy encodes at most 64 output bytes. Validate the+ // claimed size before letting the decoder allocate from its header.+ ensure!(+ length <= block.len().saturating_mul(64),+ "invalid Snappy decoded length"+ );+ let mut decoded = Vec::new();+ decoded+ .try_reserve_exact(length)+ .context("decoded LevelDB block is too large")?;+ decoded.resize(length, 0);+ let written = snap::raw::Decoder::new().decompress(&block, &mut decoded)?;+ ensure!(written == length, "invalid Snappy decoded length");+ Ok(decoded)+ }+ other => bail!("unsupported LevelDB compression type {other}"),+ }+}++/// Borrow the entries and restart offsets directly from a block, including at+/// unaligned byte offsets. Zerocopy checks the count and buffer size together.+pub(super) fn split_restarts(block: &[u8]) -> Result<(&[u8], &[U32])> {+ let (body, count) = U32::ref_from_suffix(block)+ .map_err(|_| anyhow!("truncated LevelDB block restart array"))?;+ let (entries, restarts) = <[U32]>::ref_from_suffix_with_elems(body, count.get() as usize)+ .map_err(|_| anyhow!("invalid LevelDB restart count"))?;+ // Java LevelDB can emit an empty block containing only a zero count.+ if restarts.is_empty() {+ ensure!(+ entries.is_empty(),+ "nonempty LevelDB block has no restart points"+ );+ }+ Ok((entries, restarts))+}++pub(super) fn read_entries(+ block: &[u8],+ mut visit: impl FnMut(&[u8], &[u8]) -> Result<()>,+) -> Result<()> {+ let (entries, restarts) = split_restarts(block)?;+ let entries_end = entries.len();+ let count = restarts.len();+ let mut previous = None;+ for restart in restarts {+ let offset = restart.get() as usize;+ ensure!(+ offset < entries_end || (entries_end == 0 && count == 1 && offset == 0),+ "LevelDB restart offset out of bounds"+ );+ if let Some(previous) = previous {+ ensure!(previous < offset, "unordered LevelDB restart offsets");+ } else {+ ensure!(offset == 0, "first LevelDB restart offset must be zero");+ }+ previous = Some(offset);+ }+ let mut next_restart = 0;+ let mut input = Decoder::new(entries);+ let mut key = Vec::new();+ while !input.bytes.is_empty() {+ let offset = entries_end - input.bytes.len();+ let shared = input.varint32()? as usize;+ let unshared = input.varint32()? as usize;+ let value_length = input.varint32()? as usize;+ ensure!(shared <= key.len(), "invalid LevelDB key prefix length");+ if let Some(restart) = restarts.get(next_restart) {+ let restart = restart.get() as usize;+ ensure!(restart >= offset, "LevelDB restart points inside an entry");+ if restart == offset {+ ensure!(shared == 0, "LevelDB restart entry has a shared prefix");+ next_restart += 1;+ }+ }+ let suffix = input.take(unshared)?;+ let value = input.take(value_length)?;+ key.truncate(shared);+ key.extend_from_slice(suffix);+ visit(&key, value)?;+ }+ ensure!(+ entries_end == 0 || next_restart == count,+ "LevelDB restart points inside an entry"+ );+ Ok(())+}diff --git a/moli-leveldb-parser/src/tests/coding.rs b/moli-leveldb-parser/src/tests/coding.rsnew file mode 100644index 000000000..b2c3d0984--- /dev/null+++ b/moli-leveldb-parser/src/tests/coding.rs@@ -0,0 +1,225 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Ports of util/coding_test.cc and util/crc32c_test.cc; see ../../TESTS.md,+// ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.++use zerocopy::byteorder::little_endian::{U32, U64};++use super::support::{slice, varint};+use crate::coding::{Decoder, masked_crc};++#[test]+fn fixed32() {+ let bytes: Vec<_> = (0..100_000u32).flat_map(u32::to_le_bytes).collect();+ let mut input = Decoder::new(&bytes);+ for expected in 0..100_000 {+ assert_eq!(input.fixed::<U32>().unwrap().get(), expected);+ }+ assert!(input.bytes.is_empty());+}++#[test]+fn fixed64() {+ let values: Vec<_> = (0..64)+ .flat_map(|power| {+ let v = 1u64 << power;+ [v - 1, v, v + 1]+ })+ .collect();+ let bytes: Vec<_> = values.iter().flat_map(|v| v.to_le_bytes()).collect();+ let mut input = Decoder::new(&bytes);+ for expected in values {+ assert_eq!(input.fixed::<U64>().unwrap().get(), expected);+ }+ assert!(input.bytes.is_empty());+}++#[test]+fn encoding_output() {+ assert_eq!(+ Decoder::new(&[1, 2, 3, 4]).fixed::<U32>().unwrap().get(),+ 0x0403_0201+ );+ assert_eq!(+ Decoder::new(&[1, 2, 3, 4, 5, 6, 7, 8])+ .fixed::<U64>()+ .unwrap()+ .get(),+ 0x0807_0605_0403_0201+ );+}++#[test]+fn varint32() {+ let values: Vec<_> = (0..32 * 32u32).map(|i| (i / 32) << (i % 32)).collect();+ let mut bytes = Vec::new();+ for value in &values {+ varint(u64::from(*value), &mut bytes);+ }+ let mut input = Decoder::new(&bytes);+ for value in values {+ let before = input.bytes.len();+ assert_eq!(input.varint32().unwrap(), value);+ assert_eq!(+ before - input.bytes.len(),+ ((32 - value.leading_zeros()).max(1).div_ceil(7)) as usize+ );+ }+ assert!(input.bytes.is_empty());+}++#[test]+fn varint64() {+ let mut values = vec![0, 100, u64::MAX, u64::MAX - 1];+ values.extend((0..64).flat_map(|k| {+ let v = 1u64 << k;+ [v, v - 1, v + 1]+ }));+ let mut bytes = Vec::new();+ for value in &values {+ varint(*value, &mut bytes);+ }+ let mut input = Decoder::new(&bytes);+ for value in values {+ let before = input.bytes.len();+ assert_eq!(input.varint().unwrap(), value);+ assert_eq!(+ before - input.bytes.len(),+ ((64 - value.leading_zeros()).max(1).div_ceil(7)) as usize+ );+ }+ assert!(input.bytes.is_empty());+}++#[test]+fn varint32_overflow() {+ assert!(+ Decoder::new(&[0x81, 0x82, 0x83, 0x84, 0x85, 0x11])+ .varint32()+ .is_err()+ );+}++#[test]+fn varint32_truncation() {+ let value = (1u32 << 31) + 100;+ let mut bytes = Vec::new();+ varint(u64::from(value), &mut bytes);+ for end in 0..bytes.len() {+ assert!(Decoder::new(&bytes[..end]).varint32().is_err());+ }+ assert_eq!(Decoder::new(&bytes).varint32().unwrap(), value);+}++#[test]+fn varint64_overflow() {+ assert!(+ Decoder::new(&[+ 0x81, 0x82, 0x83, 0x84, 0x85, 0x81, 0x82, 0x83, 0x84, 0x85, 0x11+ ])+ .varint()+ .is_err()+ );+}++#[test]+fn varint64_truncation() {+ let value = (1u64 << 63) + 100;+ let mut bytes = Vec::new();+ varint(value, &mut bytes);+ for end in 0..bytes.len() {+ assert!(Decoder::new(&bytes[..end]).varint().is_err());+ }+ assert_eq!(Decoder::new(&bytes).varint().unwrap(), value);+}++#[test]+fn strings() {+ let values = [vec![], b"foo".to_vec(), b"bar".to_vec(), vec![b'x'; 200]];+ let mut bytes = Vec::new();+ for value in &values {+ slice(value, &mut bytes);+ }+ let mut input = Decoder::new(&bytes);+ for value in values {+ assert_eq!(input.slice().unwrap(), value);+ }+ assert!(input.bytes.is_empty());+}++#[test]+fn crc_standard_results() {+ let vectors = [+ (vec![0; 32], 0x8a91_36aa_u32),+ (vec![0xff; 32], 0x62a8_ab43),+ ((0..32).collect(), 0x46dd_794e),+ ((0..32).rev().collect(), 0x113f_db5c),+ (+ vec![+ 0x01, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x14, 0, 0, 0, 0, 0, 0x04, 0,+ 0, 0, 0, 0x14, 0, 0, 0, 0x18, 0x28, 0, 0, 0, 0, 0, 0, 0, 0x02, 0, 0, 0, 0, 0, 0, 0,+ ],+ 0xd996_3a56,+ ),+ ];+ for (bytes, expected) in vectors {+ // Check the actual parser wrapper, including its masking step.+ assert_eq!(unmask(masked_crc(&bytes)), expected);+ }+}++fn unmask(value: u32) -> u32 {+ value.wrapping_sub(0xa282_ead8).rotate_left(15)+}++#[test]+fn crc_values() {+ assert_ne!(masked_crc(b"a"), masked_crc(b"foo"));+}++#[test]+fn crc_mask() {+ let crc = crc32c::crc32c(b"foo");+ assert_ne!(crc, masked_crc(b"foo"));+ assert_eq!(crc, unmask(masked_crc(b"foo")));+}++#[test]+fn fixed_fields_cannot_read_past_end() {+ for length in 0..8 {+ assert!(Decoder::new(&[0; 8][..length]).fixed::<U64>().is_err());+ }+}++#[test]+fn length_prefixed_fields_are_bounded() {+ for bytes in [+ vec![0xff, 0xff, 0xff, 0xff, 0x0f],+ vec![3, b'a', b'b'],+ vec![0x80],+ ] {+ assert!(Decoder::new(&bytes).slice().is_err());+ }+}++#[test]+fn varint32_overlong_zero_is_rejected() {+ assert!(+ Decoder::new(&[0x80, 0x80, 0x80, 0x80, 0x80, 0])+ .varint32()+ .is_err()+ );+}++#[test]+fn varint_high_bits_cannot_wrap() {+ assert!(+ Decoder::new(&[0xff, 0xff, 0xff, 0xff, 0x10])+ .varint32()+ .is_err()+ );+ assert!(+ Decoder::new(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2])+ .varint()+ .is_err()+ );+}diff --git a/moli-leveldb-parser/src/tests/layout.rs b/moli-leveldb-parser/src/tests/layout.rsnew file mode 100644index 000000000..aeac6e210--- /dev/null+++ b/moli-leveldb-parser/src/tests/layout.rs@@ -0,0 +1,98 @@+use zerocopy::{FromBytes, Immutable, KnownLayout, Unaligned};++use crate::{+ coding::Decoder,+ layout::{BlockTrailer, LogHeader, TableFooter, WriteBatchHeader},+ table::{read_entries, split_restarts},+};++fn at_each_alignment(bytes: &[u8], mut visit: impl FnMut(&[u8])) {+ let mut storage = vec![0xff; bytes.len() + 8];+ for offset in 0..8 {+ let input = &mut storage[offset..offset + bytes.len()];+ input.copy_from_slice(bytes);+ visit(input);+ }+}++fn borrowed<T: FromBytes + KnownLayout + Immutable + Unaligned>(input: &[u8]) -> &T {+ let mut decoder = Decoder::new(input);+ let value = decoder.fixed::<T>().unwrap();+ assert_eq!(std::ptr::from_ref(value).cast::<u8>(), input.as_ptr());+ // The header stays borrowed while the cursor advances over its payload.+ let payload = decoder.take(input.len() - size_of::<T>()).unwrap();+ assert_eq!(payload.as_ptr(), input[size_of::<T>()..].as_ptr());+ assert!(decoder.bytes.is_empty());+ value+}++#[test]+fn headers_borrow_unaligned_little_endian_fields() {+ // Literal format bytes, independent of the fixture writers and derives.+ at_each_alignment(&[1, 2, 3, 4, 5, 6, 0xfe, 0xaa], |input| {+ let header = borrowed::<LogHeader>(input);+ assert_eq!(header.checksum.get(), 0x0403_0201);+ assert_eq!(header.length.get(), 0x0605);+ // Unknown types remain representable for the reader to validate.+ assert_eq!(header.record_type, 0xfe);+ });+ at_each_alignment(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0xaa], |input| {+ let header = borrowed::<WriteBatchHeader>(input);+ assert_eq!(header.sequence.get(), 0x0807_0605_0403_0201);+ assert_eq!(header.count.get(), 0x0c0b_0a09);+ });+ at_each_alignment(&[0xfe, 1, 2, 3, 4], |input| {+ let trailer = borrowed::<BlockTrailer>(input);+ assert_eq!(trailer.compression, 0xfe);+ assert_eq!(trailer.checksum.get(), 0x0403_0201);+ });+ let mut footer = [0x10; 48];+ footer[40..].copy_from_slice(&[0x57, 0xfb, 0x80, 0x8b, 0x24, 0x75, 0x47, 0xdb]);+ at_each_alignment(&footer, |input| {+ let footer = borrowed::<TableFooter>(input);+ assert_eq!(footer.block_handles, [0x10; 40]);+ assert_eq!(footer.magic.get(), 0xdb47_7524_8b80_fb57);+ });+}++fn check_truncations<T: FromBytes + KnownLayout + Immutable + Unaligned>() {+ for length in 0..size_of::<T>() {+ let bytes = vec![0; length];+ let mut decoder = Decoder::new(&bytes);+ assert!(decoder.fixed::<T>().is_err());+ assert_eq!(decoder.bytes, bytes);+ assert_eq!(decoder.bytes.as_ptr(), bytes.as_ptr());+ }+}++#[test]+fn truncated_headers_do_not_consume_input() {+ check_truncations::<LogHeader>();+ check_truncations::<WriteBatchHeader>();+ check_truncations::<BlockTrailer>();+ check_truncations::<TableFooter>();+}++#[test]+fn restart_offsets_borrow_the_original_block_at_any_alignment() {+ // Three body bytes, two little-endian restart offsets, and their count.+ let bytes = [0xa1, 0xa2, 0xa3, 0, 0, 0, 0, 1, 2, 3, 4, 2, 0, 0, 0];+ at_each_alignment(&bytes, |input| {+ let (body, restarts) = split_restarts(input).unwrap();+ assert_eq!(body, [0xa1, 0xa2, 0xa3]);+ assert_eq!(body.as_ptr(), input.as_ptr());+ assert_eq!(restarts.len(), 2);+ assert_eq!(restarts.as_ptr().cast::<u8>(), input[3..].as_ptr());+ assert_eq!(restarts[0].get(), 0);+ assert_eq!(restarts[1].get(), 0x0403_0201);+ });+}++#[test]+fn nonempty_block_cannot_hide_entries_with_zero_restarts() {+ for body in [b"x".as_slice(), &[0; 4], &[0, 1, 1, b'k', b'v']] {+ let mut bytes = body.to_vec();+ bytes.extend_from_slice(&[0; 4]);+ assert!(read_entries(&bytes, |_, _| panic!("invalid block must be rejected")).is_err());+ }+}diff --git a/moli-leveldb-parser/src/tests/log.rs b/moli-leveldb-parser/src/tests/log.rsnew file mode 100644index 000000000..49b01b5fd--- /dev/null+++ b/moli-leveldb-parser/src/tests/log.rs@@ -0,0 +1,318 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Rust ports of db/log_test.cc. See ../../LICENSE-LevelDB, ../../AUTHORS-LevelDB+// and ../../TESTS.md. Corruption cases assert Err instead of reporter/salvage.++use std::io::{self, Cursor, Read};++use anyhow::Result;++use super::support::*;+use crate::log::read_records_from;++fn read(bytes: &[u8]) -> Result<Vec<Vec<u8>>> {+ let mut records = Vec::new();+ read_records_from(Cursor::new(bytes), |record| {+ records.push(record.to_vec());+ Ok(())+ })?;+ Ok(records)+}++fn roundtrip(records: &[Vec<u8>]) {+ assert_eq!(read(&log_bytes(records)).unwrap(), records);+}++fn error(bytes: &[u8], expected: &str) {+ let error = format!("{:#}", read(bytes).unwrap_err());+ assert!(error.contains(expected), "{error}");+}++#[test]+fn empty() {+ roundtrip(&[]);+}++#[test]+fn read_write() {+ roundtrip(&[b"foo".to_vec(), b"bar".to_vec(), vec![], b"xxxx".to_vec()]);+}++#[test]+fn many_blocks() {+ let records: Vec<_> = (0..100_000).map(|i| format!("{i}.").into_bytes()).collect();+ roundtrip(&records);+}++#[test]+fn fragmentation() {+ roundtrip(&[+ b"small".to_vec(),+ big_string(b"medium", 50_000),+ big_string(b"large", 100_000),+ ]);+}++#[test]+fn marginal_trailer() {+ roundtrip(&[+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE),+ vec![],+ b"bar".to_vec(),+ ]);+}++#[test]+fn marginal_trailer2() {+ roundtrip(&[+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE),+ b"bar".to_vec(),+ ]);+}++#[test]+fn short_trailer() {+ roundtrip(&[+ big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE + 4),+ vec![],+ b"bar".to_vec(),+ ]);+}++#[test]+fn aligned_eof() {+ roundtrip(&[big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE + 4)]);+}++#[test]+fn open_for_append() {+ let mut bytes = log_bytes(&[b"hello".to_vec()]);+ append_log(&mut bytes, b"world");+ assert_eq!(read(&bytes).unwrap(), [b"hello", b"world"]);+}++#[test]+fn random_read() {+ let mut random = Random::new(301);+ let records: Vec<_> = (0..500)+ .map(|i| big_string(format!("{i}.").as_bytes(), random.skewed(17)))+ .collect();+ roundtrip(&records);+}++#[test]+fn read_error() {+ struct Broken;+ impl Read for Broken {+ fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {+ Err(io::Error::other("injected read error"))+ }+ }+ let result = read_records_from(Broken, |_| panic!("read error must propagate"));+ assert!(format!("{:#}", result.unwrap_err()).contains("injected read error"));+}++#[test]+fn bad_record_type() {+ let mut bytes = log_bytes(&[b"foo".to_vec()]);+ bytes[6] += 100;+ fix_log_checksum(&mut bytes, 0);+ error(&bytes, "record type");+}++#[test]+fn truncated_trailing_record_is_ignored() {+ let mut bytes = log_bytes(&[b"foo".to_vec()]);+ bytes.truncate(bytes.len() - 4);+ assert!(read(&bytes).unwrap().is_empty());+}++#[test]+fn bad_length() {+ let mut bytes = log_bytes(&[+ big_string(b"bar", BLOCK_SIZE - HEADER_SIZE),+ b"foo".to_vec(),+ ]);+ bytes[4] += 1;+ error(&bytes, "block boundary");+}++#[test]+fn bad_length_at_end_is_ignored() {+ let mut bytes = log_bytes(&[b"foo".to_vec()]);+ bytes.pop();+ assert!(read(&bytes).unwrap().is_empty());+}++#[test]+fn checksum_mismatch() {+ let mut bytes = log_bytes(&[b"foo".to_vec()]);+ bytes[0] = bytes[0].wrapping_add(10);+ error(&bytes, "checksum mismatch");+}++#[test]+fn unexpected_middle_type() {+ error(&physical_record(3, b"foo"), "without FIRST");+}++#[test]+fn unexpected_last_type() {+ error(&physical_record(4, b"foo"), "without FIRST");+}++#[test]+fn unexpected_full_type() {+ let mut bytes = log_bytes(&[b"foo".to_vec(), b"bar".to_vec()]);+ bytes[6] = 2;+ fix_log_checksum(&mut bytes, 0);+ error(&bytes, "before FULL");+}++#[test]+fn unexpected_first_type() {+ let mut bytes = log_bytes(&[b"foo".to_vec(), big_string(b"bar", 100_000)]);+ bytes[6] = 2;+ fix_log_checksum(&mut bytes, 0);+ error(&bytes, "before FIRST");+}++#[test]+fn missing_last_is_ignored() {+ let mut bytes = log_bytes(&[big_string(b"bar", BLOCK_SIZE)]);+ bytes.truncate(bytes.len() - 14);+ assert!(read(&bytes).unwrap().is_empty());+}++#[test]+fn partial_last_is_ignored() {+ let mut bytes = log_bytes(&[big_string(b"bar", BLOCK_SIZE)]);+ bytes.pop();+ assert!(read(&bytes).unwrap().is_empty());+}++#[test]+fn error_joins_records() {+ let mut bytes = log_bytes(&[+ big_string(b"foo", BLOCK_SIZE),+ big_string(b"bar", BLOCK_SIZE),+ b"correct".to_vec(),+ ]);+ bytes[BLOCK_SIZE..2 * BLOCK_SIZE].fill(b'x');+ let mut seen = Vec::new();+ assert!(+ read_records_from(Cursor::new(bytes), |record| {+ seen.push(record.to_vec());+ Ok(())+ })+ .is_err()+ );+ assert!(+ seen.is_empty(),+ "fragments from different records must never be joined"+ );+}++// Additional regressions for the compatibility behavior documented in the+// pinned db/log_reader.cc and for the snapshot reader's strict error policy.+#[test]+fn legacy_empty_first_before_full_or_first() {+ for record in [b"full".to_vec(), big_string(b"fragmented", 100_000)] {+ let first = big_string(b"foo", BLOCK_SIZE - 2 * HEADER_SIZE);+ let mut bytes = log_bytes(std::slice::from_ref(&first));+ bytes.extend_from_slice(&physical_record(2, b""));+ append_log(&mut bytes, &record);+ assert_eq!(read(&bytes).unwrap(), [first, record]);+ }+}++#[test]+fn all_fragment_checksums_are_verified() {+ let original = log_bytes(&[big_string(b"payload", 3 * BLOCK_SIZE)]);+ for offset in [0, BLOCK_SIZE, 2 * BLOCK_SIZE, 3 * BLOCK_SIZE] {+ let mut bytes = original.clone();+ bytes[offset + HEADER_SIZE] ^= 1;+ error(&bytes, "checksum mismatch");+ }+}++#[test]+fn truncated_tail_never_emits_a_partial_logical_record() {+ let prefix = log_bytes(&[b"committed".to_vec()]);+ let mut bytes = prefix.clone();+ append_log(&mut bytes, &big_string(b"pending", 2 * BLOCK_SIZE));+ let mut cuts: Vec<_> = (prefix.len()..prefix.len() + HEADER_SIZE + 2).collect();+ for boundary in [BLOCK_SIZE, 2 * BLOCK_SIZE, bytes.len()] {+ cuts.extend(boundary - HEADER_SIZE..boundary + HEADER_SIZE);+ }+ for cut in cuts.into_iter().filter(|cut| *cut < bytes.len()) {+ assert_eq!(read(&bytes[..cut]).unwrap(), [b"committed"], "cut {cut}");+ }+}++#[test]+fn zero_filled_preallocation() {+ let mut bytes = log_bytes(&[b"foo".to_vec()]);+ bytes.resize(3 * BLOCK_SIZE, 0);+ append_log(&mut bytes, b"bar");+ assert_eq!(read(&bytes).unwrap(), [b"foo", b"bar"]);+}++#[test]+fn padding_cannot_join_fragments() {+ let mut bytes = physical_record(2, b"first");+ bytes.resize(BLOCK_SIZE, 0);+ bytes.extend_from_slice(&physical_record(4, b"unrelated"));+ error(&bytes, "without FIRST");+}++#[test]+fn nonzero_padding_is_corruption() {+ let mut bytes = vec![0; BLOCK_SIZE];+ bytes[HEADER_SIZE] = 1;+ error(&bytes, "padding");+ let mut bytes = log_bytes(&[big_string(b"foo", BLOCK_SIZE - HEADER_SIZE - 1)]);+ bytes.push(1);+ error(&bytes, "trailer");+}++#[test]+fn short_reads_and_interrupted_io() {+ struct Chunked {+ inner: Cursor<Vec<u8>>,+ calls: usize,+ }+ impl Read for Chunked {+ fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {+ self.calls += 1;+ if self.calls.is_multiple_of(7) {+ return Err(io::ErrorKind::Interrupted.into());+ }+ let length = buffer.len().min(13);+ self.inner.read(&mut buffer[..length])+ }+ }+ let records = [big_string(b"foo", 2 * BLOCK_SIZE), b"tail".to_vec()];+ let source = Chunked {+ inner: Cursor::new(log_bytes(&records)),+ calls: 0,+ };+ let mut seen = Vec::new();+ read_records_from(source, |record| {+ seen.push(record.to_vec());+ Ok(())+ })+ .unwrap();+ assert_eq!(seen, records);+}++#[test]+fn visitor_error_stops_the_read() {+ let bytes = log_bytes(&[b"foo".to_vec(), b"bar".to_vec()]);+ let mut calls = 0;+ let result = read_records_from(Cursor::new(bytes), |_| {+ calls += 1;+ anyhow::bail!("visitor error")+ });+ assert!(format!("{:#}", result.unwrap_err()).contains("visitor error"));+ assert_eq!(calls, 1);+}diff --git a/moli-leveldb-parser/src/tests/manifest.rs b/moli-leveldb-parser/src/tests/manifest.rsnew file mode 100644index 000000000..597b3152e--- /dev/null+++ b/moli-leveldb-parser/src/tests/manifest.rs@@ -0,0 +1,267 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// VersionEditTest.EncodeDecode plus parser regressions for db/version_edit.cc.+// See ../../TESTS.md, ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.++use std::fs;++use anyhow::Result;++use super::support::*;+use crate::{MAX_SEQUENCE, manifest::Manifest};++fn read(edits: &[Vec<u8>]) -> Result<Manifest> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), edits)?;+ Manifest::read(directory.path())+}++#[test]+fn encode_decode() -> Result<()> {+ const BIG: u64 = 1 << 50;+ let mut records = vec![metadata(BIG + 100, BIG + 1000)];+ let mut edit = Vec::new();+ for i in 0..4 {+ let parsed = read(&records)?;+ assert_eq!(parsed.tables.len(), i as usize);+ add_table(+ &mut edit,+ 3,+ BIG + 300 + i,+ BIG + 400 + i,+ &internal_key(b"foo", BIG + 500 + i, 1),+ &internal_key(b"zoo", BIG + 600 + i, 0),+ );+ delete_table(&mut edit, 4, BIG + 700 + i);+ edit.push(5);+ varint(i, &mut edit);+ slice(&internal_key(b"x", BIG + 900 + i, 1), &mut edit);+ records.truncate(1);+ records.push(edit.clone());+ }+ let parsed = read(&records)?;+ assert_eq!(parsed.log_number, BIG + 100);+ assert_eq!(parsed.last_sequence, BIG + 1000);+ for i in 0..4 {+ assert_eq!(+ parsed.tables.get(&(3, BIG + 300 + i)),+ Some(&(BIG + 400 + i))+ );+ }+ Ok(())+}++#[test]+fn comparator_check() {+ let mut edit = vec![1];+ slice(b"foo", &mut edit);+ assert!(format!("{:#}", read(&[metadata(3, 0), edit]).err().unwrap()).contains("comparator"));+}++#[test]+fn required_metadata_can_span_records() -> Result<()> {+ let parsed = read(&[vec![2, 3], vec![], vec![3, 4], vec![4, 5]])?;+ assert_eq!((parsed.log_number, parsed.last_sequence), (3, 5));+ Ok(())+}++#[test]+fn missing_required_metadata() {+ for mask in 0..7 {+ let fields = [vec![2, 3], vec![3, 4], vec![4, 5]];+ let edits: Vec<_> = fields+ .into_iter()+ .enumerate()+ .filter_map(|(i, field)| ((mask & (1 << i)) != 0).then_some(field))+ .collect();+ assert!(read(&edits).is_err(), "mask {mask}");+ }+}++#[test]+fn later_metadata_and_previous_log_override_earlier_values() -> Result<()> {+ let parsed = read(&[metadata(3, 5), vec![2, 8, 9, 3, 4, 12]])?;+ assert_eq!(+ (+ parsed.log_number,+ parsed.previous_log_number,+ parsed.last_sequence+ ),+ (8, 3, 12)+ );+ Ok(())+}++#[test]+fn unknown_tags() {+ for tag in [0, 8, 10, 100, u32::MAX as u64, u64::MAX] {+ let mut edit = Vec::new();+ varint(tag, &mut edit);+ assert!(read(&[metadata(3, 0), edit]).is_err(), "tag {tag}");+ }+}++#[test]+fn truncated_version_edit_fields() {+ let mut fields = Vec::new();+ let mut comparator = vec![1];+ slice(b"leveldb.BytewiseComparator", &mut comparator);+ fields.push(comparator);+ for tag in [2, 3, 4, 9] {+ let mut field = vec![tag];+ varint(1 << 50, &mut field);+ fields.push(field);+ }+ let mut compaction = vec![5, 1];+ slice(&internal_key(b"x", 100, 1), &mut compaction);+ fields.push(compaction);+ let mut deleted = Vec::new();+ delete_table(&mut deleted, 1, 1 << 50);+ fields.push(deleted);+ let mut added = Vec::new();+ add_table(+ &mut added,+ 1,+ 1 << 50,+ 4096,+ &internal_key(b"a", 1, 1),+ &internal_key(b"z", 9, 1),+ );+ fields.push(added);+ for field in fields {+ for end in 1..field.len() {+ assert!(+ read(&[metadata(3, 0), field[..end].to_vec()]).is_err(),+ "tag {}, end {end}",+ field[0]+ );+ }+ assert!(read(&[metadata(3, 0), field]).is_ok());+ }+}++#[test]+fn invalid_levels() {+ for level in [7, u32::MAX as u64, u64::MAX] {+ for tag in [5, 6, 7] {+ let mut edit = vec![tag];+ varint(level, &mut edit);+ edit.extend_from_slice(&[0; 32]);+ assert!(+ format!("{:#}", read(&[metadata(3, 0), edit]).err().unwrap()).contains("LevelDB")+ );+ }+ }+}++#[test]+fn truncated_internal_keys() {+ for length in 0..8 {+ let mut compaction = vec![5, 1];+ slice(&[0; 8][..length], &mut compaction);+ assert!(read(&[metadata(3, 0), compaction]).is_err());+ for short_smallest in [true, false] {+ let good = internal_key(b"key", 1, 1);+ let short = &[0; 8][..length];+ let (smallest, largest) = if short_smallest {+ (short, good.as_slice())+ } else {+ (good.as_slice(), short)+ };+ let mut edit = Vec::new();+ add_table(&mut edit, 1, 4, 4096, smallest, largest);+ assert!(read(&[metadata(3, 0), edit]).is_err());+ }+ }+}++#[test]+fn deleted_files_and_level_moves() -> Result<()> {+ let key = internal_key(b"key", 1, 1);+ let mut first = metadata(3, 1);+ add_table(&mut first, 0, 4, 100, &key, &key);+ add_table(&mut first, 1, 5, 200, &key, &key);+ let mut edit = Vec::new();+ add_table(&mut edit, 2, 4, 100, &key, &key);+ delete_table(&mut edit, 0, 4);+ delete_table(&mut edit, 1, 5);+ let parsed = read(&[first, edit])?;+ assert_eq!(parsed.tables, [((2, 4), 100)].into());+ Ok(())+}++#[test]+fn one_edit_applies_deletions_before_additions() -> Result<()> {+ let key = internal_key(b"key", 1, 1);+ let mut edit = metadata(3, 1);+ add_table(&mut edit, 1, 4, 100, &key, &key);+ delete_table(&mut edit, 1, 4);+ assert_eq!(read(&[edit])?.tables, [((1, 4), 100)].into());+ Ok(())+}++#[test]+fn sequence_overflow() {+ assert!(read(&[metadata(3, MAX_SEQUENCE)]).is_ok());+ assert!(read(&[metadata(3, MAX_SEQUENCE + 1)]).is_err());+}++#[test]+fn current_must_name_one_manifest() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ for name in [+ "",+ "\n",+ "MANIFEST-000001",+ "MANIFEST-000001\n\n",+ "../MANIFEST-000001\n",+ "MANIFEST-../000001\n",+ "MANIFEST-000001/child\n",+ "MANIFEST-abc\n",+ "/MANIFEST-000001\n",+ ] {+ fs::write(directory.path().join("CURRENT"), name)?;+ assert!(Manifest::read(directory.path()).is_err(), "{name:?}");+ }+ Ok(())+}++#[test]+fn current_selects_the_manifest_not_its_largest_number() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 5)])?;+ fs::write(+ directory.path().join("MANIFEST-999999"),+ log_bytes(&[metadata(4, 100)]),+ )?;+ assert_eq!(Manifest::read(directory.path())?.last_sequence, 5);+ Ok(())+}++#[test]+fn incomplete_last_edit_is_ignored() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let first = metadata(3, 5);+ let mut next = vec![4];+ varint(1 << 50, &mut next);+ let bytes = log_bytes(&[first, next]);+ write_manifest(directory.path(), &[])?;+ fs::write(+ directory.path().join("MANIFEST-000001"),+ &bytes[..bytes.len() - 1],+ )?;+ assert_eq!(Manifest::read(directory.path())?.last_sequence, 5);+ Ok(())+}++#[test]+fn complete_edit_checksum_failure_is_not_ignored() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 5)])?;+ let path = directory.path().join("MANIFEST-000001");+ let mut bytes = fs::read(&path)?;+ bytes[0] ^= 1;+ fs::write(path, bytes)?;+ assert!(format!("{:#}", Manifest::read(directory.path()).err().unwrap()).contains("checksum"));+ Ok(())+}diff --git a/moli-leveldb-parser/src/tests/mod.rs b/moli-leveldb-parser/src/tests/mod.rsnew file mode 100644index 000000000..db3e9d5b8--- /dev/null+++ b/moli-leveldb-parser/src/tests/mod.rs@@ -0,0 +1,9 @@+mod coding;+mod layout;+mod log;+mod manifest;+mod recovery;+mod snapshot;+mod support;+mod table;+mod write_batch;diff --git a/moli-leveldb-parser/src/tests/recovery.rs b/moli-leveldb-parser/src/tests/recovery.rsnew file mode 100644index 000000000..a72e0e2a7--- /dev/null+++ b/moli-leveldb-parser/src/tests/recovery.rs@@ -0,0 +1,509 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Read-only adaptations of db/recovery_test.cc, db/db_test.cc and+// db/corruption_test.cc. See ../../TESTS.md, ../../LICENSE-LevelDB and+// ../../AUTHORS-LevelDB for the exact scope and intentional differences.++use std::{fs, path::Path};++use anyhow::Result;++use super::support::*;+use crate::read_snapshot;++fn files(directory: &Path) -> Result<Entries> {+ fs::read_dir(directory)?+ .map(|entry| {+ let entry = entry?;+ Ok((+ entry.file_name().to_string_lossy().as_bytes().to_vec(),+ fs::read(entry.path())?,+ ))+ })+ .collect()+}++fn check(directory: &Path, expected: &Entries) -> Result<()> {+ let before = files(directory)?;+ assert_eq!(&read_snapshot(directory)?, expected);+ assert_eq!(+ files(directory)?,+ before,+ "recovery must not write, delete or create database files"+ );+ Ok(())+}++#[test]+fn manifest_reused() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])]),+ )?;+ let expected = [(b"foo".to_vec(), b"bar".to_vec())].into();+ check(directory.path(), &expected)?;+ // A second read and an appended VersionEdit both preserve CURRENT and the+ // original manifest, without requiring the engine's reuse_logs option.+ let path = directory.path().join("MANIFEST-000001");+ let mut bytes = fs::read(&path)?;+ append_log(&mut bytes, &[4, 1]);+ fs::write(path, bytes)?;+ check(directory.path(), &expected)+}++#[test]+fn large_manifest_compacted_read_without_compaction() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let path = directory.path().join("MANIFEST-000001");+ let mut bytes = fs::read(&path)?;+ bytes.resize(3 * 1048576, 0);+ fs::write(path, bytes)?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])]),+ )?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"bar".to_vec())].into(),+ )+}++#[test]+fn no_log_files() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ check(directory.path(), &Entries::new())+}++#[test]+fn log_file_reuse() -> Result<()> {+ for empty_log in [true, false] {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 1);+ let bytes = if empty_log {+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[(internal_key(b"foo", 1, 1), b"bar".to_vec())],+ )?;+ vec![]+ } else {+ log_bytes(&[batch(1, &[(b"foo", Some(b"bar"))])])+ };+ write_manifest(directory.path(), &[edit])?;+ fs::write(directory.path().join("000003.log"), bytes)?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"bar".to_vec())].into(),+ )?;+ }+ Ok(())+}++#[test]+fn multiple_mem_tables_recovered_in_memory() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let mut expected = Entries::new();+ let mut records = Vec::new();+ for i in 0..1000 {+ let key = format!("{i:050}").into_bytes();+ records.push(batch(i + 1, &[(&key, Some(&key))]));+ expected.insert(key.clone(), key);+ }+ fs::write(directory.path().join("000003.log"), log_bytes(&records))?;+ check(directory.path(), &expected)+}++#[test]+fn multiple_log_files() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let writes: &[(u64, u64, &[u8], &[u8])] = &[+ (3, 1, b"foo", b"bar"),+ (4, 1000, b"hello", b"world"),+ (5, 1001, b"hi", b"there"),+ (6, 1002, b"foo", b"bar2"),+ (2, 2000, b"hello", b"stale write"),+ ];+ for (number, sequence, key, value) in writes {+ fs::write(+ directory.path().join(format!("{number:06}.log")),+ log_bytes(&[batch(*sequence, &[(key, Some(value))])]),+ )?;+ }+ check(+ directory.path(),+ &[+ (b"foo".to_vec(), b"bar2".to_vec()),+ (b"hello".to_vec(), b"world".to_vec()),+ (b"hi".to_vec(), b"there".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn manifest_missing() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ fs::remove_file(directory.path().join("MANIFEST-000001"))?;+ let before = files(directory.path())?;+ assert!(read_snapshot(directory.path()).is_err());+ assert_eq!(files(directory.path())?, before);+ Ok(())+}++#[test]+fn get_level0_ordering() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 3);+ save_table(+ directory.path(),+ &mut edit,+ 0,+ 4,+ &[+ (internal_key(b"bar", 1, 1), b"b".to_vec()),+ (internal_key(b"foo", 2, 1), b"v1".to_vec()),+ ],+ )?;+ save_table(+ directory.path(),+ &mut edit,+ 0,+ 5,+ &[(internal_key(b"foo", 3, 1), b"v2".to_vec())],+ )?;+ write_manifest(directory.path(), &[edit])?;+ check(+ directory.path(),+ &[+ (b"bar".to_vec(), b"b".to_vec()),+ (b"foo".to_vec(), b"v2".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn get_ordered_by_levels() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 2);+ save_table(+ directory.path(),+ &mut edit,+ 2,+ 4,+ &[(internal_key(b"foo", 1, 1), b"v1".to_vec())],+ )?;+ save_table(+ directory.path(),+ &mut edit,+ 0,+ 5,+ &[(internal_key(b"foo", 2, 1), b"v2".to_vec())],+ )?;+ write_manifest(directory.path(), &[edit])?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"v2".to_vec())].into(),+ )+}++#[test]+fn recover() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let path = directory.path().join("000003.log");+ let mut bytes = log_bytes(&[batch(1, &[(b"foo", Some(b"v1")), (b"baz", Some(b"v5"))])]);+ fs::write(&path, &bytes)?;+ check(+ directory.path(),+ &[+ (b"foo".to_vec(), b"v1".to_vec()),+ (b"baz".to_vec(), b"v5".to_vec()),+ ]+ .into(),+ )?;+ append_log(+ &mut bytes,+ &batch(3, &[(b"bar", Some(b"v2")), (b"foo", Some(b"v3"))]),+ );+ fs::write(&path, &bytes)?;+ let mut expected: Entries = [+ (b"foo".to_vec(), b"v3".to_vec()),+ (b"baz".to_vec(), b"v5".to_vec()),+ (b"bar".to_vec(), b"v2".to_vec()),+ ]+ .into();+ check(directory.path(), &expected)?;+ append_log(&mut bytes, &batch(5, &[(b"foo", Some(b"v4"))]));+ fs::write(path, bytes)?;+ expected.insert(b"foo".to_vec(), b"v4".to_vec());+ check(directory.path(), &expected)+}++#[test]+fn recovery_with_empty_log() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 2);+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[+ (internal_key(b"foo", 2, 1), b"v2".to_vec()),+ (internal_key(b"foo", 1, 1), b"v1".to_vec()),+ ],+ )?;+ write_manifest(directory.path(), &[edit])?;+ let path = directory.path().join("000003.log");+ fs::write(&path, [])?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"v2".to_vec())].into(),+ )?;+ fs::write(path, log_bytes(&[batch(3, &[(b"foo", Some(b"v3"))])]))?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"v3".to_vec())].into(),+ )+}++#[test]+fn iter_multi_with_delete() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(+ 1,+ &[+ (b"a", Some(b"va")),+ (b"b", Some(b"vb")),+ (b"c", Some(b"vc")),+ (b"b", None),+ ],+ )]),+ )?;+ check(+ directory.path(),+ &[+ (b"a".to_vec(), b"va".to_vec()),+ (b"c".to_vec(), b"vc".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn iter_multi_with_delete_and_compaction() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 3);+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[+ (internal_key(b"a", 3, 1), b"va".to_vec()),+ (internal_key(b"b", 1, 1), b"vb".to_vec()),+ (internal_key(b"c", 2, 1), b"vc".to_vec()),+ ],+ )?;+ write_manifest(directory.path(), &[edit])?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(4, &[(b"b", None)])]),+ )?;+ check(+ directory.path(),+ &[+ (b"a".to_vec(), b"va".to_vec()),+ (b"c".to_vec(), b"vc".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn sequence_number_recovery() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 5);+ let entries: Vec<_> = (1..=5)+ .rev()+ .map(|i| (internal_key(b"foo", i, 1), format!("v{i}").into_bytes()))+ .collect();+ save_table(directory.path(), &mut edit, 1, 4, &entries)?;+ write_manifest(directory.path(), &[edit])?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"v5".to_vec())].into(),+ )?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(6, &[(b"foo", Some(b"v6"))])]),+ )?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"v6".to_vec())].into(),+ )+}++#[test]+fn corrupted_recovery_is_an_error_without_salvage() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let records: Vec<_> = (1..=100)+ .map(|i| {+ batch(+ i,+ &[(format!("{i:016}").as_bytes(), Some(&vec![b'x'; 1000]))],+ )+ })+ .collect();+ let mut bytes = log_bytes(&records);+ bytes[19] ^= 1;+ bytes[BLOCK_SIZE + 1000] ^= 1;+ fs::write(directory.path().join("000003.log"), bytes)?;+ let before = files(directory.path())?;+ assert!(read_snapshot(directory.path()).is_err());+ assert_eq!(files(directory.path())?, before);+ Ok(())+}++#[test]+fn sst_versions_newer_than_the_recovered_sequence_are_invisible() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 1);+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[+ (internal_key(b"foo", 5, 1), b"future".to_vec()),+ (internal_key(b"foo", 1, 1), b"current".to_vec()),+ ],+ )?;+ write_manifest(directory.path(), &[edit])?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"current".to_vec())].into(),+ )?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(5, &[(b"other", Some(b"value"))])]),+ )?;+ check(+ directory.path(),+ &[+ (b"foo".to_vec(), b"future".to_vec()),+ (b"other".to_vec(), b"value".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn table_tombstone_hides_an_older_wal_value() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 5);+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[(internal_key(b"foo", 5, 0), vec![])],+ )?;+ write_manifest(directory.path(), &[edit])?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(1, &[(b"foo", Some(b"old"))])]),+ )?;+ check(directory.path(), &Entries::new())+}++#[test]+fn empty_batch_cannot_advance_the_snapshot_sequence() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 1);+ save_table(+ directory.path(),+ &mut edit,+ 1,+ 4,+ &[+ (internal_key(b"foo", 5, 1), b"future".to_vec()),+ (internal_key(b"foo", 1, 1), b"current".to_vec()),+ ],+ )?;+ write_manifest(directory.path(), &[edit])?;+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(100, &[])]),+ )?;+ check(+ directory.path(),+ &[(b"foo".to_vec(), b"current".to_vec())].into(),+ )+}++#[test]+fn malformed_batch_does_not_return_a_partial_snapshot() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ let mut malformed = batch(2, &[(b"foo", Some(b"updated")), (b"bar", None)]);+ malformed.pop();+ fs::write(+ directory.path().join("000003.log"),+ log_bytes(&[batch(1, &[(b"foo", Some(b"old"))]), malformed]),+ )?;+ assert!(read_snapshot(directory.path()).is_err());+ Ok(())+}++#[test]+fn duplicate_log_number_is_rejected() -> Result<()> {+ let directory = tempfile::tempdir()?;+ write_manifest(directory.path(), &[metadata(3, 0)])?;+ for name in ["000003.log", "3.log"] {+ fs::write(directory.path().join(name), [])?;+ }+ assert!(+ read_snapshot(directory.path())+ .unwrap_err()+ .to_string()+ .contains("duplicate")+ );+ Ok(())+}++#[test]+fn native_chromium_tables_and_fragmented_wal() -> Result<()> {+ let directory = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/chromium/recovery");+ let mut expected = Entries::new();+ for index in 0..200 {+ if index % 7 == 0 {+ continue;+ }+ let key = format!("key-{index:04}").into_bytes();+ let value = if index % 5 == 0 {+ b"new".to_vec()+ } else {+ format!("value-{index}{}", "x".repeat(300)).into_bytes()+ };+ expected.insert(key, value);+ }+ expected.insert(vec![], vec![]);+ expected.insert(vec![0, 0xff, b'k'], vec![0xff, 0]);+ expected.insert(b"large".to_vec(), vec![b'y'; 120_000]);+ assert_eq!(expected.len(), 174);+ check(&directory, &expected)+}diff --git a/moli-leveldb-parser/src/tests/snapshot.rs b/moli-leveldb-parser/src/tests/snapshot.rsnew file mode 100644index 000000000..f182c70e1--- /dev/null+++ b/moli-leveldb-parser/src/tests/snapshot.rs@@ -0,0 +1,389 @@+use std::{+ collections::BTreeMap,+ fs,+ path::{Path, PathBuf},+ time::SystemTime,+};++use anyhow::Result;+use rusty_leveldb::{DB, LdbIterator, Options};++use super::support::{batch, log_bytes, metadata, write_manifest};++use crate::{+ Manifest,+ coding::{Decoder, masked_crc},+ log::read_records,+ read_snapshot,+};++fn open_writer(path: &Path, compression: u8) -> Result<DB> {+ Ok(DB::open(+ path,+ Options {+ compressor: compression,+ block_size: 128,+ block_restart_interval: 2,+ ..Options::default()+ },+ )?)+}++fn engine_entries(database: &mut DB) -> Result<BTreeMap<Vec<u8>, Vec<u8>>> {+ let mut iterator = database.new_iter()?;+ let mut entries = BTreeMap::new();+ while let Some((key, value)) = iterator.next() {+ entries.insert(key, value);+ }+ Ok(entries)+}++fn image(path: &Path) -> Result<BTreeMap<PathBuf, (Vec<u8>, SystemTime)>> {+ fs::read_dir(path)?+ .map(|entry| {+ let entry = entry?;+ Ok((+ entry.file_name().into(),+ (fs::read(entry.path())?, entry.metadata()?.modified()?),+ ))+ })+ .collect()+}++#[test]+fn matches_engine_for_tables_wal_versions_and_tombstones_without_writes() -> Result<()> {+ for compression in [0, 1] {+ let directory = tempfile::tempdir()?;+ let mut database = open_writer(directory.path(), compression)?;+ let mut expected = BTreeMap::new();+ for index in 0..200 {+ let key = format!("key-{index:04}").into_bytes();+ let value = format!("value-{index:04}-{}世界", "x".repeat(300)).into_bytes();+ database.put(&key, &value)?;+ expected.insert(key, value);+ }+ let large = vec![b'x'; 100_000];+ database.put(b"large", &large)?;+ database.compact_range(b"", b"\xff")?;+ assert!(!Manifest::read(directory.path())?.tables.is_empty());+ for index in 0..200 {+ let key = format!("key-{index:04}").into_bytes();+ if index % 3 == 0 {+ database.delete(&key)?;+ expected.remove(&key);+ } else if index % 5 == 0 {+ database.put(&key, b"new-value")?;+ expected.insert(key, b"new-value".to_vec());+ }+ }+ let large = vec![b'y'; 110_000];+ database.put(b"large", &large)?;+ expected.insert(b"large".to_vec(), large);+ database.put(b"", b"empty-key")?;+ expected.insert(Vec::new(), b"empty-key".to_vec());+ database.flush()?;+ assert_eq!(engine_entries(&mut database)?, expected);++ // The writer still holds LOCK, and its WAL has not been recovered into+ // an SST. Reading must not acquire a lock or modify any snapshot file.+ let before = image(directory.path())?;+ assert_eq!(read_snapshot(directory.path())?, expected);+ assert_eq!(image(directory.path())?, before);+ }+ Ok(())+}++#[test]+fn ignores_obsolete_tables_logs_and_manifests_after_compaction() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut database = open_writer(directory.path(), 1)?;+ database.put(b"deleted", b"must-not-resurrect")?;+ database.flush()?;+ let old_log_path = fs::read_dir(directory.path())?+ .map(|entry| entry.map(|entry| entry.path()))+ .collect::<std::io::Result<Vec<_>>>()?+ .into_iter()+ .find(|path| path.extension().is_some_and(|extension| extension == "log"))+ .unwrap();+ let old_log_number: u64 = old_log_path+ .file_stem()+ .unwrap()+ .to_str()+ .unwrap()+ .parse()?;+ let old_log = fs::read(&old_log_path)?;+ database.compact_range(b"", b"\xff")?;+ let tables = Manifest::read(directory.path())?.tables;+ let old_tables = tables+ .keys()+ .map(|(_, number)| {+ let name = format!("{number:06}.ldb");+ Ok((name.clone(), fs::read(directory.path().join(name))?))+ })+ .collect::<Result<Vec<_>>>()?;+ database.delete(b"deleted")?;+ database.put(b"kept", b"current")?;+ database.compact_range(b"", b"\xff")?;+ let expected = engine_entries(&mut database)?;+ database.close()?;+ drop(database);++ let live = Manifest::read(directory.path())?;+ assert!(old_log_number < live.log_number);+ assert_ne!(old_log_number, live.previous_log_number);+ for ((_, number), _) in tables {+ assert!(+ !live+ .tables+ .keys()+ .any(|(_, live_number)| *live_number == number)+ );+ }+ for (name, bytes) in old_tables {+ fs::write(directory.path().join(name), bytes)?;+ }+ fs::write(old_log_path, old_log)?;+ fs::write(+ directory.path().join("999999.ldb"),+ b"unreferenced incomplete table",+ )?;+ fs::write(+ directory.path().join("MANIFEST-999999"),+ b"unselected manifest",+ )?;+ assert_eq!(read_snapshot(directory.path())?, expected);+ assert!(!expected.contains_key(b"deleted".as_slice()));+ Ok(())+}++#[test]+fn reads_legacy_sst_extension_and_rejects_corrupt_or_missing_live_tables() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut database = open_writer(directory.path(), 1)?;+ database.put(b"key", b"value")?;+ database.compact_range(b"", b"\xff")?;+ database.close()?;+ drop(database);+ let manifest = Manifest::read(directory.path())?;+ let (_, number) = manifest.tables.keys().next().unwrap();+ let original = directory.path().join(format!("{number:06}.ldb"));+ let legacy = original.with_extension("sst");+ fs::rename(&original, &legacy)?;+ assert_eq!(+ read_snapshot(directory.path())?.get(b"key".as_slice()),+ Some(&b"value".to_vec())+ );+ let mut bytes = fs::read(&legacy)?;+ bytes[0] ^= 1;+ fs::write(&legacy, &bytes)?;+ let error = read_snapshot(directory.path()).unwrap_err();+ assert!(format!("{error:#}").contains("checksum"));+ fs::remove_file(legacy)?;+ assert!(read_snapshot(directory.path()).is_err());+ Ok(())+}++#[test]+fn rejects_malformed_table_blocks_even_with_valid_checksums() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut database = open_writer(directory.path(), 0)?;+ database.put(b"key", b"value")?;+ database.compact_range(b"", b"\xff")?;+ database.close()?;+ drop(database);+ let manifest = Manifest::read(directory.path())?;+ let (_, number) = manifest.tables.keys().next().unwrap();+ let path = directory.path().join(format!("{number:06}.ldb"));+ let original = fs::read(&path)?;++ let mut footer = Decoder::new(&original[original.len() - 48..original.len() - 8]);+ footer.varint()?;+ footer.varint()?;+ let index_offset = footer.varint()? as usize;+ let index_size = footer.varint()? as usize;+ assert_eq!(original[index_offset + index_size], 0);+ let mut index = Decoder::new(&original[index_offset..index_offset + index_size]);+ assert_eq!(index.varint32()?, 0);+ let key_length = index.varint32()? as usize;+ let value_length = index.varint32()? as usize;+ index.take(key_length)?;+ let mut handle = Decoder::new(index.take(value_length)?);+ let offset = handle.varint()? as usize;+ let size = handle.varint()? as usize;++ for case in 0..7 {+ let mut bytes = original.clone();+ match case {+ 0 => bytes[offset] = 1, // First entry cannot share a key prefix.+ 1 => bytes[offset + 1] = 127, // Key extends outside the block.+ 2 => bytes[offset + 2] = 127, // Value extends outside the block.+ 3 => bytes[offset + size - 4..offset + size].fill(0xff), // Restart count overflow.+ 4 => bytes[offset + size - 8] = 1, // First restart must start at zero.+ 5 => bytes[offset + size] = 42, // Unknown compression type.+ 6 => {+ bytes[offset..offset + 5].copy_from_slice(&[0xff, 0xff, 0xff, 0xff, 0x0f]);+ bytes[offset + size] = 1; // Snappy header claims a 4 GiB block.+ }+ _ => unreachable!(),+ }+ let checksum = masked_crc(&bytes[offset..offset + size + 1]);+ bytes[offset + size + 1..offset + size + 5].copy_from_slice(&checksum.to_le_bytes());+ fs::write(&path, bytes)?;+ assert!(read_snapshot(directory.path()).is_err(), "case {case}");+ }+ fs::write(path, original)?;+ assert_eq!(+ read_snapshot(directory.path())?.get(b"key".as_slice()),+ Some(&b"value".to_vec())+ );+ Ok(())+}++fn empty_snapshot(path: &Path) -> Result<PathBuf> {+ write_manifest(path, &[metadata(3, 0)])?;+ Ok(path.join("000003.log"))+}++fn put_batch(sequence: u64, key: &[u8], value: &[u8]) -> Vec<u8> {+ batch(sequence, &[(key, Some(value))])+}++#[test]+fn replays_new_and_previous_wals_and_uses_sequence_order() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let log = empty_snapshot(directory.path())?;+ // Include the legacy previous WAL as well as a newer, unregistered WAL.+ let manifest = directory.path().join("MANIFEST-000001");+ let mut bytes = fs::read(&manifest)?;+ bytes.extend_from_slice(&log_bytes(&[vec![9, 2]]));+ fs::write(&manifest, bytes)?;+ fs::write(+ directory.path().join("000002.log"),+ log_bytes(&[put_batch(1, b"previous", b"value")]),+ )?;+ fs::write(&log, log_bytes(&[put_batch(8, b"key", b"newer")]))?;+ fs::write(+ directory.path().join("000004.log"),+ log_bytes(&[put_batch(3, b"key", b"older")]),+ )?;+ let values = read_snapshot(directory.path())?;+ assert_eq!(values.get(b"key".as_slice()), Some(&b"newer".to_vec()));+ assert_eq!(values.get(b"previous".as_slice()), Some(&b"value".to_vec()));+ Ok(())+}++#[test]+fn ignores_only_incomplete_wal_tail_and_rejects_corrupt_complete_records() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let log = empty_snapshot(directory.path())?;+ let complete = put_batch(1, b"committed", b"value");+ let unfinished = put_batch(2, b"unfinished", &vec![b'x'; 100_000]);+ let valid = log_bytes(&[complete, unfinished]);+ for missing in [1, 1000, 40_000] {+ fs::write(&log, &valid[..valid.len() - missing])?;+ let values = read_snapshot(directory.path())?;+ assert_eq!(values.len(), 1);+ assert!(values.contains_key(b"committed".as_slice()));+ }+ let mut corrupt = valid;+ corrupt[8] ^= 1;+ fs::write(&log, corrupt)?;+ assert!(format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("checksum"));+ Ok(())+}++#[test]+fn handles_log_block_padding_and_empty_first_fragments() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let path = directory.path().join("log");+ for first_length in [32754, 32755, 32760, 32761] {+ let records = vec![+ vec![b'a'; first_length],+ vec![b'b'; 70_000],+ b"end".to_vec(),+ ];+ fs::write(&path, log_bytes(&records))?;+ let mut decoded = Vec::new();+ read_records(&path, |record| {+ decoded.push(record.to_vec());+ Ok(())+ })?;+ assert_eq!(decoded, records);+ }+ Ok(())+}++#[test]+fn rejects_malformed_batches_and_current_paths_without_panicking() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let log = empty_snapshot(directory.path())?;+ let valid = put_batch(1, b"key", b"value");+ let mut wrong_count = valid.clone();+ wrong_count[8] = 2;+ let mut unknown_type = valid.clone();+ unknown_type[12] = 2;+ let mut length_overflow = valid.clone();+ length_overflow.splice(13..14, [0xff; 10]);+ let mut extra_bytes = valid.clone();+ extra_bytes.push(0);+ for malformed in [+ valid[..11].to_vec(),+ wrong_count,+ unknown_type,+ length_overflow,+ extra_bytes,+ put_batch(u64::MAX, b"key", b"value"),+ ] {+ fs::write(&log, log_bytes(&[malformed]))?;+ assert!(read_snapshot(directory.path()).is_err());+ }+ fs::write(&log, [])?;+ for current in [+ "../MANIFEST-000001\n",+ "MANIFEST-../000001\n",+ "MANIFEST-000001",+ "\n",+ ] {+ fs::write(directory.path().join("CURRENT"), current)?;+ assert!(read_snapshot(directory.path()).is_err());+ }+ Ok(())+}++#[cfg(unix)]+#[test]+fn reads_a_directory_with_no_write_permissions() -> Result<()> {+ use std::os::unix::fs::PermissionsExt;++ struct RestorePermissions(Vec<(PathBuf, fs::Permissions)>);+ impl Drop for RestorePermissions {+ fn drop(&mut self) {+ for (path, permissions) in &self.0 {+ fs::set_permissions(path, permissions.clone()).unwrap();+ }+ }+ }++ let directory = tempfile::tempdir()?;+ let log = empty_snapshot(directory.path())?;+ fs::write(log, log_bytes(&[put_batch(1, b"key", b"value")]))?;+ let before = image(directory.path())?;+ let mut permissions = RestorePermissions(vec![(+ directory.path().to_owned(),+ fs::metadata(directory.path())?.permissions(),+ )]);+ for entry in fs::read_dir(directory.path())? {+ let entry = entry?;+ permissions+ .0+ .push((entry.path(), entry.metadata()?.permissions()));+ fs::set_permissions(entry.path(), fs::Permissions::from_mode(0o444))?;+ }+ fs::set_permissions(directory.path(), fs::Permissions::from_mode(0o555))?;+ assert_eq!(+ read_snapshot(directory.path())?.get(b"key".as_slice()),+ Some(&b"value".to_vec())+ );+ assert_eq!(image(directory.path())?, before);+ Ok(())+}diff --git a/moli-leveldb-parser/src/tests/support.rs b/moli-leveldb-parser/src/tests/support.rsnew file mode 100644index 000000000..258a813dd--- /dev/null+++ b/moli-leveldb-parser/src/tests/support.rs@@ -0,0 +1,265 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Test encoders and Random adapted from LevelDB; see ../../LICENSE-LevelDB,+// ../../AUTHORS-LevelDB and ../../TESTS.md for the pinned sources and scope.++use std::{collections::BTreeMap, fs, path::Path};++use anyhow::Result;++pub const BLOCK_SIZE: usize = 32768;+pub const HEADER_SIZE: usize = 7;+pub type Entries = BTreeMap<Vec<u8>, Vec<u8>>;++// Fixture construction deliberately does not call the production Decoder or+// masked_crc. The CRC wrapper is checked against upstream's published vectors.+pub fn crc(bytes: &[u8]) -> u32 {+ let value = crc32c::crc32c(bytes);+ value.rotate_right(15).wrapping_add(0xa282_ead8)+}++pub fn varint(mut value: u64, output: &mut Vec<u8>) {+ while value >= 128 {+ output.push((value as u8) | 128);+ value >>= 7;+ }+ output.push(value as u8);+}++pub fn slice(bytes: &[u8], output: &mut Vec<u8>) {+ varint(bytes.len() as u64, output);+ output.extend_from_slice(bytes);+}++pub fn internal_key(key: &[u8], sequence: u64, kind: u8) -> Vec<u8> {+ let mut result = key.to_vec();+ result.extend_from_slice(&((sequence << 8) | u64::from(kind)).to_le_bytes());+ result+}++pub fn batch(sequence: u64, changes: &[(&[u8], Option<&[u8]>)]) -> Vec<u8> {+ let mut result = sequence.to_le_bytes().to_vec();+ result.extend_from_slice(&(changes.len() as u32).to_le_bytes());+ for (key, value) in changes {+ result.push(u8::from(value.is_some()));+ slice(key, &mut result);+ if let Some(value) = value {+ slice(value, &mut result);+ }+ }+ result+}++pub fn physical_record(kind: u8, data: &[u8]) -> Vec<u8> {+ let length = u16::try_from(data.len()).unwrap();+ let mut checked = vec![kind];+ checked.extend_from_slice(data);+ let mut result = crc(&checked).to_le_bytes().to_vec();+ result.extend_from_slice(&length.to_le_bytes());+ result.extend_from_slice(&checked);+ result+}++pub fn append_log(output: &mut Vec<u8>, record: &[u8]) {+ let mut offset = 0;+ let mut first = true;+ loop {+ let leftover = BLOCK_SIZE - output.len() % BLOCK_SIZE;+ if leftover < HEADER_SIZE {+ output.resize(output.len() + leftover, 0);+ }+ let length =+ (record.len() - offset).min(BLOCK_SIZE - output.len() % BLOCK_SIZE - HEADER_SIZE);+ let last = offset + length == record.len();+ let kind = match (first, last) {+ (true, true) => 1,+ (true, false) => 2,+ (false, false) => 3,+ (false, true) => 4,+ };+ output.extend_from_slice(&physical_record(kind, &record[offset..offset + length]));+ offset += length;+ first = false;+ if last {+ break;+ }+ }+}++pub fn log_bytes(records: &[Vec<u8>]) -> Vec<u8> {+ let mut result = Vec::new();+ for record in records {+ append_log(&mut result, record);+ }+ result+}++pub fn fix_log_checksum(bytes: &mut [u8], offset: usize) {+ let length = u16::from_le_bytes(bytes[offset + 4..offset + 6].try_into().unwrap()) as usize;+ let checksum = crc(&bytes[offset + 6..offset + HEADER_SIZE + length]);+ bytes[offset..offset + 4].copy_from_slice(&checksum.to_le_bytes());+}++pub fn metadata(log_number: u64, sequence: u64) -> Vec<u8> {+ let mut result = vec![1];+ slice(b"leveldb.BytewiseComparator", &mut result);+ for (tag, value) in [(2, log_number), (3, 100), (4, sequence)] {+ result.push(tag);+ varint(value, &mut result);+ }+ result+}++pub fn write_manifest(directory: &Path, records: &[Vec<u8>]) -> Result<()> {+ fs::write(directory.join("CURRENT"), b"MANIFEST-000001\n")?;+ fs::write(directory.join("MANIFEST-000001"), log_bytes(records))?;+ Ok(())+}++pub fn add_table(+ edit: &mut Vec<u8>,+ level: u64,+ number: u64,+ size: u64,+ smallest: &[u8],+ largest: &[u8],+) {+ for value in [7, level, number, size] {+ varint(value, edit);+ }+ slice(smallest, edit);+ slice(largest, edit);+}++pub fn delete_table(edit: &mut Vec<u8>, level: u64, number: u64) {+ for value in [6, level, number] {+ varint(value, edit);+ }+}++// Test-only BlockBuilder/TableBuilder equivalents. They also allow constructing+// states that normal writers cannot emit, to exercise corruption validation.+pub fn block(entries: &[(Vec<u8>, Vec<u8>)], restart_interval: usize) -> Vec<u8> {+ assert!(restart_interval > 0);+ let mut output = Vec::new();+ let mut restarts = vec![0u32];+ let mut previous: &[u8] = b"";+ for (index, (key, value)) in entries.iter().enumerate() {+ let shared = if index % restart_interval == 0 {+ if index != 0 {+ restarts.push(output.len() as u32);+ }+ 0+ } else {+ previous.iter().zip(key).take_while(|(a, b)| a == b).count()+ };+ for length in [shared, key.len() - shared, value.len()] {+ varint(length as u64, &mut output);+ }+ output.extend_from_slice(&key[shared..]);+ output.extend_from_slice(value);+ previous = key;+ }+ for offset in &restarts {+ output.extend_from_slice(&offset.to_le_bytes());+ }+ output.extend_from_slice(&(restarts.len() as u32).to_le_bytes());+ output+}++fn append_block(output: &mut Vec<u8>, data: &[u8], compression: u8) -> Vec<u8> {+ let mut encoded = match compression {+ 0 => data.to_vec(),+ 1 => snap::raw::Encoder::new().compress_vec(data).unwrap(),+ _ => panic!("unsupported fixture compression"),+ };+ let mut handle = Vec::new();+ varint(output.len() as u64, &mut handle);+ varint(encoded.len() as u64, &mut handle);+ encoded.push(compression);+ let checksum = crc(&encoded);+ output.extend_from_slice(&encoded);+ output.extend_from_slice(&checksum.to_le_bytes());+ handle+}++pub fn table(+ entries: &[(Vec<u8>, Vec<u8>)],+ restart_interval: usize,+ entries_per_block: usize,+ compression: u8,+) -> Vec<u8> {+ let mut output = Vec::new();+ let mut index = Vec::new();+ for chunk in entries.chunks(entries_per_block) {+ let handle = append_block(&mut output, &block(chunk, restart_interval), compression);+ index.push((chunk.last().unwrap().0.clone(), handle));+ }+ let metaindex = append_block(&mut output, &block(&[], 1), 0);+ let index = append_block(&mut output, &block(&index, 1), compression);+ let mut footer = metaindex;+ footer.extend_from_slice(&index);+ footer.resize(40, 0);+ footer.extend_from_slice(&0xdb47_7524_8b80_fb57_u64.to_le_bytes());+ output.extend_from_slice(&footer);+ output+}++pub fn save_table(+ directory: &Path,+ edit: &mut Vec<u8>,+ level: u64,+ number: u64,+ entries: &[(Vec<u8>, Vec<u8>)],+) -> Result<()> {+ let bytes = table(entries, 2, 2, 0);+ fs::write(directory.join(format!("{number:06}.ldb")), &bytes)?;+ add_table(+ edit,+ level,+ number,+ bytes.len() as u64,+ &entries.first().unwrap().0,+ &entries.last().unwrap().0,+ );+ Ok(())+}++// LevelDB util/random.h's deterministic Park-Miller generator. Fixed seeds make+// the upstream stress cases reproducible without adding a rand dependency.+pub struct Random(u32);++impl Random {+ pub fn new(seed: u32) -> Self {+ let seed = seed & 0x7fff_ffff;+ Self(if seed == 0 || seed == 0x7fff_ffff {+ 1+ } else {+ seed+ })+ }++ pub fn uniform(&mut self, limit: u32) -> u32 {+ self.0 = (u64::from(self.0) * 16807 % 0x7fff_ffff) as u32;+ self.0 % limit+ }++ pub fn skewed(&mut self, max_log: u32) -> usize {+ let bits = self.uniform(max_log + 1);+ self.uniform(1 << bits) as usize+ }++ pub fn key(&mut self, length: usize) -> Vec<u8> {+ const CHARS: &[u8] = &[0, 1, b'a', b'b', b'c', b'd', b'e', 0xfd, 0xfe, 0xff];+ (0..length)+ .map(|_| CHARS[self.uniform(CHARS.len() as u32) as usize])+ .collect()+ }++ pub fn string(&mut self, length: usize) -> Vec<u8> {+ (0..length).map(|_| b' ' + self.uniform(95) as u8).collect()+ }+}++pub fn big_string(partial: &[u8], length: usize) -> Vec<u8> {+ partial.iter().copied().cycle().take(length).collect()+}diff --git a/moli-leveldb-parser/src/tests/table.rs b/moli-leveldb-parser/src/tests/table.rsnew file mode 100644index 000000000..d4a8cfb72--- /dev/null+++ b/moli-leveldb-parser/src/tests/table.rs@@ -0,0 +1,384 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Ports of table/table_test.cc and db/dbformat_test.cc, plus corruption cases.+// See ../../TESTS.md, ../../LICENSE-LevelDB and ../../AUTHORS-LevelDB.++use std::fs;++use anyhow::Result;++use super::support::*;+use crate::{+ MAX_SEQUENCE,+ coding::Decoder,+ read_snapshot,+ table::{read_entries, read_table},+};++fn scan(bytes: &[u8]) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {+ let directory = tempfile::tempdir()?;+ let path = directory.path().join("table.ldb");+ fs::write(&path, bytes)?;+ let mut result = Vec::new();+ read_table(&path, bytes.len() as u64, |key, value| {+ result.push((key.to_vec(), value.to_vec()));+ Ok(())+ })?;+ Ok(result)+}++fn check(entries: &Entries) -> Result<()> {+ let expected: Vec<_> = entries+ .iter()+ .map(|(k, v)| (k.clone(), v.clone()))+ .collect();+ for restart_interval in [1, 16, 1024] {+ let mut actual = Vec::new();+ read_entries(&block(&expected, restart_interval), |k, v| {+ actual.push((k.to_vec(), v.to_vec()));+ Ok(())+ })?;+ assert_eq!(actual, expected, "restart interval {restart_interval}");+ for compression in [0, 1] {+ let bytes = table(&expected, restart_interval, 13, compression);+ assert_eq!(+ scan(&bytes)?,+ expected,+ "restart interval {restart_interval}, compression {compression}"+ );+ }+ }+ Ok(())+}++#[test]+fn empty() -> Result<()> {+ check(&Entries::new())+}++#[test]+fn zero_restart_points_in_block() -> Result<()> {+ // Upstream explicitly accepts this four-byte Java LevelDB empty block.+ read_entries(&[0; 4], |_, _| panic!("zero-restart block must be empty"))+}++#[test]+fn simple_empty_key() -> Result<()> {+ check(&[(vec![], b"v".to_vec())].into())+}++#[test]+fn simple_single() -> Result<()> {+ check(&[(b"abc".to_vec(), b"v".to_vec())].into())+}++#[test]+fn simple_multi() -> Result<()> {+ check(+ &[+ (b"abc".to_vec(), b"v".to_vec()),+ (b"abcd".to_vec(), b"v".to_vec()),+ (b"ac".to_vec(), b"v2".to_vec()),+ ]+ .into(),+ )+}++#[test]+fn simple_special_key() -> Result<()> {+ check(&[(vec![0xff, 0xff], b"v3".to_vec())].into())+}++#[test]+fn randomized() -> Result<()> {+ // Same entry-count schedule, key alphabet and skewed lengths as Harness.+ let mut random = Random::new(306);+ for count in (0..50).chain((50..2000).step_by(200)) {+ let mut entries = Entries::new();+ for _ in 0..count {+ let key_length = random.skewed(4);+ let key = random.key(key_length);+ let value_length = random.skewed(5);+ entries.insert(key, random.string(value_length));+ }+ check(&entries)?;+ }+ Ok(())+}++#[test]+fn randomized_long_db() -> Result<()> {+ // Generated by Chromium's pinned C++ engine. The Rust fixture engine panics+ // in its shortest-separator routine for empty keys during this workload.+ let directory = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("tests/fixtures/chromium/randomized-long-db");+ let mut random = Random::new(301);+ let mut expected = Entries::new();+ for _ in 0..100_000 {+ let key_length = random.skewed(4);+ let key = random.key(key_length);+ let value_length = random.skewed(5);+ let value = random.string(value_length);+ expected.insert(key, value);+ }+ assert!(+ !crate::manifest::Manifest::read(&directory)?+ .tables+ .is_empty()+ );+ assert_eq!(read_snapshot(&directory)?, expected);+ Ok(())+}++#[test]+fn approximate_offset_of_plain_read_contents() -> Result<()> {+ // Keep upstream's large-value/block-boundary workload; this parser exposes+ // a full scan, so validate contents instead of a writer's offset estimate.+ let entries = vec![+ (b"k01".to_vec(), b"hello".to_vec()),+ (b"k02".to_vec(), b"hello2".to_vec()),+ (b"k03".to_vec(), vec![b'x'; 10_000]),+ (b"k04".to_vec(), vec![b'x'; 200_000]),+ (b"k05".to_vec(), vec![b'x'; 300_000]),+ (b"k06".to_vec(), b"hello3".to_vec()),+ (b"k07".to_vec(), vec![b'x'; 100_000]),+ ];+ assert_eq!(scan(&table(&entries, 16, 2, 0))?, entries);+ Ok(())+}++#[test]+fn approximate_offset_of_compressed_read_contents() -> Result<()> {+ let mut random = Random::new(301);+ let first = big_string(&random.string(2500), 10_000);+ let second = big_string(&random.string(2500), 10_000);+ let entries = vec![+ (b"k01".to_vec(), b"hello".to_vec()),+ (b"k02".to_vec(), first),+ (b"k03".to_vec(), b"hello3".to_vec()),+ (b"k04".to_vec(), second),+ ];+ let compressed = table(&entries, 16, 2, 1);+ assert!(compressed.len() < 10_000);+ assert_eq!(scan(&compressed)?, entries);+ Ok(())+}++#[test]+fn internal_key_encode_decode() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let sequences = [+ 1,+ 2,+ 3,+ (1 << 8) - 1,+ 1 << 8,+ (1 << 8) + 1,+ (1 << 16) - 1,+ 1 << 16,+ (1 << 16) + 1,+ (1 << 32) - 1,+ 1 << 32,+ (1 << 32) + 1,+ MAX_SEQUENCE,+ ];+ for sequence in sequences {+ let mut edit = metadata(3, sequence);+ let entries: Vec<_> = [b"".as_slice(), b"k", b"hello", b"longggggggggggggggggggggg"]+ .into_iter()+ .map(|key| (internal_key(key, sequence, 1), b"value".to_vec()))+ .collect();+ // The data reader consumes the on-disk order; public output is bytewise.+ let mut entries = entries;+ entries.sort_by(|a, b| a.0[..a.0.len() - 8].cmp(&b.0[..b.0.len() - 8]));+ save_table(directory.path(), &mut edit, 0, 4, &entries)?;+ write_manifest(directory.path(), &[edit])?;+ let expected: Entries = [b"".as_slice(), b"k", b"hello", b"longggggggggggggggggggggg"]+ .into_iter()+ .map(|key| (key.to_vec(), b"value".to_vec()))+ .collect();+ assert_eq!(read_snapshot(directory.path())?, expected);+ let mut edit = metadata(3, sequence);+ save_table(+ directory.path(),+ &mut edit,+ 0,+ 4,+ &[(internal_key(b"hello", sequence, 0), vec![])],+ )?;+ write_manifest(directory.path(), &[edit])?;+ assert!(read_snapshot(directory.path())?.is_empty());+ }+ Ok(())+}++#[test]+fn internal_key_decode_from_empty() -> Result<()> {+ let directory = tempfile::tempdir()?;+ for length in 0..8 {+ let bytes = table(&[(vec![b'k'; length], b"value".to_vec())], 1, 1, 0);+ fs::write(directory.path().join("000004.ldb"), &bytes)?;+ let mut edit = metadata(3, 1);+ let key = internal_key(b"key", 1, 1);+ add_table(&mut edit, 0, 4, bytes.len() as u64, &key, &key);+ write_manifest(directory.path(), &[edit])?;+ assert!(+ format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("internal key")+ );+ }+ Ok(())+}++#[test]+fn unknown_internal_value_type() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let mut edit = metadata(3, 1);+ save_table(+ directory.path(),+ &mut edit,+ 0,+ 4,+ &[(internal_key(b"key", 1, 2), vec![])],+ )?;+ write_manifest(directory.path(), &[edit])?;+ assert!(format!("{:#}", read_snapshot(directory.path()).unwrap_err()).contains("value type"));+ Ok(())+}++#[test]+fn truncated_footer_and_bad_magic() {+ for length in 0..48 {+ assert!(scan(&[0; 48][..length]).is_err());+ }+ let mut bytes = table(&[], 1, 1, 0);+ let length = bytes.len();+ bytes[length - 1] ^= 1;+ assert!(scan(&bytes).unwrap_err().to_string().contains("magic"));+}++#[test]+fn manifest_table_size_must_match() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let path = directory.path().join("table.ldb");+ let bytes = table(&[], 1, 1, 0);+ fs::write(&path, &bytes)?;+ for size in [0, bytes.len() as u64 - 1, bytes.len() as u64 + 1, u64::MAX] {+ assert!(read_table(&path, size, |_, _| Ok(())).is_err());+ }+ Ok(())+}++#[test]+fn index_handles_cannot_point_outside_the_table() {+ for (offset, size) in [(u64::MAX, 1), (0, u64::MAX), (10_000, 1), (0, 10_000)] {+ let mut bytes = table(&[], 1, 1, 0);+ let mut footer = vec![0, 0];+ varint(offset, &mut footer);+ varint(size, &mut footer);+ footer.resize(40, 0);+ let start = bytes.len() - 48;+ bytes[start..start + 40].copy_from_slice(&footer);+ assert!(scan(&bytes).unwrap_err().to_string().contains("outside"));+ }+}++#[test]+fn data_and_index_checksums() {+ let original = table(&[(b"key".to_vec(), b"value".to_vec())], 1, 1, 0);+ let mut footer = Decoder::new(&original[original.len() - 48..original.len() - 8]);+ footer.varint().unwrap();+ footer.varint().unwrap();+ let index_offset = footer.varint().unwrap() as usize;+ for offset in [0, index_offset] {+ let mut bytes = original.clone();+ bytes[offset] ^= 1;+ assert!(scan(&bytes).unwrap_err().to_string().contains("checksum"));+ }+}++#[test]+fn corrupt_compression_with_valid_checksum() {+ let entries = [(b"key".to_vec(), vec![b'x'; 100])];+ let original = table(&entries, 1, 1, 0);+ let size = block(&entries, 1).len();+ for compression in [1, 2, 255] {+ let mut bytes = original.clone();+ bytes[..size].fill(0xff);+ bytes[size] = compression;+ let checksum = crc(&bytes[..size + 1]);+ bytes[size + 1..size + 5].copy_from_slice(&checksum.to_le_bytes());+ assert!(scan(&bytes).is_err(), "compression {compression}");+ }+}++#[test]+fn truncated_and_excessive_restart_arrays() {+ for length in 0..4 {+ assert!(read_entries(&[0; 4][..length], |_, _| Ok(())).is_err());+ }+ for count in [2u32, u32::MAX] {+ let mut bytes = vec![0; 4];+ bytes.extend_from_slice(&count.to_le_bytes());+ assert!(read_entries(&bytes, |_, _| Ok(())).is_err());+ }+}++#[test]+fn restart_offsets_must_be_ordered_entry_boundaries() {+ let original = block(+ &[+ (b"abc".to_vec(), vec![1]),+ (b"abcd".to_vec(), vec![2]),+ (b"abd".to_vec(), vec![3]),+ ],+ 1,+ );+ let restarts = original.len() - 16;+ for (index, offset) in [(0, 1u32), (1, 0), (1, 1), (1, u32::MAX), (2, 0)] {+ let mut bytes = original.clone();+ let start = restarts + index * 4;+ bytes[start..start + 4].copy_from_slice(&offset.to_le_bytes());+ assert!(+ read_entries(&bytes, |_, _| Ok(())).is_err(),+ "index {index}, offset {offset}"+ );+ }+}++#[test]+fn restart_entry_cannot_share_a_prefix() {+ let mut bytes = block(&[(b"abc".to_vec(), vec![]), (b"abcd".to_vec(), vec![])], 1);+ bytes[6] = 1;+ assert!(+ read_entries(&bytes, |_, _| Ok(()))+ .unwrap_err()+ .to_string()+ .contains("shared prefix")+ );+}++#[test]+fn entries_cannot_extend_into_the_restart_array() {+ let original = block(&[(b"key".to_vec(), b"value".to_vec())], 1);+ for (index, value) in [(0, 1), (1, 100), (2, 100)] {+ let mut bytes = original.clone();+ bytes[index] = value;+ assert!(read_entries(&bytes, |_, _| Ok(())).is_err());+ }+}++#[test]+fn table_visitor_error_is_propagated() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let path = directory.path().join("table.ldb");+ let bytes = table(&[(b"a".to_vec(), vec![]), (b"b".to_vec(), vec![])], 1, 1, 0);+ fs::write(&path, &bytes)?;+ let mut calls = 0;+ let error = read_table(&path, bytes.len() as u64, |_, _| {+ calls += 1;+ anyhow::bail!("visitor failed")+ })+ .unwrap_err();+ assert_eq!(calls, 1);+ assert!(error.to_string().contains("visitor failed"));+ Ok(())+}diff --git a/moli-leveldb-parser/src/tests/write_batch.rs b/moli-leveldb-parser/src/tests/write_batch.rsnew file mode 100644index 000000000..2bd6e47f0--- /dev/null+++ b/moli-leveldb-parser/src/tests/write_batch.rs@@ -0,0 +1,163 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// Ports of db/write_batch_test.cc; see ../../TESTS.md, ../../LICENSE-LevelDB+// and ../../AUTHORS-LevelDB. Assertions observe the latest sequence per key.++use super::support::*;+use crate::{MAX_SEQUENCE, Versions, read_batch};++#[test]+fn empty() {+ let mut versions = Versions::new();+ assert_eq!(read_batch(&batch(0, &[]), &mut versions).unwrap(), None);+ assert!(versions.is_empty());+}++#[test]+fn multiple() {+ let mut versions = Versions::new();+ let bytes = batch(+ 100,+ &[+ (b"foo", Some(b"bar")),+ (b"box", None),+ (b"baz", Some(b"boo")),+ ],+ );+ assert_eq!(read_batch(&bytes, &mut versions).unwrap(), Some(102));+ assert_eq!(+ versions,+ Versions::from([+ (b"baz".to_vec(), (102, Some(b"boo".to_vec()))),+ (b"box".to_vec(), (101, None)),+ (b"foo".to_vec(), (100, Some(b"bar".to_vec()))),+ ])+ );+}++#[test]+fn corruption() {+ let mut bytes = batch(200, &[(b"foo", Some(b"bar")), (b"box", None)]);+ bytes.pop();+ assert!(read_batch(&bytes, &mut Versions::new()).is_err());+}++#[test]+fn append() {+ // Upstream appends batches with a different original sequence. On disk the+ // combined header supplies the sequence, and each appended operation adds 1.+ let mut bytes = batch(200, &[(b"a", Some(b"va"))]);+ let appended = batch(300, &[(b"b", Some(b"vb"))]);+ bytes.extend_from_slice(&appended[12..]);+ let appended = batch(300, &[(b"b", Some(b"vb")), (b"foo", None)]);+ bytes.extend_from_slice(&appended[12..]);+ bytes[8..12].copy_from_slice(&4u32.to_le_bytes());+ let mut versions = Versions::new();+ assert_eq!(read_batch(&bytes, &mut versions).unwrap(), Some(203));+ assert_eq!(+ versions,+ Versions::from([+ (b"a".to_vec(), (200, Some(b"va".to_vec()))),+ (b"b".to_vec(), (202, Some(b"vb".to_vec()))),+ (b"foo".to_vec(), (203, None)),+ ])+ );+}++#[test]+fn truncated_header() {+ for end in 0..12 {+ assert!(read_batch(&[0; 12][..end], &mut Versions::new()).is_err());+ }+}++#[test]+fn every_truncated_key_and_value_is_rejected() {+ let key = vec![b'k'; 130];+ let value = vec![b'v'; 300];+ let bytes = batch(12, &[(&key, Some(&value)), (b"deleted", None)]);+ for end in 12..bytes.len() {+ assert!(+ read_batch(&bytes[..end], &mut Versions::new()).is_err(),+ "end {end}"+ );+ }+ assert_eq!(read_batch(&bytes, &mut Versions::new()).unwrap(), Some(13));+}++#[test]+fn count_must_match_in_both_directions() {+ for count in [0u32, 2, u32::MAX] {+ let mut bytes = batch(100, &[(b"foo", Some(b"bar"))]);+ bytes[8..12].copy_from_slice(&count.to_le_bytes());+ assert!(+ read_batch(&bytes, &mut Versions::new()).is_err(),+ "count {count}"+ );+ }+}++#[test]+fn unknown_value_type() {+ for tag in [2, 7, 0xff] {+ let mut bytes = batch(100, &[(b"foo", Some(b"bar"))]);+ bytes[12] = tag;+ let error = read_batch(&bytes, &mut Versions::new()).unwrap_err();+ assert!(error.to_string().contains("value type"));+ }+}++#[test]+fn sequence_limits() {+ let changes: &[(&[u8], Option<&[u8]>)] = &[(b"a", Some(b"v")), (b"b", None)];+ assert_eq!(+ read_batch(&batch(MAX_SEQUENCE, &changes[..1]), &mut Versions::new()).unwrap(),+ Some(MAX_SEQUENCE)+ );+ assert_eq!(+ read_batch(&batch(MAX_SEQUENCE - 1, changes), &mut Versions::new()).unwrap(),+ Some(MAX_SEQUENCE)+ );+ for (sequence, changes) in [+ (MAX_SEQUENCE, changes),+ (MAX_SEQUENCE + 1, &changes[..1]),+ (u64::MAX, &changes[..0]),+ ] {+ assert!(read_batch(&batch(sequence, changes), &mut Versions::new()).is_err());+ }+}++#[test]+fn empty_key_and_value_are_distinct_from_delete() {+ let mut versions = Versions::new();+ read_batch(+ &batch(0, &[(b"", Some(b"")), (b"deleted", None)]),+ &mut versions,+ )+ .unwrap();+ assert_eq!(versions.get(b"".as_slice()), Some(&(0, Some(vec![]))));+ assert_eq!(versions.get(b"deleted".as_slice()), Some(&(1, None)));+}++#[test]+fn later_delete_cannot_be_resurrected_by_an_older_batch() {+ let mut versions = Versions::new();+ read_batch(&batch(20, &[(b"key", None)]), &mut versions).unwrap();+ read_batch(&batch(10, &[(b"key", Some(b"old"))]), &mut versions).unwrap();+ assert_eq!(versions.get(b"key".as_slice()), Some(&(20, None)));+ read_batch(&batch(30, &[(b"key", Some(b"new"))]), &mut versions).unwrap();+ assert_eq!(+ versions.get(b"key".as_slice()),+ Some(&(30, Some(b"new".to_vec())))+ );+}++#[test]+fn duplicate_sequence_requires_identical_contents() {+ let mut versions = Versions::new();+ let bytes = batch(10, &[(b"key", Some(b"value"))]);+ read_batch(&bytes, &mut versions).unwrap();+ read_batch(&bytes, &mut versions).unwrap();+ for value in [Some(b"other".as_slice()), None] {+ assert!(read_batch(&batch(10, &[(b"key", value)]), &mut versions).is_err());+ }+}diff --git a/moli-leveldb-parser/tests/fixtures/.gitattributes b/moli-leveldb-parser/tests/fixtures/.gitattributesnew file mode 100644index 000000000..671dbfff4--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/.gitattributes@@ -0,0 +1 @@+chromium/** -textdiff --git a/moli-leveldb-parser/tests/fixtures/.gitignore b/moli-leveldb-parser/tests/fixtures/.gitignorenew file mode 100644index 000000000..15498bbfb--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/.gitignore@@ -0,0 +1 @@+!*.logdiff --git a/moli-leveldb-parser/tests/fixtures/README.md b/moli-leveldb-parser/tests/fixtures/README.mdnew file mode 100644index 000000000..a9d518566--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/README.md@@ -0,0 +1,42 @@+# Chromium LevelDB fixtures++These closed databases were generated by `generate.cc`, linked against+Chromium's LevelDB revision `7ee830d02b623e8ffe0b95d59a74db1e58da04c5`.+The generator checks every entry against the native engine's iterator before+closing each database. Rust tests independently reconstruct the expected data.++- `chromium/randomized-long-db`: the 100,000-operation `Harness.RandomizedLongDB`+ workload, fixed seed 301, binary and empty keys, duplicate-key updates,+ multiple live SSTs, and the final WAL not yet converted into an SST.+- `chromium/recovery`: SSTs plus a fragmented WAL with updates, deletes, binary+ and empty keys/values, and a 120,000-byte replacement value.++The native fixtures use uncompressed tables. The Rust ports exercise both+uncompressed and Snappy-compressed data/index blocks, independently of whether+the host has a C++ Snappy library installed. Cargo tests read the checked-in+fixtures directly; they need no C++ compiler, downloads, or fixture generation.++To regenerate, run from the repository root with an empty scratch directory:++```sh+git clone https://chromium.googlesource.com/external/leveldb /tmp/moli-leveldb-fixture-source+git -C /tmp/moli-leveldb-fixture-source checkout 7ee830d02b623e8ffe0b95d59a74db1e58da04c5+cmake -S /tmp/moli-leveldb-fixture-source -B /tmp/moli-leveldb-fixture-build \+ -DCMAKE_BUILD_TYPE=Release -DLEVELDB_BUILD_TESTS=OFF \+ -DLEVELDB_BUILD_BENCHMARKS=OFF -DLEVELDB_INSTALL=OFF+cmake --build /tmp/moli-leveldb-fixture-build+c++ -std=c++17 -O2 moli-leveldb-parser/tests/fixtures/generate.cc \+ -I/tmp/moli-leveldb-fixture-source/include -I/tmp/moli-leveldb-fixture-source \+ /tmp/moli-leveldb-fixture-build/libleveldb.a -pthread \+ -o /tmp/moli-leveldb-fixture-generator+/tmp/moli-leveldb-fixture-generator /tmp/moli-leveldb-fixtures+```++If CMake detects optional native Snappy, CRC32C or Zstd libraries, link those as+well. Inspect the generated contents before replacing `chromium/`. Background+compaction can change file numbers/layout between runs; the expected logical+contents are fixed. `SHA256SUMS` records the checked-in files, not an assertion+that regenerated database files must be byte-for-byte identical.++See [TESTS.md](../../TESTS.md) for upstream sources and port coverage and+[LICENSE-LevelDB](../../LICENSE-LevelDB) for the upstream license.diff --git a/moli-leveldb-parser/tests/fixtures/SHA256SUMS b/moli-leveldb-parser/tests/fixtures/SHA256SUMSnew file mode 100644index 000000000..c0f401a88--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/SHA256SUMS@@ -0,0 +1,12 @@+de6ebc200dfc05ac59f36b97ec5e839b9b677265b8ca771f842daa185f43d33b chromium/randomized-long-db/000005.ldb+6be74a84a3a6b50b5d2417572f5833d3e531afb05a4b6b90ee2083289d25a32c chromium/randomized-long-db/000142.ldb+07e309106e9e6636b0f8dff4b065f4d4053ee9b87b010079da0c07ba5b1d2b85 chromium/randomized-long-db/000144.ldb+ea05f11a8a054ae8b475480ad119da17f8e8f751483a981a6d64e4abeb4b6413 chromium/randomized-long-db/000145.log+57fb102f8f75cd2586fbc5ef4a4b2a9784b841e4ef52e8cc4568dabb694dbd36 chromium/randomized-long-db/000146.ldb+1005a525006f148c86efcbfb36c6eac091b311532448010f70f7de9a68007167 chromium/randomized-long-db/CURRENT+0fda46e0ab68223da33a71e967ddfc8d7b85a9b11caf1c877c764aa57f79b983 chromium/randomized-long-db/MANIFEST-000002+47d6efcc0fe417cf7642d09b6804f28f7dcd979d0d55e3d51c54a4edceb892a3 chromium/recovery/000008.log+d4471cfe94c0d4d8e7a9843e09da3b4f07bfcf62f69c51e09ffa3dadfe774c15 chromium/recovery/000009.ldb+9535df112704ffae2a304b10a30fad7ef8dd084a34cbe4df48e45a9d887c2b6e chromium/recovery/000010.ldb+1005a525006f148c86efcbfb36c6eac091b311532448010f70f7de9a68007167 chromium/recovery/CURRENT+a0ed0bb5358d12e3dc82265a281ff274800ae7922cc43bea82ff56e44a4dbdad chromium/recovery/MANIFEST-000002diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldbnew file mode 100644index 000000000..9e7212d30Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000005.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldbnew file mode 100644index 000000000..2f4a4f0f0Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000142.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldbnew file mode 100644index 000000000..cac084a76Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000144.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.log b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.lognew file mode 100644index 000000000..06d11f7c2Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000145.log differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldb b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldbnew file mode 100644index 000000000..04f950cdbBinary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/000146.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENT b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENTnew file mode 100644index 000000000..1a8485221--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/CURRENT@@ -0,0 +1 @@+MANIFEST-000002diff --git a/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002 b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002new file mode 100644index 000000000..436049cdfBinary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/randomized-long-db/MANIFEST-000002 differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.log b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.lognew file mode 100644index 000000000..fac922d28Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000008.log differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldb b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldbnew file mode 100644index 000000000..93b650b2eBinary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000009.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldb b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldbnew file mode 100644index 000000000..85e473d02Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/000010.ldb differdiff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENT b/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENTnew file mode 100644index 000000000..1a8485221--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/chromium/recovery/CURRENT@@ -0,0 +1 @@+MANIFEST-000002diff --git a/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002 b/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002new file mode 100644index 000000000..e832f65a1Binary files /dev/null and b/moli-leveldb-parser/tests/fixtures/chromium/recovery/MANIFEST-000002 differdiff --git a/moli-leveldb-parser/tests/fixtures/generate.cc b/moli-leveldb-parser/tests/fixtures/generate.ccnew file mode 100644index 000000000..b1fa39744--- /dev/null+++ b/moli-leveldb-parser/tests/fixtures/generate.cc@@ -0,0 +1,135 @@+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.+// RandomKey/RandomString and the stress workload are adapted from LevelDB's+// util/testutil.cc and table/table_test.cc. See ../../LICENSE-LevelDB,+// ../../AUTHORS-LevelDB and ../../TESTS.md.+//+// Build against the pinned LevelDB source as documented in README.md here.+// This generator is not compiled or run by Cargo.++#include <cstdio>+#include <filesystem>+#include <iostream>+#include <map>+#include <memory>+#include <stdexcept>+#include <string>++#include "leveldb/db.h"+#include "leveldb/write_batch.h"+#include "util/random.h"++using Entries = std::map<std::string, std::string>;++static void Check(const leveldb::Status& status) {+ if (!status.ok()) throw std::runtime_error(status.ToString());+}++static std::string RandomKey(leveldb::Random& random, int length) {+ static const char chars[] = {'\0', '\1', 'a', 'b', 'c', 'd', 'e',+ '\xfd', '\xfe', '\xff'};+ std::string key;+ for (int i = 0; i < length; ++i) key += chars[random.Uniform(sizeof(chars))];+ return key;+}++static std::string RandomString(leveldb::Random& random, int length) {+ std::string value;+ for (int i = 0; i < length; ++i) value += char(' ' + random.Uniform(95));+ return value;+}++static std::unique_ptr<leveldb::DB> Open(const std::filesystem::path& path) {+ leveldb::Options options;+ options.create_if_missing = true;+ options.error_if_exists = true;+ options.write_buffer_size = 10000;+ options.block_size = 256;+ options.compression = leveldb::kNoCompression;+ leveldb::DB* db = nullptr;+ Check(leveldb::DB::Open(options, path.string(), &db));+ return std::unique_ptr<leveldb::DB>(db);+}++static void Verify(leveldb::DB& db, const Entries& expected) {+ auto iter = std::unique_ptr<leveldb::Iterator>(db.NewIterator(leveldb::ReadOptions()));+ iter->SeekToFirst();+ for (const auto& [key, value] : expected) {+ if (!iter->Valid() || iter->key().ToString() != key || iter->value().ToString() != value)+ throw std::runtime_error("native iterator differs from expected contents");+ iter->Next();+ }+ Check(iter->status());+ if (iter->Valid()) throw std::runtime_error("native iterator has extra entries");+}++static void RandomizedLongDB(const std::filesystem::path& path) {+ auto db = Open(path);+ leveldb::Random random(301);+ Entries expected;+ for (int i = 0; i < 100000; ++i) {+ // Explicit evaluation order matches the Rust port on every C++ compiler.+ const int key_length = random.Skewed(4);+ const auto key = RandomKey(random, key_length);+ const int value_length = random.Skewed(5);+ const auto value = RandomString(random, value_length);+ Check(db->Put(leveldb::WriteOptions(), key, value));+ expected[key] = value;+ }+ Verify(*db, expected);+ std::cout << path.filename() << ": " << expected.size() << " entries\n";+}++static void Recovery(const std::filesystem::path& path) {+ auto db = Open(path);+ Entries expected;+ for (int i = 0; i < 200; ++i) {+ char key[16]; std::snprintf(key, sizeof(key), "key-%04d", i);+ const std::string value = "value-" + std::to_string(i) + std::string(300, 'x');+ Check(db->Put(leveldb::WriteOptions(), key, value));+ expected[key] = value;+ }+ Check(db->Put(leveldb::WriteOptions(), "deleted", "must-not-return"));+ Check(db->Put(leveldb::WriteOptions(), "large", std::string(100000, 'x')));+ Check(db->Put(leveldb::WriteOptions(), "", "")); expected[""] = "";+ const std::string binary_key("\0\xffk", 3), binary_value("\xff\0", 2);+ Check(db->Put(leveldb::WriteOptions(), binary_key, binary_value));+ expected[binary_key] = binary_value;+ db->CompactRange(nullptr, nullptr);++ leveldb::WriteBatch batch;+ for (int i = 0; i < 200; ++i) {+ char key[16]; std::snprintf(key, sizeof(key), "key-%04d", i);+ if (i % 7 == 0) { batch.Delete(key); expected.erase(key); }+ else if (i % 5 == 0) { batch.Put(key, "new"); expected[key] = "new"; }+ }+ batch.Delete("deleted");+ batch.Put("large", std::string(120000, 'y'));+ expected["large"] = std::string(120000, 'y');+ leveldb::WriteOptions write; write.sync = true;+ Check(db->Write(write, &batch));+ Verify(*db, expected);+ std::cout << path.filename() << ": " << expected.size() << " entries\n";+}++int main(int argc, char** argv) {+ try {+ if (argc != 2) throw std::runtime_error("usage: generate OUTPUT_DIRECTORY");+ const std::filesystem::path root(argv[1]);+ if (!std::filesystem::create_directory(root))+ throw std::runtime_error("output directory must not already exist");+ RandomizedLongDB(root / "randomized-long-db");+ Recovery(root / "recovery");+ // Keep only the closed database's format-bearing files. Removing LOCK/LOG+ // here does not change snapshot contents; the Rust tests never open writers.+ for (const auto& directory : std::filesystem::directory_iterator(root)) {+ for (const auto& entry : std::filesystem::directory_iterator(directory)) {+ const auto name = entry.path().filename().string();+ if (name == "LOCK" || name == "LOG" || name == "LOG.old")+ std::filesystem::remove(entry);+ }+ }+ } catch (const std::exception& error) {+ std::cerr << error.what() << '\n';+ return 1;+ }+}