Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use std::{
collections::BTreeMap,
fmt, fs,
path::{Path, PathBuf},
};
use moli_crypto::sha256_hex;
use serde::{Deserialize, Serialize};
use crate::{
IndexedDbError, IndexedDbExternalObject, IndexedDbValue, Key, KeyPath,
state::{DatabaseData, IndexData, IndexedDbManager, ObjectStoreData, OriginState},
};
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistentOrigin {
#[serde(default)]
origin: Option<String>,
databases: BTreeMap<String, PersistentDatabase>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistentDatabase {
version: u64,
stores: BTreeMap<String, PersistentObjectStore>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistentObjectStore {
key_path: Option<KeyPath>,
auto_increment: bool,
auto_increment_counter: u64,
#[serde(default)]
indexes: BTreeMap<String, PersistentIndex>,
records: Vec<PersistentRecord>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistentIndex {
key_path: KeyPath,
unique: bool,
multi_entry: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistentRecord {
key: Key,
value: Vec<u8>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
external_objects: Vec<IndexedDbExternalObject>,
}
pub(crate) enum IndexedDbPersistenceBackend {
InMemory,
JsonFiles { storage_root: PathBuf },
}
impl IndexedDbManager {
pub(crate) fn ensure_origin_loaded(&mut self, origin: &str) -> Result<(), IndexedDbError> {
if self.origins.contains_key(origin) {
return Ok(());
}
let state = if let Some(bytes) = self.read_persisted_origin(origin)? {
let persisted: PersistentOrigin = serde_json::from_slice(&bytes).map_err(|err| {
IndexedDbError::Corruption(format!("failed to decode origin state: {err}"))
})?;
origin_state_from_persistent(persisted)
} else {
OriginState {
databases: BTreeMap::new(),
}
};
self.origins.insert(origin.to_owned(), state);
Ok(())
}
pub(crate) fn persist_origin(&self, origin: &str) -> Result<(), IndexedDbError> {
let Some(state) = self.origins.get(origin) else {
return Ok(());
};
let persisted = persistent_origin_from_state(origin, state);
let bytes = serde_json::to_vec_pretty(&persisted).map_err(|err| {
IndexedDbError::Serialization(format!("failed to encode origin state: {err}"))
})?;
self.write_persisted_origin(origin, &bytes)
}
pub(crate) fn read_persisted_origin(
&self,
origin: &str,
) -> Result<Option<Vec<u8>>, IndexedDbError> {
match &self.backend {
IndexedDbPersistenceBackend::InMemory => Ok(None),
IndexedDbPersistenceBackend::JsonFiles { storage_root } => {
let path = origin_path(storage_root, origin);
if !path.exists() {
return Ok(None);
}
fs::read(&path).map(Some).map_err(|err| {
IndexedDbError::Io(format!(
"failed to read origin state `{}`: {err}",
path.display()
))
})
}
}
}
pub(crate) fn write_persisted_origin(
&self,
origin: &str,
bytes: &[u8],
) -> Result<(), IndexedDbError> {
match &self.backend {
IndexedDbPersistenceBackend::InMemory => Ok(()),
IndexedDbPersistenceBackend::JsonFiles { storage_root } => {
let path = origin_path(storage_root, origin);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|err| {
IndexedDbError::Io(format!(
"failed to create parent directory `{}`: {err}",
parent.display()
))
})?;
}
fs::write(&path, bytes).map_err(|err| {
IndexedDbError::Io(format!(
"failed to write origin state `{}`: {err}",
path.display()
))
})
}
}
}
pub(crate) fn remove_persisted_origin(&self, origin: &str) -> Result<(), IndexedDbError> {
match &self.backend {
IndexedDbPersistenceBackend::InMemory => Ok(()),
IndexedDbPersistenceBackend::JsonFiles { storage_root } => {
let path = origin_path(storage_root, origin);
match fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(IndexedDbError::Io(format!(
"failed to remove origin state `{}`: {error}",
path.display()
))),
}
}
}
}
pub(crate) fn persisted_origins_with_prefix(
&self,
origin_prefix: &str,
) -> Result<Vec<(String, OriginState)>, IndexedDbError> {
match &self.backend {
IndexedDbPersistenceBackend::InMemory => Ok(Vec::new()),
IndexedDbPersistenceBackend::JsonFiles { storage_root } => {
let entries = fs::read_dir(storage_root).map_err(|err| {
IndexedDbError::Io(format!(
"failed to read IndexedDB storage root `{}`: {err}",
storage_root.display()
))
})?;
let mut origins = Vec::new();
for entry in entries {
let entry = entry.map_err(|err| {
IndexedDbError::Io(format!(
"failed to read IndexedDB storage root entry `{}`: {err}",
storage_root.display()
))
})?;
let path = entry.path();
if !path.is_file() {
continue;
}
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
let Some(sanitized_origin) = file_name.strip_suffix(".json") else {
continue;
};
let bytes = fs::read(&path).map_err(|err| {
IndexedDbError::Io(format!(
"failed to read origin state `{}`: {err}",
path.display()
))
})?;
let persisted: PersistentOrigin =
serde_json::from_slice(&bytes).map_err(|err| {
IndexedDbError::Corruption(format!(
"failed to decode origin state: {err}"
))
})?;
let origin = persisted
.origin
.clone()
.or_else(|| unsanitize_origin(sanitized_origin));
let Some(origin) = origin else {
continue;
};
if !origin.starts_with(origin_prefix) {
continue;
}
origins.push((origin, origin_state_from_persistent(persisted)));
}
Ok(origins)
}
}
}
pub(crate) fn remove_persisted_origins_with_prefix(
&self,
origin_prefix: &str,
) -> Result<(), IndexedDbError> {
match &self.backend {
IndexedDbPersistenceBackend::InMemory => Ok(()),
IndexedDbPersistenceBackend::JsonFiles { storage_root } => {
let sanitized_prefix = sanitize_origin(origin_prefix);
let entries = fs::read_dir(storage_root).map_err(|err| {
IndexedDbError::Io(format!(
"failed to read IndexedDB storage root `{}`: {err}",
storage_root.display()
))
})?;
for entry in entries {
let entry = entry.map_err(|err| {
IndexedDbError::Io(format!(
"failed to read IndexedDB storage root entry `{}`: {err}",
storage_root.display()
))
})?;
let path = entry.path();
if !path.is_file() {
continue;
}
let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
let Some(sanitized_origin) = file_name.strip_suffix(".json") else {
continue;
};
let remove = if sanitized_origin.starts_with(&sanitized_prefix) {
true
} else {
let bytes = fs::read(&path).map_err(|err| {
IndexedDbError::Io(format!(
"failed to read origin state `{}`: {err}",
path.display()
))
})?;
let persisted: PersistentOrigin =
serde_json::from_slice(&bytes).map_err(|err| {
IndexedDbError::Corruption(format!(
"failed to decode origin state: {err}"
))
})?;
persisted
.origin
.as_deref()
.is_some_and(|origin| origin.starts_with(origin_prefix))
};
if !remove {
continue;
}
fs::remove_file(&path).map_err(|error| {
IndexedDbError::Io(format!(
"failed to remove origin state `{}`: {error}",
path.display()
))
})?;
}
Ok(())
}
}
}
}
pub(crate) fn sanitize_origin(origin: &str) -> String {
let mut encoded = String::with_capacity(origin.len() * 2);
for byte in origin.as_bytes() {
let _ = fmt::Write::write_fmt(&mut encoded, format_args!("{byte:02x}"));
}
encoded
}
pub(crate) fn origin_file_stem(origin: &str) -> String {
let encoded = sanitize_origin(origin);
if encoded.len() <= 180 {
return encoded;
}
format!("h-{}", sha256_hex(origin.as_bytes()))
}
fn unsanitize_origin(encoded: &str) -> Option<String> {
if !encoded.len().is_multiple_of(2) {
return None;
}
let mut bytes = Vec::with_capacity(encoded.len() / 2);
for pair in encoded.as_bytes().as_chunks::<2>().0 {
let pair = std::str::from_utf8(pair).ok()?;
bytes.push(u8::from_str_radix(pair, 16).ok()?);
}
String::from_utf8(bytes).ok()
}
pub(crate) fn origin_path(storage_root: &Path, origin: &str) -> PathBuf {
storage_root.join(format!("{}.json", origin_file_stem(origin)))
}
fn origin_state_from_persistent(persistent: PersistentOrigin) -> OriginState {
let mut databases = BTreeMap::new();
for (name, database) in persistent.databases {
let mut stores = BTreeMap::new();
for (store_name, store) in database.stores {
let records = store
.records
.into_iter()
.map(|record| {
(
record.key,
IndexedDbValue::new(record.value, record.external_objects),
)
})
.collect();
stores.insert(
store_name,
ObjectStoreData {
key_path: store.key_path,
auto_increment: store.auto_increment,
auto_increment_counter: store.auto_increment_counter,
indexes: store
.indexes
.into_iter()
.map(|(name, index)| {
(
name,
IndexData {
key_path: index.key_path,
unique: index.unique,
multi_entry: index.multi_entry,
},
)
})
.collect(),
records,
},
);
}
databases.insert(
name,
DatabaseData {
version: database.version,
stores,
},
);
}
OriginState { databases }
}
fn persistent_origin_from_state(origin: &str, state: &OriginState) -> PersistentOrigin {
let mut databases = BTreeMap::new();
for (name, database) in &state.databases {
let mut stores = BTreeMap::new();
for (store_name, store) in &database.stores {
stores.insert(
store_name.clone(),
PersistentObjectStore {
key_path: store.key_path.clone(),
auto_increment: store.auto_increment,
auto_increment_counter: store.auto_increment_counter,
indexes: store
.indexes
.iter()
.map(|(name, index)| {
(
name.clone(),
PersistentIndex {
key_path: index.key_path.clone(),
unique: index.unique,
multi_entry: index.multi_entry,
},
)
})
.collect(),
records: store
.records
.iter()
.map(|(key, value)| PersistentRecord {
key: key.clone(),
value: value.wire_bytes.clone(),
external_objects: value.external_objects.clone(),
})
.collect(),
},
);
}
databases.insert(
name.clone(),
PersistentDatabase {
version: database.version,
stores,
},
);
}
PersistentOrigin {
origin: Some(origin.to_owned()),
databases,
}
}