Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
use std::{fs, path::Path};
#[cfg(target_os = "macos")]
use std::process::Command;
use anyhow::{Context, Result, anyhow, bail};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use moli_cookie_jar::{
CookiePriority, StoredCookie, StoredCookiePartitionKey, StoredCookieSameSite,
StoredCookieSourceScheme,
};
#[cfg(target_os = "macos")]
use moli_crypto::derive_pbkdf2_hmac_sha1;
use moli_crypto::{aes_128_cbc_pkcs7_decrypt, aes_256_gcm_decrypt, sha256_digest};
use rusqlite::Connection;
use tempfile::TempDir;
use time::OffsetDateTime;
use crate::{ChromeCryptoKey, sqlite::SqliteSnapshot, storage::LocalStorage};
const CHROME_EPOCH_OFFSET_MICROS: i64 = 11_644_473_600_000_000;
pub(crate) fn import_cookies(
profile: &Path,
key_source: &ChromeCryptoKey,
) -> Result<Vec<StoredCookie>> {
let source = [profile.join("Network/Cookies"), profile.join("Cookies")]
.into_iter()
.find(|path| path.is_file())
.ok_or_else(|| {
anyhow!(
"Chrome Cookies database was not found under `{}`",
profile.display()
)
})?;
let snapshot = SqliteSnapshot::open(&source, "Chrome Cookies")?;
let connection = &snapshot.connection;
let schema_version = connection
.query_row("SELECT value FROM meta WHERE key = 'version'", [], |row| {
row.get::<_, String>(0)
})
.ok()
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(0);
let (partition_key_column, cross_site_column) = cookie_partition_columns(connection)?;
let sql = format!(
"SELECT host_key,name,value,encrypted_value,path,expires_utc,is_secure,is_httponly,samesite,priority,source_scheme,source_port,{partition_key_column},{cross_site_column} FROM cookies"
);
let mut statement = connection
.prepare(&sql)
.context("Chrome Cookies database has an unsupported schema")?;
let rows = statement.query_map([], |row| {
Ok(ChromeCookieRow {
host: row.get(0)?,
name: row.get(1)?,
value: row.get(2)?,
encrypted_value: row.get(3)?,
path: row.get(4)?,
expires_utc: row.get(5)?,
secure: row.get::<_, i64>(6)? != 0,
http_only: row.get::<_, i64>(7)? != 0,
same_site: row.get(8)?,
priority: row.get(9)?,
source_scheme: row.get(10)?,
source_port: row.get(11)?,
partition_key: row.get(12)?,
has_cross_site_ancestor: row.get::<_, i64>(13)? != 0,
})
})?;
let mut cookies = Vec::new();
let mut key = None;
for row in rows {
let mut row = row?;
let value = if row.encrypted_value.is_empty() {
std::mem::take(&mut row.value)
} else {
if key.is_none() {
key = Some(resolve_crypto_key(key_source)?);
}
decrypt_cookie_value(
&row.encrypted_value,
key.as_deref().expect("key was resolved"),
&row.host,
schema_version,
)
.with_context(|| {
format!(
"failed to decrypt Chrome cookie `{}` for `{}`",
row.name, row.host
)
})?
};
cookies.push(row.into_stored(value));
}
Ok(cookies)
}
struct ChromeCookieRow {
host: String,
name: String,
value: String,
encrypted_value: Vec<u8>,
path: String,
expires_utc: i64,
secure: bool,
http_only: bool,
same_site: i64,
priority: i64,
source_scheme: i64,
source_port: i32,
partition_key: Option<String>,
has_cross_site_ancestor: bool,
}
impl ChromeCookieRow {
fn into_stored(self, value: String) -> StoredCookie {
StoredCookie {
name: self.name,
value,
host_only: !self.host.starts_with('.'),
domain: self.host,
path: self.path,
secure: self.secure,
http_only: self.http_only,
expires: chrome_expiry(self.expires_utc),
same_site: match self.same_site {
0 => StoredCookieSameSite::None,
1 => StoredCookieSameSite::Lax,
2 => StoredCookieSameSite::Strict,
_ => StoredCookieSameSite::Unspecified,
},
priority: Some(match self.priority {
0 => CookiePriority::Low,
2 => CookiePriority::High,
_ => CookiePriority::Medium,
}),
partition_key: self
.partition_key
.filter(|key| !key.is_empty())
.map(|key| StoredCookiePartitionKey::site(key, self.has_cross_site_ancestor)),
source_scheme: match self.source_scheme {
1 => StoredCookieSourceScheme::NonSecure,
2 => StoredCookieSourceScheme::Secure,
_ => StoredCookieSourceScheme::Unset,
},
source_port: self.source_port,
creation_index: 0,
last_access_index: 0,
}
}
}
fn cookie_partition_columns(connection: &Connection) -> Result<(&'static str, &'static str)> {
let mut statement = connection.prepare("PRAGMA table_info(cookies)")?;
let columns = statement.query_map([], |row| row.get::<_, String>(1))?;
let mut partition_key = "NULL";
let mut cross_site_ancestor = "0";
for column in columns {
match column?.as_str() {
"top_frame_site_key" => partition_key = "top_frame_site_key",
"has_cross_site_ancestor" => cross_site_ancestor = "has_cross_site_ancestor",
_ => {}
}
}
Ok((partition_key, cross_site_ancestor))
}
fn chrome_expiry(micros: i64) -> Option<OffsetDateTime> {
if micros == 0 {
return None;
}
let unix_micros = micros.checked_sub(CHROME_EPOCH_OFFSET_MICROS)?;
OffsetDateTime::from_unix_timestamp_nanos(i128::from(unix_micros) * 1_000).ok()
}
fn resolve_crypto_key(source: &ChromeCryptoKey) -> Result<Vec<u8>> {
match source {
ChromeCryptoKey::Base64(value) => STANDARD
.decode(value)
.context("--chrome-crypto-key base64 value is invalid"),
ChromeCryptoKey::System => system_crypto_key(),
}
}
#[cfg(target_os = "macos")]
fn system_crypto_key() -> Result<Vec<u8>> {
let output = Command::new("security")
.args(["find-generic-password", "-w", "-s", "Chrome Safe Storage"])
.output()
.context("failed to ask macOS Keychain for Chrome Safe Storage")?;
if !output.status.success() {
bail!("macOS Keychain did not grant access to Chrome Safe Storage");
}
let password =
String::from_utf8(output.stdout).context("Chrome Safe Storage password was not UTF-8")?;
let mut key = [0_u8; 16];
derive_pbkdf2_hmac_sha1(
password.trim_end().as_bytes(),
b"saltysalt",
std::num::NonZeroU32::new(1003).expect("Chrome iteration count is non-zero"),
&mut key,
);
Ok(key.to_vec())
}
#[cfg(not(target_os = "macos"))]
fn system_crypto_key() -> Result<Vec<u8>> {
bail!(
"system Chrome key import is currently available on macOS; use --chrome-crypto-key base64:<raw-key> on this platform"
)
}
fn decrypt_cookie_value(
encrypted: &[u8],
key: &[u8],
host: &str,
schema_version: i64,
) -> Result<String> {
let payload = encrypted
.strip_prefix(b"v10")
.or_else(|| encrypted.strip_prefix(b"v11"))
.ok_or_else(|| anyhow!("unsupported encrypted cookie prefix"))?;
let plaintext = if key.len() == 32 && payload.len() > 12 + 16 {
aes_256_gcm_decrypt(key, &payload[..12], &payload[12..])
.map_err(|_| anyhow!("AES-GCM authentication failed"))?
} else if key.len() == 16 {
aes_128_cbc_pkcs7_decrypt(key, &[b' '; 16], payload)
.map_err(|_| anyhow!("AES-CBC decryption failed"))?
} else {
bail!("Chrome crypto key must decode to 16 or 32 bytes");
};
let plaintext = if schema_version >= 24 {
let digest = sha256_digest(host.as_bytes());
plaintext
.strip_prefix(&digest)
.ok_or_else(|| anyhow!("decrypted cookie host digest does not match"))?
} else {
&plaintext
};
String::from_utf8(plaintext.to_vec()).context("decrypted cookie value is not UTF-8")
}
pub(crate) fn read_local_storage(profile: &Path) -> Result<LocalStorage> {
let source = profile.join("Local Storage/leveldb");
if !source.is_dir() {
return Ok(LocalStorage::new());
}
let snapshot = TempDir::new().context("failed to create localStorage snapshot directory")?;
copy_dir(&source, snapshot.path())?;
let entries = moli_leveldb_parser::read_snapshot(snapshot.path())
.context("failed to read Chrome localStorage LevelDB snapshot")?;
let mut origins = LocalStorage::new();
for (key, value) in entries {
let Some((origin, item_key)) = decode_local_storage_key(&key) else {
continue;
};
let Some(item_value) = decode_chrome_string(&value) else {
continue;
};
origins
.entry(origin)
.or_default()
.insert(item_key, item_value.into());
}
Ok(origins)
}
fn decode_local_storage_key(key: &[u8]) -> Option<(String, Vec<u16>)> {
let key = key.strip_prefix(b"_")?;
let separator = key.iter().position(|byte| *byte == 0)?;
let origin = std::str::from_utf8(&key[..separator]).ok()?.to_owned();
let item = decode_chrome_string(&key[separator + 1..])?;
Some((origin, item))
}
fn decode_chrome_string(value: &[u8]) -> Option<Vec<u16>> {
let (&encoding, bytes) = value.split_first()?;
match encoding {
0 => {
if bytes.len() % 2 != 0 {
return None;
}
Some(
bytes
.as_chunks::<2>()
.0
.iter()
.map(|pair| u16::from_le_bytes([pair[0], pair[1]]))
.collect(),
)
}
1 => Some(bytes.iter().map(|byte| u16::from(*byte)).collect()),
_ => None,
}
}
fn copy_dir(source: &Path, destination: &Path) -> Result<()> {
for entry in
fs::read_dir(source).with_context(|| format!("failed to read `{}`", source.display()))?
{
let entry = entry?;
let target = destination.join(entry.file_name());
if entry.file_type()?.is_dir() {
fs::create_dir_all(&target)?;
copy_dir(&entry.path(), &target)?;
} else {
fs::copy(entry.path(), target)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn imports_chrome_local_storage_from_tables_and_wal_without_resurrecting_deleted_keys()
-> Result<()> {
let profile = tempfile::tempdir()?;
let source = profile.path().join("Local Storage/leveldb");
fs::create_dir_all(&source)?;
let mut database = rusty_leveldb::DB::open(
&source,
rusty_leveldb::Options {
compressor: 1,
..rusty_leveldb::Options::default()
},
)?;
let token = b"_https://example.com\0\x01token";
let deleted = b"_https://example.com\0\x01deleted";
let surrogate = b"_https://example.com\0\x00\x00\xd8";
database.put(b"VERSION", b"1")?;
database.put(token, b"\x01old")?;
database.put(deleted, b"\x01removed")?;
database.put(surrogate, b"\x00\x00\xdc")?;
database.compact_range(b"", b"\xff")?;
database.put(token, b"\x01v\xff")?;
database.delete(deleted)?;
database.flush()?;
let destination = profile.path().join("moli-profile");
let request = crate::ImportRequest {
profile_dir: &destination,
includes: "storage".parse().unwrap(),
chrome_profile_dir: Some(profile.path()),
chrome_crypto_key: None,
firefox_profile_dir: None,
cookie_jars: &[],
};
assert_eq!(crate::import_session_state(&request)?.storage, 2);
let storage_path = destination.join("partitions/default/localstorage.json");
let bytes = fs::read(&storage_path)?;
let json: serde_json::Value = serde_json::from_slice(&bytes)?;
assert_eq!(
json["origins"]["https://example.com"]["entries"],
serde_json::json!([
{ "key": "token", "value": "vÿ" },
{ "key": { "utf16": [0xd800] }, "value": { "utf16": [0xdc00] } },
])
);
// An unreadable live SST must fail the import before replacing a
// previously imported profile with partial data.
let table = fs::read_dir(&source)?
.map(|entry| entry.map(|entry| entry.path()))
.collect::<std::io::Result<Vec<_>>>()?
.into_iter()
.find(|path| path.extension().is_some_and(|extension| extension == "ldb"))
.unwrap();
let mut corrupt = fs::read(&table)?;
corrupt[0] ^= 1;
fs::write(table, corrupt)?;
assert!(crate::import_session_state(&request).is_err());
assert_eq!(fs::read(storage_path)?, bytes);
Ok(())
}
#[test]
fn imports_chrome_cookies_with_and_without_partition_columns() -> Result<()> {
check_chrome_cookie_imports("DELETE")
}
#[test]
fn imports_chrome_cookies_from_wal() -> Result<()> {
check_chrome_cookie_imports("WAL")
}
fn check_chrome_cookie_imports(journal_mode: &str) -> Result<()> {
for (partitioned, cross_site) in
[(false, false), (true, false), (false, true), (true, true)]
{
let profile = tempfile::tempdir()?;
let network = profile.path().join("Network");
fs::create_dir(&network)?;
let connection = Connection::open(network.join("Cookies"))?;
connection.pragma_update(None, "journal_mode", journal_mode)?;
connection.pragma_update(None, "wal_autocheckpoint", 0)?;
connection.execute_batch(
"CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
INSERT INTO meta VALUES ('version', '24');
CREATE TABLE cookies (
host_key TEXT, name TEXT, value TEXT, encrypted_value BLOB,
path TEXT, expires_utc INTEGER, is_secure INTEGER,
is_httponly INTEGER, samesite INTEGER, priority INTEGER,
source_scheme INTEGER, source_port INTEGER
);
INSERT INTO cookies VALUES (
'.example.com', 'session', 'stored-value', X'', '/',
0, 1, 1, 1, 2, 2, 443
);",
)?;
if partitioned {
connection.execute_batch(
"ALTER TABLE cookies ADD COLUMN top_frame_site_key TEXT;
UPDATE cookies SET top_frame_site_key = 'https://example.org';",
)?;
}
if cross_site {
connection.execute_batch(
"ALTER TABLE cookies ADD COLUMN has_cross_site_ancestor INTEGER;
UPDATE cookies SET has_cross_site_ancestor = 1;",
)?;
}
// Keep the writer open so uncheckpointed data remains in the WAL.
if journal_mode == "WAL" {
assert!(network.join("Cookies-wal").is_file());
}
let cookies = import_cookies(profile.path(), &ChromeCryptoKey::System)?;
assert_eq!(cookies.len(), 1);
let cookie = &cookies[0];
assert_eq!(cookie.name, "session");
assert_eq!(cookie.value, "stored-value");
assert_eq!(cookie.domain, ".example.com");
assert!(!cookie.host_only);
assert!(cookie.secure);
assert!(cookie.http_only);
assert_eq!(cookie.expires, None);
assert_eq!(cookie.same_site, StoredCookieSameSite::Lax);
assert_eq!(cookie.priority, Some(CookiePriority::High));
assert_eq!(cookie.source_scheme, StoredCookieSourceScheme::Secure);
assert_eq!(cookie.source_port, 443);
assert_eq!(
cookie.partition_key,
partitioned.then(|| StoredCookiePartitionKey::site(
"https://example.org".to_owned(),
cross_site,
)),
);
}
Ok(())
}
#[test]
fn decodes_chrome_local_storage_latin1_key() {
let decoded = decode_local_storage_key(b"_https://example.test\0\x01token").unwrap();
assert_eq!(decoded.0, "https://example.test");
assert_eq!(String::from_utf16(&decoded.1).unwrap(), "token");
}
#[test]
fn decodes_chrome_utf16_string() {
assert_eq!(
decode_chrome_string(&[0, b'a', 0, 0xac, 0x20]),
Some(vec![97, 8364])
);
}
#[test]
fn converts_chrome_epoch() {
assert_eq!(
chrome_expiry(11_644_473_601_000_000)
.unwrap()
.unix_timestamp(),
1
);
}
}