The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
//! Step 6 and `storage.restore`: the `tar.zst` archive of what the migration
//! moves and deletes.
//!
//! The archive is streamed: files are read, hashed and written one at a time
//! into a zstd encoder over the output file, so a 16 GB, 300k-file profile
//! never sits in memory. Entries are stored under `workspace/<path>`, the
//! place they had in layout v1, whatever their location when archived (moved
//! entries are read from `files/`). The last tar entry is `MANIFEST.json`
//! (path, size and sha256 of every file); the same manifest is written beside
//! the archive as `<name>.manifest.json`.
//!
//! [`verify`] re-reads the whole archive and compares every entry to the
//! manifest. The migration writes `<name>.tmp`, verifies it, and only then
//! renames it into place, so a file with the final name has been verified.
//! [`restore`] verifies first, then puts the entries back under `workspace/`.
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use walkdir::WalkDir;
use super::plan::Action;
use super::state::LayoutError;
/// The manifest's tar entry name.
pub const MANIFEST_ENTRY: &str = "MANIFEST.json";
/// The prefix every archived path carries.
pub const ROOT: &str = "workspace";
/// The prefix of profile-level files (beside `config.toml`) in the archive.
pub const PROFILE_ROOT: &str = "profile";
const ZSTD_LEVEL: i32 = 3;
/// One archived file.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileRecord {
/// The path inside the archive (`workspace/skills/a/SKILL.md`).
pub path: String,
/// `file` or `symlink`.
pub kind: String,
/// Bytes of a file (0 for a symlink).
pub size: u64,
/// Hex sha256 of the contents (of the link target for a symlink).
pub sha256: String,
}
/// A top-level plan entry the archive holds.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EntryRecord {
/// The path relative to `workspace/`.
pub path: String,
/// What the migration did with it.
pub action: Action,
}
/// The archive's table of contents.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Manifest {
/// Format version.
pub version: u32,
/// The user the profile belongs to.
pub user: String,
/// When the archive was written.
pub created_ms: i64,
/// The plan entries archived.
pub entries: Vec<EntryRecord>,
/// Every file, in archive order.
pub files: Vec<FileRecord>,
}
/// One thing to archive: where it is now and what the plan calls it.
#[derive(Debug, Clone)]
pub struct Source {
/// The plan path relative to `workspace/`.
pub rel: String,
/// The action recorded in the manifest.
pub action: Action,
/// Where the entry is on disk now.
pub at: PathBuf,
}
struct HashingReader<R> {
inner: R,
hasher: Sha256,
}
impl<R: Read> Read for HashingReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.hasher.update(&buf[..n]);
Ok(n)
}
}
fn io(error: impl Into<anyhow::Error>) -> LayoutError {
LayoutError::Other(error.into())
}
/// Streams `sources` into `dest` (created, replaced if present) and returns
/// the manifest it wrote. The sources are only read.
///
/// # Errors
///
/// An I/O error. A partly written `dest` is removed.
pub fn write(
dest: &Path,
user: &str,
created_ms: i64,
sources: &[Source],
) -> Result<Manifest, LayoutError> {
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)?;
restrict(parent, 0o700);
}
let result = write_inner(dest, user, created_ms, sources);
if result.is_err() {
let _ = std::fs::remove_file(dest);
}
result
}
fn write_inner(
dest: &Path,
user: &str,
created_ms: i64,
sources: &[Source],
) -> Result<Manifest, LayoutError> {
let file = File::create(dest)?;
restrict(dest, 0o600);
let encoder = zstd::stream::write::Encoder::new(BufWriter::new(file), ZSTD_LEVEL)?;
let mut tar = tar::Builder::new(encoder);
tar.follow_symlinks(false);
let mut manifest = Manifest {
version: 1,
user: user.to_owned(),
created_ms,
entries: sources
.iter()
.map(|s| EntryRecord {
path: s.rel.clone(),
action: s.action,
})
.collect(),
files: Vec::new(),
};
for source in sources {
// A profile-level credential file (`@profile/<name>`) is archived as
// `profile/<name>` and restored beside `config.toml`.
let root = match source.rel.strip_prefix(super::credentials::PROFILE_PREFIX) {
Some(name) => format!("{PROFILE_ROOT}/{name}"),
None => format!("{ROOT}/{}", source.rel),
};
for item in WalkDir::new(&source.at)
.follow_links(false)
.sort_by_file_name()
{
let item = item.map_err(io)?;
let relative = item.path().strip_prefix(&source.at).map_err(io)?;
let name = if relative.as_os_str().is_empty() {
root.clone()
} else {
format!("{root}/{}", relative.to_string_lossy().replace('\\', "/"))
};
let kind = item.file_type();
if kind.is_dir() {
tar.append_dir(&name, item.path())?;
} else if kind.is_symlink() {
let target = std::fs::read_link(item.path())?;
let mut header = tar::Header::new_gnu();
header.set_entry_type(tar::EntryType::Symlink);
header.set_size(0);
header.set_mode(0o777);
tar.append_link(&mut header, &name, &target)?;
manifest.files.push(FileRecord {
path: name,
kind: "symlink".to_owned(),
size: 0,
sha256: hex::encode(Sha256::digest(target.to_string_lossy().as_bytes())),
});
} else if kind.is_file() {
let meta = item.metadata().map_err(io)?;
let mut header = tar::Header::new_gnu();
header.set_metadata(&meta);
let mut reader = HashingReader {
inner: BufReader::new(File::open(item.path())?),
hasher: Sha256::new(),
};
// The size in the header is what the tar stream will carry; a
// file that changes while it is read fails the append.
tar.append_data(&mut header, &name, &mut reader)?;
manifest.files.push(FileRecord {
path: name,
kind: "file".to_owned(),
size: meta.len(),
sha256: hex::encode(reader.hasher.finalize()),
});
}
}
}
let body = serde_json::to_vec(&manifest).map_err(io)?;
let mut header = tar::Header::new_gnu();
header.set_size(body.len() as u64);
header.set_mode(0o600);
header.set_mtime(u64::try_from(created_ms / 1000).unwrap_or(0));
tar.append_data(&mut header, MANIFEST_ENTRY, body.as_slice())?;
let encoder = tar.into_inner()?;
let mut out = encoder.finish()?;
out.flush()?;
out.get_ref().sync_all()?;
write_sidecar(dest, &body)?;
tracing::info!(
target: "openhuman::storage",
files = manifest.files.len(),
"[layout] archive written"
);
Ok(manifest)
}
/// `<archive>.manifest.json`.
pub fn sidecar_path(archive: &Path) -> PathBuf {
let mut name = archive.file_name().unwrap_or_default().to_os_string();
name.push(".manifest.json");
archive.with_file_name(name)
}
fn write_sidecar(archive: &Path, body: &[u8]) -> Result<(), LayoutError> {
std::fs::write(sidecar_path(archive), body)?;
Ok(())
}
#[cfg(unix)]
fn restrict(path: &Path, mode: u32) {
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode));
}
#[cfg(not(unix))]
fn restrict(_path: &Path, _mode: u32) {}
fn open_archive(
path: &Path,
) -> Result<tar::Archive<zstd::stream::read::Decoder<'static, BufReader<File>>>, LayoutError> {
let decoder = zstd::stream::read::Decoder::new(File::open(path)?)?;
Ok(tar::Archive::new(decoder))
}
/// Re-reads `archive` and checks every entry against the manifest it carries:
/// every manifest file present with the recorded size and sha256, no file the
/// manifest does not list.
///
/// # Errors
///
/// [`LayoutError::Verify`] naming the first discrepancy (or a read/decode
/// failure, which is also a corrupt archive).
pub fn verify(archive: &Path) -> Result<Manifest, LayoutError> {
let bad = |m: String| LayoutError::Verify(format!("archive {}: {m}", archive.display()));
let mut tar = open_archive(archive)?;
let mut seen: BTreeMap<String, (String, u64, String)> = BTreeMap::new();
let mut manifest: Option<Manifest> = None;
for entry in tar.entries().map_err(|e| bad(format!("unreadable: {e}")))? {
let mut entry = entry.map_err(|e| bad(format!("unreadable entry: {e}")))?;
let path = entry
.path()
.map_err(|e| bad(e.to_string()))?
.to_string_lossy()
.into_owned();
let kind = entry.header().entry_type();
if path == MANIFEST_ENTRY {
let mut body = Vec::new();
entry
.read_to_end(&mut body)
.map_err(|e| bad(format!("manifest unreadable: {e}")))?;
manifest = Some(
serde_json::from_slice(&body).map_err(|e| bad(format!("manifest damaged: {e}")))?,
);
} else if kind.is_file() {
let mut hasher = Sha256::new();
let mut size = 0u64;
let mut buf = vec![0u8; 64 * 1024];
loop {
let n = entry
.read(&mut buf)
.map_err(|e| bad(format!("{path}: {e}")))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
size += n as u64;
}
seen.insert(
path,
("file".to_owned(), size, hex::encode(hasher.finalize())),
);
} else if kind.is_symlink() {
let target = entry
.link_name()
.map_err(|e| bad(e.to_string()))?
.map(|t| t.to_string_lossy().into_owned())
.unwrap_or_default();
seen.insert(
path,
(
"symlink".to_owned(),
0,
hex::encode(Sha256::digest(target.as_bytes())),
),
);
}
}
let manifest = manifest.ok_or_else(|| bad("no manifest entry".to_owned()))?;
for record in &manifest.files {
match seen.remove(&record.path) {
None => return Err(bad(format!("{} is missing", record.path))),
Some((kind, size, sha)) => {
if kind != record.kind || size != record.size || sha != record.sha256 {
return Err(bad(format!("{} does not match the manifest", record.path)));
}
}
}
}
if let Some(extra) = seen.keys().next() {
return Err(bad(format!("{extra} is not in the manifest")));
}
Ok(manifest)
}
/// Verifies `archive`, then extracts every entry under `profile_dir` (the
/// entries carry the `workspace/` prefix). Existing files are overwritten.
/// Returns the manifest.
///
/// # Errors
///
/// A failed verification (nothing is extracted then), a path that would leave
/// `profile_dir`, or an I/O error.
pub fn restore(archive: &Path, profile_dir: &Path) -> Result<Manifest, LayoutError> {
let manifest = verify(archive)?;
let mut tar = open_archive(archive)?;
tar.set_preserve_permissions(true);
for entry in tar.entries()? {
let mut entry = entry?;
let path = entry.path()?.into_owned();
if path == Path::new(MANIFEST_ENTRY) {
continue;
}
if let Ok(name) = path.strip_prefix(PROFILE_ROOT) {
// Only a plain file name directly below `profile/`.
let mut parts = name.components();
let plain = matches!(
(parts.next(), parts.next()),
(Some(std::path::Component::Normal(_)), None)
);
if !plain {
return Err(LayoutError::Verify(format!(
"{} escapes the profile directory",
path.display()
)));
}
entry.unpack(profile_dir.join(name))?;
continue;
}
if !path.starts_with(ROOT) {
return Err(LayoutError::Verify(format!(
"{} is outside {ROOT}/ and {PROFILE_ROOT}/",
path.display()
)));
}
if !entry.unpack_in(profile_dir)? {
return Err(LayoutError::Verify(format!(
"{} escapes the profile directory",
path.display()
)));
}
}
Ok(manifest)
}
#[cfg(all(test, feature = "legacy-import"))]
#[path = "archive_tests.rs"]
mod tests;