The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
//! The source store's contract, through separate build-script processes: only verified
//! archives are published, independent builds can share them, and a full store needs no network.
use std::fs;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
struct Source {
work: tempfile::TempDir,
hash: String,
url: String,
}
impl Source {
fn new() -> Self {
let work = tempfile::tempdir().unwrap();
let served = work.path().join("served");
fs::create_dir_all(served.join("tree/sub")).unwrap();
fs::write(served.join("tree/a.txt"), "a").unwrap();
fs::write(served.join("tree/sub/b.txt"), "b").unwrap();
let status = Command::new("tar").current_dir(&served).args(["-czf", "tree.tar.gz", "tree"]).status().unwrap();
assert!(status.success());
let hash = fetch::files_hash(&served.join("tree"));
let path = served.join("tree.tar.gz").display().to_string().replace('\\', "/");
let url = format!("file:///{}", path.trim_start_matches('/'));
Self { work, hash, url }
}
fn path(&self, name: &str) -> PathBuf {
self.work.path().join(name)
}
fn build(&self, out: &str) -> Command {
let mut command = Command::new(std::env::current_exe().unwrap());
command.args(["--exact", "build_script", "--nocapture"]);
command.env("FETCH_TEST_URL", &self.url).env("FETCH_TEST_HASH", &self.hash);
command.env("OUT_DIR", self.path(out)).env("MANIMGX_SOURCES", self.path("sources"));
command.stdout(Stdio::piped()).stderr(Stdio::piped());
command
}
fn check(&self, out: &str) {
assert_eq!(fetch::files_hash(&self.path(out).join(&self.hash)), self.hash);
assert_eq!(entries(&self.path("sources")), ["tree.tar.gz"]);
assert_eq!(entries(&self.path(out)), std::slice::from_ref(&self.hash));
}
}
fn entries(path: &Path) -> Vec<String> {
let mut names: Vec<_> = fs::read_dir(path).unwrap().map(|entry| entry.unwrap().file_name().into_string().unwrap()).collect();
names.sort();
names
}
fn success(output: Output) {
assert!(output.status.success(), "{}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
}
#[test]
fn build_script() {
// Children exercise the public API without mutating the test runner's environment.
if let Ok(url) = std::env::var("FETCH_TEST_URL") {
let acquire = if std::env::var("FETCH_TEST_KIND").as_deref() == Ok("file") { fetch::file } else { fetch::tree };
acquire(&url, &std::env::var("FETCH_TEST_HASH").unwrap());
}
}
#[test]
fn an_empty_folder_gathers_archives_and_a_full_one_builds_offline() {
let source = Source::new();
success(source.build("first").output().unwrap());
source.check("first");
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
success(source.build("second").output().unwrap());
source.check("second");
}
#[test]
fn release_tooling_collects_and_reuses_the_same_verified_source_store() {
use sha2::{Digest, Sha256};
let source = Source::new();
let bytes = fs::read(source.path("served/tree.tar.gz")).unwrap();
let hash = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect::<String>();
for out in ["first", "offline"] {
let output = Command::new(env!("CARGO_BIN_EXE_fetch-file"))
.args([&source.url, &hash])
.env("OUT_DIR", source.path(out))
.env("MANIMGX_SOURCES", source.path("sources"))
.output().unwrap();
success(output);
assert_eq!(fs::read(source.path(out).join(&hash)).unwrap(), bytes);
assert_eq!(fs::read(source.path("sources/tree.tar.gz")).unwrap(), bytes);
if out == "first" { fs::remove_file(source.path("served/tree.tar.gz")).unwrap(); }
}
}
#[test]
fn no_archive_is_retained_without_a_source_store_and_the_tree_is_reused_offline() {
let source = Source::new();
success(source.build("first").env_remove("MANIMGX_SOURCES").output().unwrap());
assert_eq!(entries(&source.path("first")), std::slice::from_ref(&source.hash));
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
success(source.build("first").env_remove("MANIMGX_SOURCES").output().unwrap());
}
#[test]
fn a_warm_tree_still_populates_a_new_source_store() {
let source = Source::new();
success(source.build("first").env_remove("MANIMGX_SOURCES").output().unwrap());
success(source.build("first").output().unwrap());
source.check("first");
}
#[test]
fn invalid_downloads_publish_nothing_and_a_later_attempt_recovers() {
let source = Source::new();
let archive = source.path("served/tree.tar.gz");
let valid = fs::read(&archive).unwrap();
for invalid in [b"<html>upstream error</html>".as_slice(), &valid[..valid.len() / 2]] {
fs::write(&archive, invalid).unwrap();
let output = source.build("first").output().unwrap();
assert!(!output.status.success());
assert!(String::from_utf8_lossy(&output.stderr).contains("tar"));
assert!(entries(&source.path("sources")).is_empty());
assert!(entries(&source.path("first")).is_empty());
}
fs::write(archive, valid).unwrap();
success(source.build("first").output().unwrap());
source.check("first");
}
#[test]
fn a_successful_http_response_must_still_be_a_verified_archive() {
let mut source = Source::new();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
source.url = format!("http://{}/tree.tar.gz", listener.local_addr().unwrap());
let valid = fs::read(source.path("served/tree.tar.gz")).unwrap();
let server = std::thread::spawn(move || {
for body in [b"<html>upstream challenge</html>".to_vec(), valid] {
let (mut stream, _) = listener.accept().unwrap();
stream.set_read_timeout(Some(std::time::Duration::from_secs(10))).unwrap();
let mut request = Vec::new();
while !request.ends_with(b"\r\n\r\n") {
let mut byte = [0];
stream.read_exact(&mut byte).unwrap();
request.push(byte[0]);
}
write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).unwrap();
stream.write_all(&body).unwrap();
}
});
assert!(!source.build("first").output().unwrap().status.success());
assert!(entries(&source.path("sources")).is_empty());
success(source.build("first").output().unwrap());
source.check("first");
server.join().unwrap();
}
#[test]
fn a_source_that_temporarily_refuses_connections_recovers() {
use std::sync::mpsc;
use std::time::Duration;
let mut source = Source::new();
let reserved = TcpListener::bind("127.0.0.1:0").unwrap();
let address = reserved.local_addr().unwrap();
drop(reserved); // The first requests see a refused connection, not an HTTP error.
source.url = format!("http://{address}/tree.tar.gz");
let archive = fs::read(source.path("served/tree.tar.gz")).unwrap();
let (finished, stopped) = mpsc::channel();
let server = std::thread::spawn(move || {
if stopped.recv_timeout(Duration::from_secs(2)).is_ok() {
return; // A downloader without retries already failed.
}
let listener = TcpListener::bind(address).unwrap();
listener.set_nonblocking(true).unwrap();
loop {
match listener.accept() {
Ok((mut stream, _)) => {
// Winsock inherits the listener's mode; request reads are blocking.
stream.set_nonblocking(false).unwrap();
stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap();
let mut request = Vec::new();
while !request.ends_with(b"\r\n\r\n") {
let mut byte = [0];
stream.read_exact(&mut byte).unwrap();
request.push(byte[0]);
}
write!(stream, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", archive.len()).unwrap();
stream.write_all(&archive).unwrap();
return;
}
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
if stopped.recv_timeout(Duration::from_millis(10)).is_ok() {
return;
}
}
Err(error) => panic!("{error}"),
}
}
});
let result = source.build("first").output().unwrap();
let _ = finished.send(());
server.join().unwrap();
success(result);
source.check("first");
}
#[test]
fn the_pin_rejects_valid_archives_with_different_files() {
let source = Source::new();
let output = source.build("first").env("FETCH_TEST_HASH", "0".repeat(64)).output().unwrap();
assert!(!output.status.success());
let error = String::from_utf8_lossy(&output.stderr);
assert!(error.contains(&source.hash) && error.contains(&"0".repeat(64)), "{error}");
assert!(entries(&source.path("sources")).is_empty());
assert!(entries(&source.path("first")).is_empty());
}
#[test]
fn a_corrupted_archive_is_replaced_only_by_verified_content() {
let source = Source::new();
success(source.build("first").output().unwrap());
let archive = source.path("sources/tree.tar.gz");
fs::write(&archive, "broken cache").unwrap();
success(source.build("second").output().unwrap());
assert_eq!(fs::read(archive).unwrap(), fs::read(source.path("served/tree.tar.gz")).unwrap());
source.check("second");
}
#[test]
fn cached_files_with_the_wrong_hash_are_refetched() {
let source = Source::new();
success(source.build("first").output().unwrap());
let served = source.path("served");
fs::write(served.join("tree/a.txt"), "tampered").unwrap();
let status = Command::new("tar").current_dir(&served).args(["-czf", "../sources/tree.tar.gz", "tree"]).status().unwrap();
assert!(status.success());
success(source.build("second").output().unwrap());
source.check("second");
}
#[test]
fn failed_recovery_does_not_replace_cached_data_or_publish_a_tree() {
let source = Source::new();
success(source.build("first").output().unwrap());
let archive = source.path("sources/tree.tar.gz");
fs::write(&archive, "broken cache").unwrap();
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
let output = source.build("second").output().unwrap();
assert!(!output.status.success());
let error = String::from_utf8_lossy(&output.stderr);
assert!(error.contains("invalid cached archive") && error.contains("curl"), "{error}");
assert_eq!(fs::read_to_string(archive).unwrap(), "broken cache");
assert!(entries(&source.path("second")).is_empty());
}
#[test]
fn concurrent_builds_publish_whole_archives_and_trees() {
let source = Source::new();
// Share both a source store and some output directories, as independent Cargo invocations do.
let children: Vec<_> = (0..12).map(|i| source.build(&format!("build-{}", i % 3)).spawn().unwrap()).collect();
for child in children {
success(child.wait_with_output().unwrap());
}
for i in 0..3 {
source.check(&format!("build-{i}"));
}
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
success(source.build("offline").output().unwrap());
source.check("offline");
}
#[test]
fn archives_without_a_top_folder_are_supported() {
let source = Source::new();
let status = Command::new("tar").current_dir(source.path("served/tree")).args(["-czf", "../tree.tar.gz", "a.txt", "sub"]).status().unwrap();
assert!(status.success());
success(source.build("first").output().unwrap());
source.check("first");
}
#[test]
fn output_and_source_paths_may_contain_spaces() {
let source = Source::new();
let sources = source.path("source archives");
success(source.build("build output").env("MANIMGX_SOURCES", &sources).output().unwrap());
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
success(source.build("offline output").env("MANIMGX_SOURCES", sources).output().unwrap());
assert_eq!(fetch::files_hash(&source.path("offline output").join(&source.hash)), source.hash);
}
#[cfg(unix)]
#[test]
fn an_offline_source_store_can_be_read_only() {
use std::os::unix::fs::PermissionsExt;
let source = Source::new();
success(source.build("first").output().unwrap());
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
fs::set_permissions(source.path("sources"), fs::Permissions::from_mode(0o555)).unwrap();
let output = source.build("second").output().unwrap();
fs::set_permissions(source.path("sources"), fs::Permissions::from_mode(0o755)).unwrap();
success(output);
source.check("second");
}
#[test]
fn an_immutable_file_is_verified_collected_and_reused_offline() {
use sha2::{Digest, Sha256};
let mut source = Source::new();
let bytes = fs::read(source.path("served/tree.tar.gz")).unwrap();
source.hash = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect();
success(source.build("first").env("FETCH_TEST_KIND", "file").env_remove("MANIMGX_SOURCES").output().unwrap());
success(source.build("first").env("FETCH_TEST_KIND", "file").output().unwrap());
fs::remove_file(source.path("served/tree.tar.gz")).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(source.path("sources"), fs::Permissions::from_mode(0o555)).unwrap();
}
let offline = source.build("offline").env("FETCH_TEST_KIND", "file").output().unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(source.path("sources"), fs::Permissions::from_mode(0o755)).unwrap();
}
success(offline);
assert_eq!(fs::read(source.path("offline").join(&source.hash)).unwrap(), bytes);
assert_eq!(entries(&source.path("sources")), ["tree.tar.gz"]);
}
#[test]
fn immutable_file_cache_corruption_is_repaired_only_by_verified_bytes() {
use sha2::{Digest, Sha256};
let mut source = Source::new();
let bytes = fs::read(source.path("served/tree.tar.gz")).unwrap();
source.hash = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect();
success(source.build("first").env("FETCH_TEST_KIND", "file").output().unwrap());
fs::write(source.path("sources/tree.tar.gz"), "corrupt cache").unwrap();
fs::write(source.path("served/tree.tar.gz"), "invalid response").unwrap();
let output = source.build("second").env("FETCH_TEST_KIND", "file").output().unwrap();
assert!(!output.status.success());
assert!(String::from_utf8_lossy(&output.stderr).contains("its bytes hash to"));
assert_eq!(fs::read_to_string(source.path("sources/tree.tar.gz")).unwrap(), "corrupt cache");
assert!(entries(&source.path("second")).is_empty());
fs::write(source.path("served/tree.tar.gz"), &bytes).unwrap();
success(source.build("second").env("FETCH_TEST_KIND", "file").output().unwrap());
assert_eq!(fs::read(source.path("sources/tree.tar.gz")).unwrap(), bytes);
}
#[test]
fn concurrent_builds_publish_whole_immutable_files() {
use sha2::{Digest, Sha256};
let mut source = Source::new();
let bytes = fs::read(source.path("served/tree.tar.gz")).unwrap();
source.hash = Sha256::digest(&bytes).iter().map(|byte| format!("{byte:02x}")).collect();
let children: Vec<_> = (0..12).map(|i| source.build(&format!("build-{}", i % 3)).env("FETCH_TEST_KIND", "file").spawn().unwrap()).collect();
for child in children { success(child.wait_with_output().unwrap()); }
for i in 0..3 { assert_eq!(fs::read(source.path(&format!("build-{i}")).join(&source.hash)).unwrap(), bytes); }
}