The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Keep others' sources between CI runs, and hold a refused test port
Sina Atalaya7be82b2c405
diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yamlindex 0abe30b6..75091da0 100644--- a/.github/workflows/benchmark.yaml+++ b/.github/workflows/benchmark.yaml@@ -50,6 +50,8 @@ jobs:timeout-minutes: 120permissions:contents: read+ env:+ MANIMGX_SOURCES: ${{ github.workspace }}/sourcessteps:# all of history: the commit compared with is made from it- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -70,6 +72,14 @@ jobs:shared-key: enginesave-if: ${{ github.ref == 'refs/heads/main' }}+ # The source store: the archives build scripts fetch, kept between runs, so a host+ # that is down stops only a build whose pins changed (see `rust/fetch`).+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: sources+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}+ restore-keys: sources-+# a pull request is checked out merged into its base: the merge's first parent is the base- name: Benchmarkrun: |diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yamlindex 753d9d19..db85aa7d 100644--- a/.github/workflows/create-wheels.yaml+++ b/.github/workflows/create-wheels.yaml@@ -57,10 +57,13 @@ jobs:--target wasm32-unknown-emscripten --target wasm32-unknown-unknown# cibuildwheel copies /project into its container; its documented /host mount- # lets the verified source store survive the container, alongside the wheel.+ # lets the verified source store survive the container, alongside the wheel. It+ # gathers the complete source: DXC's too, which only Windows builds read.- name: Retain the Linux builds' sourcesif: runner.os == 'Linux' && matrix.platform != 'pyodide'- run: echo "MANIMGX_SOURCES=/host${GITHUB_WORKSPACE}/sources" >> "${GITHUB_ENV}"+ run: |+ echo "MANIMGX_SOURCES=/host${GITHUB_WORKSPACE}/sources" >> "${GITHUB_ENV}"+ echo "MANIMGX_COMPLETE_SOURCE=1" >> "${GITHUB_ENV}"- name: Build and test the wheelrun: just build-wheel "${PLATFORM}"diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yamlindex 04f13b4b..8709fcec 100644--- a/.github/workflows/deploy-docs.yaml+++ b/.github/workflows/deploy-docs.yaml@@ -32,6 +32,8 @@ jobs:timeout-minutes: 90permissions:contents: read+ env:+ MANIMGX_SOURCES: ${{ github.workspace }}/sourcessteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:@@ -54,6 +56,14 @@ jobs:shared-key: enginesave-if: ${{ github.ref == 'refs/heads/main' }}+ # The source store: the archives build scripts fetch, kept between runs, so a host+ # that is down stops only a build whose pins changed (see `rust/fetch`).+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: sources+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}+ restore-keys: sources-+- name: Identify the rendered examplesid: filmsrun: |diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex d8b51996..8c832f2f 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -94,6 +94,8 @@ jobs:timeout-minutes: 60permissions:contents: read+ env:+ MANIMGX_SOURCES: ${{ github.workspace }}/sourcessteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:@@ -113,6 +115,14 @@ jobs:shared-key: enginesave-if: ${{ github.event_name != 'pull_request' }}+ # The source store: the archives build scripts fetch, kept between runs, so a host+ # that is down stops only a build whose pins changed (see `rust/fetch`).+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: sources+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}+ restore-keys: sources-+- name: Test source acquisition and linked codecsrun: cargo test --locked --release --manifest-path rust/Cargo.toml -p fetch -p ffmpegdiff --git a/docs/content/developer-guide/engine.md b/docs/content/developer-guide/engine.mdindex af66ac9b..713655b6 100644--- a/docs/content/developer-guide/engine.md+++ b/docs/content/developer-guide/engine.md@@ -369,7 +369,10 @@ for crates. `MANIMGX_SOURCES` names a folder for the archives (an absolute path)an archive there, and downloads it there first if it isn't. A folder that holds them buildsoffline; one shared by your checkouts downloads each archive once; and a build into an emptyone gathers what it was built from. A build script fetches the same archives on every target,-whatever it compiles of them, so one build gathers what every build reads.+whatever it compiles of them, but for DXC, which only Windows builds read: a build of the+complete source (`MANIMGX_COMPLETE_SOURCE`, which the release sets beside its store) gathers+it on every target, so one such build gathers what every build reads. CI's everyday builds+keep a store between runs, so a host that is down stops only a build whose pins changed.The release ships `manimgx-X.Y.Z-source.tar.xz` beside the wheels (`just build-source` makes it).It is the committed repository, identified by `SOURCE_COMMIT`, with `vendor/`, every crate `rust/Cargo.lock` namesdiff --git a/justfile b/justfileindex cd984c79..591933cf 100644--- a/justfile+++ b/justfile@@ -201,7 +201,7 @@ build-source sources="release-sources": build-sdistcp -R "${sources}" sourcesmkdir .cargocargo vendor --locked --manifest-path rust/Cargo.toml vendor > .cargo/config.toml- printf '\n[net]\noffline = true\n\n[env]\nMANIMGX_SOURCES = { value = "sources", relative = true }\n' \+ printf '\n[net]\noffline = true\n\n[env]\nMANIMGX_SOURCES = { value = "sources", relative = true }\nMANIMGX_COMPLETE_SOURCE = "1"\n' \>> .cargo/config.tomlCARGO_TARGET_DIR="${work}/target" MANIMGX_SOURCES="$PWD/sources" \sh scripts/release/build_lavapipe.sh "${work}/lavapipe" --sources-onlydiff --git a/pyproject.toml b/pyproject.tomlindex 762905cb..60c12151 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -141,7 +141,7 @@ before-all = ["sh scripts/release/build_lavapipe.sh {package}/src/manimgx/lavapipe",]environment = { PATH = "$HOME/.cargo/bin:$PATH" }-environment-pass = ["MANIMGX_SOURCES"]+environment-pass = ["MANIMGX_SOURCES", "MANIMGX_COMPLETE_SOURCE"]repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && python {project}/scripts/release/linux_sources.py {dest_dir}"[tool.cibuildwheel.pyodide]diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 3a10a3e9..e6c7e10d 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -1026,6 +1026,7 @@ name = "fetch"version = "0.1.0"dependencies = ["sha2",+ "socket2","tempfile",]@@ -3845,6 +3846,16 @@ dependencies = ["serde",]+[[package]]+name = "socket2"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"+dependencies = [+ "libc",+ "windows-sys 0.61.2",+]+[[package]]name = "spin"version = "0.9.9"diff --git a/rust/dxc/README.md b/rust/dxc/README.mdindex e0054289..92687270 100644--- a/rust/dxc/README.md+++ b/rust/dxc/README.md@@ -5,9 +5,10 @@ beside the engine. The engine selects it explicitly by absolute path. `dxil.dll`optional with this compiler version and is not distributed. `LICENSE-DXC` carriesthe compiler's complete notices, including its SPIR-V dependencies.-With `MANIMGX_SOURCES` set, this crate collects the original binary release and-matching compiler source, plus every Git submodule at that revision. Ordinary-non-Windows builds do not fetch DXC. The source bundle can build ManimGX offline+A build of the complete source (`MANIMGX_COMPLETE_SOURCE`, set beside the source+store `MANIMGX_SOURCES`) collects the original binary release and matching compiler+source, plus every Git submodule at that revision, on any platform. Other non-Windows+builds do not fetch DXC, with a source store or without. The source bundle can build ManimGX offlineusing the included compiler payload.To rebuild the compiler itself, unpack these source archives from the completediff --git a/rust/dxc/src/lib.rs b/rust/dxc/src/lib.rsindex 5ee43453..9ebc33e6 100644--- a/rust/dxc/src/lib.rs+++ b/rust/dxc/src/lib.rs@@ -16,13 +16,15 @@ const SUBMODULES: &[(&str, &str)] = &[("https://github.com/KhronosGroup/SPIRV-Tools/archive/ef96ed763b43b59b33b31b362f09a02b729fa1c9.tar.gz", "a2e2378712cd1a21fdbfc24c7fc8533df44b33ca0a051e258e2f3ddf3a7b2ee5"),];-/// Stage DXC in the engine's OUT_DIR for maturin. A complete-source build on any platform-/// collects both the distributable and its matching source; ordinary non-Windows builds do no-/// DXC work. Only dxcompiler.dll is needed: since DXC 1.8.2502, dxil.dll is optional.+/// Stage DXC in the engine's OUT_DIR for maturin. A build of the complete source on any+/// platform (MANIMGX_COMPLETE_SOURCE, set beside its source store) collects both the+/// distributable and its matching source; other non-Windows builds do no DXC work, with a+/// source store or without. Only dxcompiler.dll is needed: since DXC 1.8.2502, dxil.dll is+/// optional.pub fn prepare() {- println!("cargo:rerun-if-env-changed=MANIMGX_SOURCES");+ println!("cargo:rerun-if-env-changed=MANIMGX_COMPLETE_SOURCE");let windows = std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows");- let collecting = std::env::var_os("MANIMGX_SOURCES").is_some();+ let collecting = std::env::var_os("MANIMGX_COMPLETE_SOURCE").is_some();if !windows && !collecting { return; }let archive = fetch::file(BINARY_URL, BINARY_HASH);if collecting {diff --git a/rust/fetch/Cargo.toml b/rust/fetch/Cargo.tomlindex 15c062ca..1f085153 100644--- a/rust/fetch/Cargo.toml+++ b/rust/fetch/Cargo.toml@@ -9,3 +9,7 @@ description = "Others' sources for a build script: fetched at build time, named[dependencies]sha2 = "0.11.0"tempfile = "3.27.0"++[dev-dependencies]+# a test port bound before it listens: connections to it are refused, and it stays the test's+socket2 = "0.6.5"diff --git a/rust/fetch/tests/folder.rs b/rust/fetch/tests/folder.rsindex 64c2d542..d058683e 100644--- a/rust/fetch/tests/folder.rs+++ b/rust/fetch/tests/folder.rs@@ -164,9 +164,11 @@ fn a_source_that_temporarily_refuses_connections_recovers() {use std::time::Duration;let mut source = Source::new();- let reserved = TcpListener::bind("127.0.0.1:0").unwrap();- let address = reserved.local_addr().unwrap();- drop(reserved); // The first requests see a refused connection, not an HTTP error.+ // The port is the test's throughout: bound, so no other socket is given it, but not listening+ // yet, so the first requests see a refused connection, not an HTTP error.+ let socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, None).unwrap();+ socket.bind(&std::net::SocketAddr::from(([127, 0, 0, 1], 0)).into()).unwrap();+ let address = socket.local_addr().unwrap().as_socket().unwrap();source.url = format!("http://{address}/tree.tar.gz");let archive = fs::read(source.path("served/tree.tar.gz")).unwrap();let (finished, stopped) = mpsc::channel();@@ -174,7 +176,8 @@ fn a_source_that_temporarily_refuses_connections_recovers() {if stopped.recv_timeout(Duration::from_secs(2)).is_ok() {return; // A downloader without retries already failed.}- let listener = TcpListener::bind(address).unwrap();+ socket.listen(1).unwrap();+ let listener = TcpListener::from(socket);listener.set_nonblocking(true).unwrap();loop {match listener.accept() {