Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python

Keep others' sources between CI runs, and hold a refused test port

Sina Atalaya7be82b2c405
diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
index 0abe30b6..75091da0 100644
--- a/.github/workflows/benchmark.yaml
+++ b/.github/workflows/benchmark.yaml
@@ -50,6 +50,8 @@ jobs:
timeout-minutes: 120
permissions:
contents: read
+ env:
+ MANIMGX_SOURCES: ${{ github.workspace }}/sources
steps:
# all of history: the commit compared with is made from it
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -70,6 +72,14 @@ jobs:
shared-key: engine
save-if: ${{ github.ref == 'refs/heads/main' }}
+ # The source store: the archives build scripts fetch, kept between runs, so a host
+ # that is down stops only a build whose pins changed (see `rust/fetch`).
+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: sources
+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}
+ restore-keys: sources-
+
# a pull request is checked out merged into its base: the merge's first parent is the base
- name: Benchmark
run: |
diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yaml
index 753d9d19..db85aa7d 100644
--- a/.github/workflows/create-wheels.yaml
+++ b/.github/workflows/create-wheels.yaml
@@ -57,10 +57,13 @@ jobs:
--target wasm32-unknown-emscripten --target wasm32-unknown-unknown
# cibuildwheel copies /project into its container; its documented /host mount
- # lets the verified source store survive the container, alongside the wheel.
+ # lets the verified source store survive the container, alongside the wheel. It
+ # gathers the complete source: DXC's too, which only Windows builds read.
- name: Retain the Linux builds' sources
if: runner.os == 'Linux' && matrix.platform != 'pyodide'
- run: echo "MANIMGX_SOURCES=/host${GITHUB_WORKSPACE}/sources" >> "${GITHUB_ENV}"
+ run: |
+ echo "MANIMGX_SOURCES=/host${GITHUB_WORKSPACE}/sources" >> "${GITHUB_ENV}"
+ echo "MANIMGX_COMPLETE_SOURCE=1" >> "${GITHUB_ENV}"
- name: Build and test the wheel
run: just build-wheel "${PLATFORM}"
diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml
index 04f13b4b..8709fcec 100644
--- a/.github/workflows/deploy-docs.yaml
+++ b/.github/workflows/deploy-docs.yaml
@@ -32,6 +32,8 @@ jobs:
timeout-minutes: 90
permissions:
contents: read
+ env:
+ MANIMGX_SOURCES: ${{ github.workspace }}/sources
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -54,6 +56,14 @@ jobs:
shared-key: engine
save-if: ${{ github.ref == 'refs/heads/main' }}
+ # The source store: the archives build scripts fetch, kept between runs, so a host
+ # that is down stops only a build whose pins changed (see `rust/fetch`).
+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: sources
+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}
+ restore-keys: sources-
+
- name: Identify the rendered examples
id: films
run: |
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index d8b51996..8c832f2f 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -94,6 +94,8 @@ jobs:
timeout-minutes: 60
permissions:
contents: read
+ env:
+ MANIMGX_SOURCES: ${{ github.workspace }}/sources
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -113,6 +115,14 @@ jobs:
shared-key: engine
save-if: ${{ github.event_name != 'pull_request' }}
+ # The source store: the archives build scripts fetch, kept between runs, so a host
+ # that is down stops only a build whose pins changed (see `rust/fetch`).
+ - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: sources
+ key: sources-${{ hashFiles('rust/*/build.rs', 'rust/dxc/src/lib.rs') }}
+ restore-keys: sources-
+
- name: Test source acquisition and linked codecs
run: cargo test --locked --release --manifest-path rust/Cargo.toml -p fetch -p ffmpeg
diff --git a/docs/content/developer-guide/engine.md b/docs/content/developer-guide/engine.md
index af66ac9b..713655b6 100644
--- a/docs/content/developer-guide/engine.md
+++ b/docs/content/developer-guide/engine.md
@@ -369,7 +369,10 @@ for crates. `MANIMGX_SOURCES` names a folder for the archives (an absolute path)
an archive there, and downloads it there first if it isn't. A folder that holds them builds
offline; one shared by your checkouts downloads each archive once; and a build into an empty
one gathers what it was built from. A build script fetches the same archives on every target,
-whatever it compiles of them, so one build gathers what every build reads.
+whatever it compiles of them, but for DXC, which only Windows builds read: a build of the
+complete source (`MANIMGX_COMPLETE_SOURCE`, which the release sets beside its store) gathers
+it on every target, so one such build gathers what every build reads. CI's everyday builds
+keep a store between runs, so a host that is down stops only a build whose pins changed.
The release ships `manimgx-X.Y.Z-source.tar.xz` beside the wheels (`just build-source` makes it).
It is the committed repository, identified by `SOURCE_COMMIT`, with `vendor/`, every crate `rust/Cargo.lock` names
diff --git a/justfile b/justfile
index cd984c79..591933cf 100644
--- a/justfile
+++ b/justfile
@@ -201,7 +201,7 @@ build-source sources="release-sources": build-sdist
cp -R "${sources}" sources
mkdir .cargo
cargo vendor --locked --manifest-path rust/Cargo.toml vendor > .cargo/config.toml
- printf '\n[net]\noffline = true\n\n[env]\nMANIMGX_SOURCES = { value = "sources", relative = true }\n' \
+ printf '\n[net]\noffline = true\n\n[env]\nMANIMGX_SOURCES = { value = "sources", relative = true }\nMANIMGX_COMPLETE_SOURCE = "1"\n' \
>> .cargo/config.toml
CARGO_TARGET_DIR="${work}/target" MANIMGX_SOURCES="$PWD/sources" \
sh scripts/release/build_lavapipe.sh "${work}/lavapipe" --sources-only
diff --git a/pyproject.toml b/pyproject.toml
index 762905cb..60c12151 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -141,7 +141,7 @@ before-all = [
"sh scripts/release/build_lavapipe.sh {package}/src/manimgx/lavapipe",
]
environment = { PATH = "$HOME/.cargo/bin:$PATH" }
-environment-pass = ["MANIMGX_SOURCES"]
+environment-pass = ["MANIMGX_SOURCES", "MANIMGX_COMPLETE_SOURCE"]
repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && python {project}/scripts/release/linux_sources.py {dest_dir}"
[tool.cibuildwheel.pyodide]
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 3a10a3e9..e6c7e10d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -1026,6 +1026,7 @@ name = "fetch"
version = "0.1.0"
dependencies = [
"sha2",
+ "socket2",
"tempfile",
]
@@ -3845,6 +3846,16 @@ dependencies = [
"serde",
]
+[[package]]
+name = "socket2"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "spin"
version = "0.9.9"
diff --git a/rust/dxc/README.md b/rust/dxc/README.md
index e0054289..92687270 100644
--- a/rust/dxc/README.md
+++ b/rust/dxc/README.md
@@ -5,9 +5,10 @@ beside the engine. The engine selects it explicitly by absolute path. `dxil.dll`
optional with this compiler version and is not distributed. `LICENSE-DXC` carries
the compiler's complete notices, including its SPIR-V dependencies.
-With `MANIMGX_SOURCES` set, this crate collects the original binary release and
-matching compiler source, plus every Git submodule at that revision. Ordinary
-non-Windows builds do not fetch DXC. The source bundle can build ManimGX offline
+A build of the complete source (`MANIMGX_COMPLETE_SOURCE`, set beside the source
+store `MANIMGX_SOURCES`) collects the original binary release and matching compiler
+source, plus every Git submodule at that revision, on any platform. Other non-Windows
+builds do not fetch DXC, with a source store or without. The source bundle can build ManimGX offline
using the included compiler payload.
To rebuild the compiler itself, unpack these source archives from the complete
diff --git a/rust/dxc/src/lib.rs b/rust/dxc/src/lib.rs
index 5ee43453..9ebc33e6 100644
--- a/rust/dxc/src/lib.rs
+++ b/rust/dxc/src/lib.rs
@@ -16,13 +16,15 @@ const SUBMODULES: &[(&str, &str)] = &[
("https://github.com/KhronosGroup/SPIRV-Tools/archive/ef96ed763b43b59b33b31b362f09a02b729fa1c9.tar.gz", "a2e2378712cd1a21fdbfc24c7fc8533df44b33ca0a051e258e2f3ddf3a7b2ee5"),
];
-/// Stage DXC in the engine's OUT_DIR for maturin. A complete-source build on any platform
-/// collects both the distributable and its matching source; ordinary non-Windows builds do no
-/// DXC work. Only dxcompiler.dll is needed: since DXC 1.8.2502, dxil.dll is optional.
+/// Stage DXC in the engine's OUT_DIR for maturin. A build of the complete source on any
+/// platform (MANIMGX_COMPLETE_SOURCE, set beside its source store) collects both the
+/// distributable and its matching source; other non-Windows builds do no DXC work, with a
+/// source store or without. Only dxcompiler.dll is needed: since DXC 1.8.2502, dxil.dll is
+/// optional.
pub fn prepare() {
- println!("cargo:rerun-if-env-changed=MANIMGX_SOURCES");
+ println!("cargo:rerun-if-env-changed=MANIMGX_COMPLETE_SOURCE");
let windows = std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows");
- let collecting = std::env::var_os("MANIMGX_SOURCES").is_some();
+ let collecting = std::env::var_os("MANIMGX_COMPLETE_SOURCE").is_some();
if !windows && !collecting { return; }
let archive = fetch::file(BINARY_URL, BINARY_HASH);
if collecting {
diff --git a/rust/fetch/Cargo.toml b/rust/fetch/Cargo.toml
index 15c062ca..1f085153 100644
--- a/rust/fetch/Cargo.toml
+++ b/rust/fetch/Cargo.toml
@@ -9,3 +9,7 @@ description = "Others' sources for a build script: fetched at build time, named
[dependencies]
sha2 = "0.11.0"
tempfile = "3.27.0"
+
+[dev-dependencies]
+# a test port bound before it listens: connections to it are refused, and it stays the test's
+socket2 = "0.6.5"
diff --git a/rust/fetch/tests/folder.rs b/rust/fetch/tests/folder.rs
index 64c2d542..d058683e 100644
--- a/rust/fetch/tests/folder.rs
+++ b/rust/fetch/tests/folder.rs
@@ -164,9 +164,11 @@ fn a_source_that_temporarily_refuses_connections_recovers() {
use std::time::Duration;
let mut source = Source::new();
- let reserved = TcpListener::bind("127.0.0.1:0").unwrap();
- let address = reserved.local_addr().unwrap();
- drop(reserved); // The first requests see a refused connection, not an HTTP error.
+ // The port is the test's throughout: bound, so no other socket is given it, but not listening
+ // yet, so the first requests see a refused connection, not an HTTP error.
+ let socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, None).unwrap();
+ socket.bind(&std::net::SocketAddr::from(([127, 0, 0, 1], 0)).into()).unwrap();
+ let address = socket.local_addr().unwrap().as_socket().unwrap();
source.url = format!("http://{address}/tree.tar.gz");
let archive = fs::read(source.path("served/tree.tar.gz")).unwrap();
let (finished, stopped) = mpsc::channel();
@@ -174,7 +176,8 @@ fn a_source_that_temporarily_refuses_connections_recovers() {
if stopped.recv_timeout(Duration::from_secs(2)).is_ok() {
return; // A downloader without retries already failed.
}
- let listener = TcpListener::bind(address).unwrap();
+ socket.listen(1).unwrap();
+ let listener = TcpListener::from(socket);
listener.set_nonblocking(true).unwrap();
loop {
match listener.accept() {