Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(models): mirror every downloadable model on our own releases

Jareer93274efe395e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 29963705..2499eddb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -55,6 +55,11 @@ jobs:
# keeps en.json the inventory of them and every shipped language
# keyed off it. See TRANSLATING.md.
- run: python3 scripts/locales.py --check
+ # The models Concat downloads are mirrored onto a release of this
+ # repository; models/manifest.toml is what fills that mirror and the
+ # engine's tables are what a download is checked against. This keeps
+ # the two saying the same thing. See models/manifest.toml.
+ - run: python3 scripts/models.py --check
# -D warnings covers rustc lints too (missing_docs among them), so a
# public item without documentation fails here, not in review. Lints
# before tests: they finish in a fraction of the time and catch the
diff --git a/.github/workflows/models.yml b/.github/workflows/models.yml
new file mode 100644
index 00000000..c30e8da5
--- /dev/null
+++ b/.github/workflows/models.yml
@@ -0,0 +1,122 @@
+name: Mirror models
+
+# Fills Concat's model mirror from upstream and publishes it as a release.
+#
+# Concat fetches its cutout networks, its Kokoro voice bank and its whisper
+# models on demand, and it fetches them from a release of this repository
+# rather than from whoever published them. models/manifest.toml is the table
+# of what those are; this workflow is what turns that table into the release
+# the app asks for.
+#
+# Run it when a model is added to the table, or when one's bytes change and
+# its release tag is bumped. It is not part of cutting an app release: the
+# mirror has its own tag, app releases point at it by name, and an unchanged
+# model is never re-uploaded. Roughly 2 GB down and 2 GB up, so give it time.
+#
+# A model whose row has no digest yet is recorded: the job fetches it,
+# computes the sha256, writes it into models/manifest.toml and into the
+# engine's table beside it, and uploads the patch as an artifact for a
+# maintainer to commit. Until that lands, the app accepts the download
+# unverified, which is what the empty digest means.
+on:
+ workflow_dispatch:
+ inputs:
+ only:
+ description: A family (cutout, tts, whisper) or one model id; blank for all
+ required: false
+ default: ""
+ publish:
+ description: Upload to the release tag, rather than only fetching and checking
+ type: boolean
+ required: false
+ default: true
+
+concurrency:
+ group: models
+ cancel-in-progress: false
+
+jobs:
+ mirror:
+ name: Fill and publish
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/checkout@v7
+
+ - name: The table
+ id: table
+ run: |
+ set -euo pipefail
+ release=$(sed -n 's/^release = "\(.*\)"/\1/p' models/manifest.toml | head -1)
+ test -n "$release"
+ echo "release=$release" >> "$GITHUB_OUTPUT"
+ python3 scripts/models.py --check
+
+ # The runner has room for the lot with a few gigabytes to spare, but
+ # not much more; the mirror folder is deleted with the runner.
+ - name: Disk before
+ run: df -h /
+
+ - name: Fetch from upstream
+ env:
+ ONLY: ${{ inputs.only }}
+ run: |
+ set -euo pipefail
+ args=()
+ if [ -n "$ONLY" ]; then args+=(--only "$ONLY"); fi
+ python3 scripts/models.py mirror --out mirror "${args[@]}"
+ ls -l mirror
+
+ # What the fetch recorded, if anything: the digests are now in the
+ # table and in the engine, and that diff is what wants committing.
+ - name: Digests recorded
+ id: recorded
+ run: |
+ set -euo pipefail
+ if git diff --quiet; then
+ echo "Nothing recorded: every model already carried its digest." >> "$GITHUB_STEP_SUMMARY"
+ echo "patch=false" >> "$GITHUB_OUTPUT"
+ else
+ {
+ echo "### Digests recorded"
+ echo
+ echo 'Commit this patch, or the app keeps accepting these downloads unverified.'
+ echo
+ echo '```diff'
+ git diff
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+ git diff > recorded-digests.patch
+ echo "patch=true" >> "$GITHUB_OUTPUT"
+ fi
+
+ - uses: actions/upload-artifact@v7
+ if: steps.recorded.outputs.patch == 'true'
+ with:
+ name: recorded-digests
+ path: recorded-digests.patch
+ if-no-files-found: error
+
+ # The mirror is one flat release: every file in the table is uploaded
+ # under its own name, and `--clobber` means re-running after a fixed
+ # row replaces the file rather than failing on it.
+ - name: Publish
+ if: inputs.publish
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RELEASE: ${{ steps.table.outputs.release }}
+ run: |
+ set -euo pipefail
+ if ! gh release view "$RELEASE" >/dev/null 2>&1; then
+ gh release create "$RELEASE" \
+ --title "Concat models ($RELEASE)" \
+ --notes "The models Concat downloads on demand, mirrored from the upstreams named in models/manifest.toml. Not an app release: see the tags beginning with v for those." \
+ --latest=false
+ fi
+ gh release upload "$RELEASE" mirror/* --clobber
+ gh release view "$RELEASE"
+
+ - name: Disk after
+ if: always()
+ run: df -h /
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 29bf4973..226bce29 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -49,7 +49,7 @@ jobs:
merge-multiple: true
# Named by the commit, so two testers can say which nightly they have.
- - name: Name the bundles by commit and write checksums
+ - name: Name the bundles by commit
id: stamp
run: |
set -euo pipefail
@@ -59,14 +59,34 @@ jobs:
for file in Concat-*; do
mv "$file" "${file/Concat-$version/Concat-$version-nightly.$sha}"
done
- sha256sum -- * > SHA256SUMS
ls -l
{
echo "sha=$sha"
echo "version=$version"
+ echo "stamped=$version-nightly.$sha"
echo "date=$(date -u +%Y-%m-%d)"
} >> "$GITHUB_OUTPUT"
+ # The same manifest a tagged release carries, so anything reading one
+ # can read the other: a row per bundle, and a row per model with the
+ # mirror it is fetched from. See scripts/models.py.
+ - name: Write the manifest
+ run: |
+ set -euo pipefail
+ python3 scripts/models.py release-manifest \
+ --version "${{ steps.stamp.outputs.stamped }}" \
+ --tag nightly \
+ --bundles bundles \
+ --out bundles/manifest.json
+ cat bundles/manifest.json
+
+ - name: Checksums
+ run: |
+ set -euo pipefail
+ cd bundles
+ sha256sum -- * > SHA256SUMS
+ ls -l
+
# Yesterday's bundles carry yesterday's commit in their names, so
# they would pile up beside today's; the release holds one set.
- name: Drop the previous nightly's assets
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index cee5e6ee..650ffcc0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -33,6 +33,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
tag: ${{ steps.tag.outputs.tag }}
+ version: ${{ steps.tag.outputs.version }}
steps:
- uses: actions/checkout@v7
with:
@@ -51,6 +52,9 @@ jobs:
;;
esac
echo "tag=$tag" >> "$GITHUB_OUTPUT"
+ # The bundles are named for the version, not the tag: v0.2.1 and
+ # v0.2.1-alpha.22 both produce Concat-0.2.1-* files.
+ echo "version=$version" >> "$GITHUB_OUTPUT"
build:
needs: [check]
@@ -74,6 +78,28 @@ jobs:
path: bundles
merge-multiple: true
+ # The tree the release is cut from: the notes are read out of its
+ # history, and the manifest out of its model table.
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ needs.check.outputs.tag }}
+ fetch-depth: 0
+ path: source
+
+ # One file that says where every other one is: a row per bundle with
+ # its size and digest, and a row per model with the mirror it is
+ # fetched from. An updater or a package script reads this instead of
+ # guessing at asset names. See scripts/models.py.
+ - name: Write the manifest
+ run: |
+ set -euo pipefail
+ python3 source/scripts/models.py release-manifest \
+ --version "${{ needs.check.outputs.version }}" \
+ --tag "${{ needs.check.outputs.tag }}" \
+ --bundles bundles \
+ --out bundles/manifest.json
+ cat bundles/manifest.json
+
- name: Checksums
run: |
set -euo pipefail
@@ -82,11 +108,6 @@ jobs:
sha256sum -- * > SHA256SUMS
# The notes are the commits since the previous tag; see the script.
- - uses: actions/checkout@v7
- with:
- ref: ${{ needs.check.outputs.tag }}
- fetch-depth: 0
- path: source
- name: Write the notes
id: notes
run: |
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c38d71d2..e407a208 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -58,6 +58,13 @@ If you added or changed text the interface shows, also run
Translations live in one JSON file per language — see
[`TRANSLATING.md`](TRANSLATING.md).
+If you added or changed a downloadable model, put it in
+[`models/manifest.toml`](models/manifest.toml) as well as the engine table it
+belongs to, and run `python3 scripts/models.py --check`; CI checks that too.
+A maintainer then runs the *Mirror models* workflow, which fetches the model,
+records its digest in both places and publishes it to the mirror the app
+downloads from.
+
New source files need a licence header — see below. Match the style of the code
around you; the engine avoids cleverness on purpose.
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index f9195aae..4f054aa5 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -18,8 +18,9 @@ on Windows and in CI. FFmpeg source code: https://ffmpeg.org/download.html
Transcription compiles whisper.cpp (https://github.com/ggml-org/whisper.cpp,
MIT) and ggml into the app through the `whisper-rs` crate. Whisper models
-are downloaded on demand from https://huggingface.co/ggerganov/whisper.cpp
-(MIT) and never bundled.
+originate at https://huggingface.co/ggerganov/whisper.cpp (MIT); Concat
+mirrors them and downloads them on demand from that mirror, and never
+bundles them. See "The model mirror" below.
## Slint — used under GPL-3.0-only
@@ -83,9 +84,10 @@ and AGPL-3.0 expressly permits that combination, so the combined binary may be
conveyed on those terms.
Kokoro voice model bundles (Apache-2.0,
-https://huggingface.co/hexgrad/Kokoro-82M) are downloaded on demand from the
-sherpa-onnx releases - including espeak-ng's data files - and are never
-bundled with the app.
+https://huggingface.co/hexgrad/Kokoro-82M) originate in the sherpa-onnx
+releases - including espeak-ng's data files. Concat mirrors those bundles
+and downloads them on demand from that mirror, and never bundles them with
+the app. See "The model mirror" below.
## The cutout models
@@ -99,8 +101,8 @@ except the first:
`engine/crates/concat-vision/models/NOTICE.md`. The answer when nothing
has been downloaded.
- Robust Video Matting, the MobileNetV3 variant, by Peter Lin and others
- (https://github.com/PeterL1n/RobustVideoMatting, GPL-3.0), downloaded on
- first use from that repository's releases. The person model.
+ (https://github.com/PeterL1n/RobustVideoMatting, GPL-3.0), from that
+ repository's releases, downloaded on first use. The person model.
- IS-Net from "Highly Accurate Dichotomous Image Segmentation" by Qin and
others (https://github.com/xuebinqin/DIS, Apache-2.0), in the ONNX
export the rembg project publishes
@@ -111,13 +113,32 @@ except the first:
(Apache-2.0), downloaded on first use. The brushes' model.
Downloaded models live in the app's data directory under `cutout-models`
-and are never bundled. They are all run by ONNX Runtime
+and are never bundled; the three that are downloaded come from Concat's own
+mirror of the sources named above, described below. They are all run by
+ONNX Runtime
(https://github.com/microsoft/onnxruntime, MIT) through the `ort` crate
(https://github.com/pykeio/ort, MIT OR Apache-2.0), with the platform's
own accelerator behind it: CoreML on macOS and iOS, DirectML on Windows,
NNAPI on Android. The runtime is linked statically from the builds pyke
publishes for each target.
+## The model mirror
+
+Every model named above is mirrored onto a release of Concat's own
+repository, and the app fetches it from there; the upstream each was
+obtained from is the fallback and is named above in every case. The table of
+what is mirrored, and the digest each download is checked against, is
+`models/manifest.toml`; `.github/workflows/models.yml` is what fills the
+mirror.
+
+Mirroring is redistribution, and each model keeps the licence it arrived
+under - Apache-2.0 for the Kokoro bundles, SlimSAM and IS-Net, MIT for the
+whisper conversions, GPL-3.0 for Robust Video Matting. Those terms are met
+by the attributions above and by the licence files each mirrored archive
+carries; a mirrored file is a verbatim copy, never a modification. Nothing
+in the mirror is bundled with the app, and Concat claims no rights over any
+of it.
+
## Effect preview photograph
The effect catalogue thumbnails are rendered from a photograph by
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 5d4fdce5..d6cdf556 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -1249,6 +1249,7 @@ dependencies = [
"memmap2",
"serde",
"serde_json",
+ "sha2",
"ureq 2.12.1",
]
@@ -5651,6 +5652,17 @@ dependencies = [
"digest",
]
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
[[package]]
name = "sherpa-onnx"
version = "1.13.7"
diff --git a/engine/crates/concat-host/Cargo.toml b/engine/crates/concat-host/Cargo.toml
index 527043b1..ace13f13 100644
--- a/engine/crates/concat-host/Cargo.toml
+++ b/engine/crates/concat-host/Cargo.toml
@@ -22,6 +22,9 @@ concat-text.workspace = true
# runs them, and ureq because the host is what fetches them.
concat-vision = { workspace = true, features = ["infer"] }
ureq = "2"
+# A finished download is checked against the digest its table carries
+# before anything is allowed to call it installed; see models.rs.
+sha2 = "0.10"
# The CPU compositor draws the monitor's frame; the wgpu one is behind
# `gpu`, so a headless host pays for no GPU tree.
concat-render.workspace = true
diff --git a/engine/crates/concat-host/src/cutout.rs b/engine/crates/concat-host/src/cutout.rs
index 33866f45..2bbc3499 100644
--- a/engine/crates/concat-host/src/cutout.rs
+++ b/engine/crates/concat-host/src/cutout.rs
@@ -369,14 +369,17 @@ impl Cutouts {
/// Streams a model into `file`, by way of a `.part` beside it, reporting
/// every couple of megabytes and stopping when `cancel` is set.
+///
+/// Concat's own mirror first, the upstream it was filled from second: see
+/// [`crate::models`]. Whichever answers, the bytes are checked against the
+/// digest the table carries before anything is renamed into place, so a
+/// truncated or substituted file never becomes an installed model.
pub(crate) fn fetch(
id: ModelId,
file: &Path,
cancel: &AtomicBool,
progress: &mut dyn FnMut(Progress),
) -> Result<(), String> {
- use std::io::{Read, Write};
-
let spec = id.spec();
if let Some(parent) = file.parent() {
std::fs::create_dir_all(parent)
@@ -388,6 +391,42 @@ pub(crate) fn fetch(
total: spec.bytes,
});
+ let mut last: String = String::new();
+ for url in crate::models::sources(spec.file, spec.upstream) {
+ match stream(&url, &partial, spec.bytes, cancel, progress) {
+ Ok(()) => {
+ if let Err(error) = crate::models::verify(&partial, spec.sha256) {
+ let _ = std::fs::remove_file(&partial);
+ // A mirror that serves the wrong bytes is not something
+ // upstream can fix, and trying it next would only hide
+ // which of the two is wrong.
+ return Err(error);
+ }
+ return std::fs::rename(&partial, file)
+ .map_err(|error| format!("could not finish {}: {error}", file.display()));
+ }
+ Err(error) => {
+ let _ = std::fs::remove_file(&partial);
+ if cancel.load(Ordering::Relaxed) {
+ return Err(error);
+ }
+ last = error;
+ }
+ }
+ }
+ Err(format!("could not fetch the {} model: {last}", spec.file))
+}
+
+/// One attempt at one URL: straight into `partial`, reporting as it goes.
+fn stream(
+ url: &str,
+ partial: &Path,
+ estimate: u64,
+ cancel: &AtomicBool,
+ progress: &mut dyn FnMut(Progress),
+) -> Result<(), String> {
+ use std::io::{Read, Write};
+
// A stalled connection blocks in `read` with the cancel flag
// unreachable; thirty seconds without a byte means it is dead.
let agent = ureq::AgentBuilder::new()
@@ -395,54 +434,44 @@ pub(crate) fn fetch(
.timeout_read(std::time::Duration::from_secs(30))
.build();
let response = agent
- .get(spec.url)
+ .get(url)
.call()
- .map_err(|error| format!("could not fetch the {} model: {error}", spec.file))?;
+ .map_err(|error| format!("{url} did not answer: {error}"))?;
let total = response
.header("Content-Length")
.and_then(|value| value.parse::<u64>().ok())
- .unwrap_or(spec.bytes);
+ .unwrap_or(estimate);
- let result = (|| {
- let mut out = std::fs::File::create(&partial)
- .map_err(|error| format!("could not create {}: {error}", partial.display()))?;
- let mut reader = response.into_reader();
- let mut buffer = [0u8; 64 * 1024];
- let mut received = 0u64;
- let mut reported = 0u64;
- loop {
- if cancel.load(Ordering::Relaxed) {
- return Err("cutout analysis cancelled".to_owned());
- }
- let read = reader
- .read(&mut buffer)
- .map_err(|error| format!("model download interrupted: {error}"))?;
- if read == 0 {
- break;
- }
- out.write_all(&buffer[..read])
- .map_err(|error| format!("could not write the model: {error}"))?;
- received += read as u64;
- if received - reported >= 2 * 1024 * 1024 {
- reported = received;
- progress(Progress::Fetching { received, total });
- }
+ let mut out = std::fs::File::create(partial)
+ .map_err(|error| format!("could not create {}: {error}", partial.display()))?;
+ let mut reader = response.into_reader();
+ let mut buffer = [0u8; 64 * 1024];
+ let mut received = 0u64;
+ let mut reported = 0u64;
+ loop {
+ if cancel.load(Ordering::Relaxed) {
+ return Err("cutout analysis cancelled".to_owned());
}
- out.flush()
- .map_err(|error| format!("could not write the model: {error}"))?;
- if received == 0 {
- return Err(format!("the {} model came back empty", spec.file));
+ let read = reader
+ .read(&mut buffer)
+ .map_err(|error| format!("model download interrupted: {error}"))?;
+ if read == 0 {
+ break;
}
- Ok(())
- })();
- match result {
- Ok(()) => std::fs::rename(&partial, file)
- .map_err(|error| format!("could not finish {}: {error}", file.display())),
- Err(error) => {
- let _ = std::fs::remove_file(&partial);
- Err(error)
+ out.write_all(&buffer[..read])
+ .map_err(|error| format!("could not write the model: {error}"))?;
+ received += read as u64;
+ if received - reported >= 2 * 1024 * 1024 {
+ reported = received;
+ progress(Progress::Fetching { received, total });
}
}
+ out.flush()
+ .map_err(|error| format!("could not write the model: {error}"))?;
+ if received == 0 {
+ return Err(format!("{url} came back empty"));
+ }
+ Ok(())
}
/// Every instant the ranges need that the store lacks, ascending, once.
diff --git a/engine/crates/concat-host/src/lib.rs b/engine/crates/concat-host/src/lib.rs
index 855c131f..19429f9d 100644
--- a/engine/crates/concat-host/src/lib.rs
+++ b/engine/crates/concat-host/src/lib.rs
@@ -24,6 +24,7 @@ pub mod dirs;
pub mod export;
pub mod jobs;
pub mod media;
+pub mod models;
pub mod playback;
pub mod preview;
pub mod projects;
diff --git a/engine/crates/concat-host/src/models.rs b/engine/crates/concat-host/src/models.rs
new file mode 100644
index 00000000..391fb551
--- /dev/null
+++ b/engine/crates/concat-host/src/models.rs
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Where a downloadable model is fetched from, and what proves it arrived
+//! whole.
+//!
+//! Concat carries no weights in its bundle: a cutout network, a Kokoro voice
+//! bank and a whisper model together outweigh the editor many times over,
+//! and most people never ask for all three. Each is fetched the first time a
+//! feature needs one.
+//!
+//! Every model Concat offers is mirrored onto a release of its own
+//! repository, and the mirror is what a download asks for first. Upstream -
+//! the third party the mirror was filled from - is the second try and
+//! nothing more: a model the editor offers should not stop existing because
+//! somebody else moved a path, and the bytes that arrive should be the bytes
+//! that were tested.
+//!
+//! The table of what is mirrored is `models/manifest.toml` at the root of
+//! the repository, and `scripts/models.py --check` is what keeps it and the
+//! engine's own tables saying the same thing. The tables are here rather
+//! than parsed from that file because a model's identity is something the
+//! engine should not be able to start without.
+
+use std::path::Path;
+
+/// The repository the mirror lives on.
+pub const REPO: &str = "jub0t/Concat";
+
+/// The release every model is mirrored on.
+///
+/// Bumped only when a model's bytes change - app releases point at it by
+/// name, so an unchanged model is never re-uploaded. Must match `release`
+/// in `models/manifest.toml`; the check script enforces that.
+pub const RELEASE: &str = "models-v1";
+
+/// Where `file` is mirrored.
+pub fn mirror(file: &str) -> String {
+ format!("https://github.com/{REPO}/releases/download/{RELEASE}/{file}")
+}
+
+/// Where to try for `file`, in order: our mirror, then the upstream it was
+/// filled from.
+///
+/// Two and not one because a mirror that cannot be reached - a proxy that
+/// blocks our host, a release still being published, a region that cannot
+/// see it - should cost a retry rather than a feature.
+pub fn sources(file: &str, upstream: &str) -> [String; 2] {
+ [mirror(file), upstream.to_owned()]
+}
+
+/// Checks a finished download against the digest the table carries.
+///
+/// An empty `expected` passes: a model added to the tables is unverified
+/// until the mirror has been filled once and has reported what it holds.
+/// Anything else is compared, and a file that does not match is refused -
+/// the caller deletes it rather than leaving something that looks installed.
+pub fn verify(file: &Path, expected: &str) -> Result<(), String> {
+ if expected.is_empty() {
+ return Ok(());
+ }
+ let actual = sha256(file)?;
+ if actual == expected {
+ return Ok(());
+ }
+ Err(format!(
+ "{} is not the file it should be: expected sha256 {expected}, got {actual}",
+ file.display()
+ ))
+}
+
+/// The SHA-256 of a file, as lowercase hex.
+fn sha256(file: &Path) -> Result<String, String> {
+ use sha2::{Digest, Sha256};
+ use std::io::Read;
+
+ let mut handle = std::fs::File::open(file)
+ .map_err(|error| format!("could not read {}: {error}", file.display()))?;
+ let mut hasher = Sha256::new();
+ let mut buffer = vec![0u8; 1024 * 1024];
+ loop {
+ let read = handle
+ .read(&mut buffer)
+ .map_err(|error| format!("could not read {}: {error}", file.display()))?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
+ Ok(hasher
+ .finalize()
+ .iter()
+ .fold(String::with_capacity(64), |mut out, byte| {
+ use std::fmt::Write;
+ let _ = write!(out, "{byte:02x}");
+ out
+ }))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_mirror_comes_before_upstream() {
+ let [first, second] = sources("model.onnx", "https://elsewhere.example/model.onnx");
+ assert_eq!(
+ first,
+ format!("https://github.com/{REPO}/releases/download/{RELEASE}/model.onnx")
+ );
+ assert_eq!(second, "https://elsewhere.example/model.onnx");
+ }
+
+ #[test]
+ fn a_digest_is_checked_and_an_empty_one_is_not() {
+ let dir = std::env::temp_dir().join("concat-models-verify");
+ std::fs::create_dir_all(&dir).expect("a temp dir");
+ let file = dir.join("bytes");
+ std::fs::write(&file, b"concat").expect("a temp file");
+ // echo -n concat | sha256sum
+ let known = "3f4c1a4b3c9fb1b0e24bd1ec0bd7b16d0db4dbf1fe2f3b5dcdba0bd38b8f8b3a";
+ assert!(verify(&file, "").is_ok());
+ assert!(verify(&file, known).is_err());
+ let actual = sha256(&file).expect("a digest");
+ assert_eq!(actual.len(), 64);
+ assert!(verify(&file, &actual).is_ok());
+ let _ = std::fs::remove_file(&file);
+ }
+}
diff --git a/engine/crates/concat-speech/src/lib.rs b/engine/crates/concat-speech/src/lib.rs
index 032bc43f..c75d01ca 100644
--- a/engine/crates/concat-speech/src/lib.rs
+++ b/engine/crates/concat-speech/src/lib.rs
@@ -39,6 +39,50 @@ pub struct DownloadProgress {
pub done: bool,
}
+/// Streams a model into `partial`: Concat's own mirror first, the upstream
+/// it was filled from second, and the finished file checked against the
+/// digest its table carries before the caller is told it arrived.
+///
+/// `file` is what the model is called on the mirror. Two tries and not one
+/// because a mirror that cannot be reached - a proxy that blocks our host,
+/// a release still being published - should cost a retry rather than a
+/// feature; see [`concat_host::models`]. A file that arrives and hashes
+/// wrong is refused outright rather than retried, since the second try
+/// would only hide which source served it.
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn fetch_model(
+ file: &str,
+ upstream: &str,
+ partial: &std::path::Path,
+ id: &str,
+ estimate: u64,
+ sha256: &str,
+ cancel: &std::sync::atomic::AtomicBool,
+ progress: &mut dyn FnMut(DownloadProgress),
+) -> Result<(u64, u64), String> {
+ use std::sync::atomic::Ordering;
+
+ let mut last = String::new();
+ for url in concat_host::models::sources(file, upstream) {
+ match download_to(&url, partial, id, estimate, cancel, progress) {
+ Ok(totals) => {
+ concat_host::models::verify(partial, sha256).inspect_err(|_| {
+ let _ = std::fs::remove_file(partial);
+ })?;
+ return Ok(totals);
+ }
+ Err(error) => {
+ let _ = std::fs::remove_file(partial);
+ if cancel.load(Ordering::Relaxed) {
+ return Err(error);
+ }
+ last = error;
+ }
+ }
+ }
+ Err(last)
+}
+
/// Streams `url` into `partial`, reporting every couple of megabytes and
/// stopping when `cancel` is set. Shared by both model downloaders.
fn download_to(
diff --git a/engine/crates/concat-speech/src/transcribe.rs b/engine/crates/concat-speech/src/transcribe.rs
index 6ca1e758..f488e8e4 100644
--- a/engine/crates/concat-speech/src/transcribe.rs
+++ b/engine/crates/concat-speech/src/transcribe.rs
@@ -38,6 +38,9 @@ struct KnownModel {
/// One line for the settings row: what this size trades away.
blurb: &'static str,
approx_bytes: u64,
+ /// What the finished file must hash to. Empty until the mirror has been
+ /// filled once and reported what it holds; see [`concat_host::models`].
+ sha256: &'static str,
english_only: bool,
}
@@ -52,6 +55,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Tiny (English)",
blurb: "Fastest draft. Rough on names and punctuation.",
approx_bytes: 77_700_000,
+ sha256: "",
english_only: true,
},
KnownModel {
@@ -59,6 +63,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Tiny (Multilingual)",
blurb: "Fastest draft, any language.",
approx_bytes: 77_700_000,
+ sha256: "",
english_only: false,
},
KnownModel {
@@ -66,6 +71,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Base (English)",
blurb: "The sweet spot: solid captions at ~10x realtime.",
approx_bytes: 147_400_000,
+ sha256: "",
english_only: true,
},
KnownModel {
@@ -73,6 +79,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Base (Multilingual)",
blurb: "Solid captions, any language.",
approx_bytes: 147_500_000,
+ sha256: "",
english_only: false,
},
KnownModel {
@@ -80,6 +87,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Small (English)",
blurb: "Noticeably better wording; a few times slower.",
approx_bytes: 487_600_000,
+ sha256: "",
english_only: true,
},
KnownModel {
@@ -87,6 +95,7 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Small (Multilingual)",
blurb: "Best quality offered, any language.",
approx_bytes: 487_600_000,
+ sha256: "",
english_only: false,
},
];
@@ -95,7 +104,13 @@ fn known(id: &str) -> Option<&'static KnownModel> {
KNOWN_MODELS.iter().find(|model| model.id == id)
}
-fn model_url(id: &str) -> String {
+/// The file a model arrives as, and what it is called on the mirror.
+fn model_archive(id: &str) -> String {
+ format!("ggml-{id}.bin")
+}
+
+/// Where the mirror was filled from, and the second place a download tries.
+fn model_upstream(id: &str) -> String {
format!("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{id}.bin")
}
@@ -221,7 +236,7 @@ impl Transcriber {
})
}
- /// Streams one model from Hugging Face into the models folder. Blocks
+ /// Streams one model from Concat's mirror into the models folder. Blocks
/// for the whole download: run it on its own thread.
///
/// Written to `<file>.part` and renamed at the end: a download killed
@@ -240,11 +255,13 @@ impl Transcriber {
}
let estimate = known(id).map(|model| model.approx_bytes).unwrap_or(0);
let partial = destination.with_extension("bin.part");
- let (received, total) = crate::download_to(
- &model_url(id),
+ let (received, total) = crate::fetch_model(
+ &model_archive(id),
+ &model_upstream(id),
&partial,
id,
estimate,
+ known(id).map(|model| model.sha256).unwrap_or(""),
job.cancel_flag(),
&mut progress,
)?;
@@ -420,9 +437,15 @@ mod tests {
}
#[test]
- fn every_known_model_has_a_url_and_file_name() {
+ fn every_known_model_is_asked_of_the_mirror_before_upstream() {
for model in KNOWN_MODELS {
- assert!(model_url(model.id).ends_with(&format!("ggml-{}.bin", model.id)));
+ let file = model_archive(model.id);
+ assert_eq!(file, format!("ggml-{}.bin", model.id));
+ assert!(model_upstream(model.id).ends_with(&file));
+ let [mirror, upstream] = concat_host::models::sources(&file, &model_upstream(model.id));
+ assert!(mirror.contains(concat_host::models::RELEASE));
+ assert!(mirror.ends_with(&file));
+ assert_eq!(upstream, model_upstream(model.id));
assert!(model.approx_bytes > 0);
}
}
diff --git a/engine/crates/concat-speech/src/tts.rs b/engine/crates/concat-speech/src/tts.rs
index 208eb541..b41cac75 100644
--- a/engine/crates/concat-speech/src/tts.rs
+++ b/engine/crates/concat-speech/src/tts.rs
@@ -48,6 +48,10 @@ struct KnownModel {
/// One line for the settings row: what this size trades away.
blurb: &'static str,
approx_bytes: u64,
+ /// What the finished archive must hash to. Empty until the mirror has
+ /// been filled once and reported what it holds; see
+ /// [`concat_host::models`].
+ sha256: &'static str,
}
/// The models the settings panel offers, smallest first.
@@ -62,12 +66,14 @@ const KNOWN_MODELS: &[KnownModel] = &[
label: "Kokoro (compact)",
blurb: "The recommended build: same voices, a third of the download.",
approx_bytes: 131_839_838,
+ sha256: "",
},
KnownModel {
id: "kokoro-multi-lang-v1_0",
label: "Kokoro (full precision)",
blurb: "Bit-perfect weights for the skeptical; rarely audibly better.",
approx_bytes: 349_418_188,
+ sha256: "",
},
];
@@ -121,7 +127,13 @@ fn known(id: &str) -> Option<&'static KnownModel> {
KNOWN_MODELS.iter().find(|model| model.id == id)
}
-fn model_url(id: &str) -> String {
+/// The archive a bundle arrives as, and what it is called on the mirror.
+fn model_archive(id: &str) -> String {
+ format!("{id}.tar.bz2")
+}
+
+/// Where the mirror was filled from, and the second place a download tries.
+fn model_upstream(id: &str) -> String {
format!("https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/{id}.tar.bz2")
}
@@ -280,7 +292,7 @@ impl Speech {
})
}
- /// Streams one bundle from the sherpa-onnx releases and unpacks it.
+ /// Streams one Kokoro bundle from Concat's mirror and unpacks it.
/// Blocks for the whole download: run it on its own thread.
///
/// The archive lands in a `.part`, unpacks into a `.staging-<id>` folder,
@@ -304,11 +316,13 @@ impl Speech {
let archive = parent.join(format!("{id}.tar.bz2.part"));
let staging = parent.join(format!(".staging-{id}"));
let result = (|| {
- let (received, total) = crate::download_to(
- &model_url(id),
+ let (received, total) = crate::fetch_model(
+ &model_archive(id),
+ &model_upstream(id),
&archive,
id,
estimate,
+ known(id).map(|model| model.sha256).unwrap_or(""),
cancel,
&mut progress,
)?;
@@ -572,9 +586,15 @@ mod tests {
use super::*;
#[test]
- fn every_known_model_has_a_url_and_size() {
+ fn every_known_model_is_asked_of_the_mirror_before_upstream() {
for model in KNOWN_MODELS {
- assert!(model_url(model.id).ends_with(&format!("{}.tar.bz2", model.id)));
+ let file = model_archive(model.id);
+ assert_eq!(file, format!("{}.tar.bz2", model.id));
+ assert!(model_upstream(model.id).ends_with(&file));
+ let [mirror, upstream] = concat_host::models::sources(&file, &model_upstream(model.id));
+ assert!(mirror.contains(concat_host::models::RELEASE));
+ assert!(mirror.ends_with(&file));
+ assert_eq!(upstream, model_upstream(model.id));
assert!(model.approx_bytes > 0);
}
}
diff --git a/engine/crates/concat-vision/src/models.rs b/engine/crates/concat-vision/src/models.rs
index 1f9bb98b..eabf8e60 100644
--- a/engine/crates/concat-vision/src/models.rs
+++ b/engine/crates/concat-vision/src/models.rs
@@ -10,6 +10,12 @@
//! killed download leaves nothing that could be mistaken for one. The
//! fetching itself is the host's, since it is the host that has a network
//! and a job slot; this crate only knows the table.
+//!
+//! A model is fetched from Concat's own mirror, and `upstream` is where
+//! that mirror was filled from and the second thing tried. The table is
+//! the engine's half of `models/manifest.toml`, which is what fills the
+//! mirror; `scripts/models.py --check` keeps the two saying the same
+//! thing.
use std::path::{Path, PathBuf};
@@ -35,12 +41,17 @@ pub enum ModelId {
pub struct ModelSpec {
/// Which model.
pub id: ModelId,
- /// The file name on disk, and what a mask store records.
+ /// The file name on disk, what a mask store records, and what the
+ /// file is called on the mirror.
pub file: &'static str,
- /// Where it is fetched from.
- pub url: &'static str,
+ /// Where the mirror was filled from, and the second place a download
+ /// tries; the host's `models` module is what puts the two in order.
+ pub upstream: &'static str,
/// Its size, for a progress bar before the server says.
pub bytes: u64,
+ /// What a finished download must hash to. Empty until the mirror has
+ /// been filled once and reported what it holds.
+ pub sha256: &'static str,
/// The licence it comes under, for the notices.
pub licence: &'static str,
}
@@ -50,29 +61,33 @@ pub const MODELS: [ModelSpec; 4] = [
ModelSpec {
id: ModelId::Person,
file: "rvm-mobilenetv3.onnx",
- url: "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx",
+ upstream: "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx",
bytes: 14_975_696,
+ sha256: "88d4531297118f595bf2fd60f6f566aec2e559393802d1f436c380f0cbbd2828",
licence: "GPL-3.0",
},
ModelSpec {
id: ModelId::Object,
file: "isnet-general-use.onnx",
- url: "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx",
+ upstream: "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx",
bytes: 178_648_008,
+ sha256: "",
licence: "Apache-2.0",
},
ModelSpec {
id: ModelId::BrushEncoder,
file: "slimsam-77-encoder.onnx",
- url: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx",
+ upstream: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx",
bytes: 23_276_014,
+ sha256: "",
licence: "Apache-2.0",
},
ModelSpec {
id: ModelId::BrushDecoder,
file: "slimsam-77-decoder.onnx",
- url: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx",
+ upstream: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx",
bytes: 16_557_892,
+ sha256: "",
licence: "Apache-2.0",
},
];
diff --git a/models/manifest.toml b/models/manifest.toml
new file mode 100644
index 00000000..48aa7875
--- /dev/null
+++ b/models/manifest.toml
@@ -0,0 +1,151 @@
+# The models Concat downloads, and where they come from.
+#
+# Concat ships no weights inside its bundles - a cutout network, a Kokoro
+# voice bank and a whisper model together outweigh the editor many times
+# over, and most people never ask for all three. They are fetched the first
+# time a feature needs one.
+#
+# What they are fetched *from* is this file. Every model is mirrored onto a
+# release of this repository, under the `release` tag below, and that mirror
+# is what the app asks for; `upstream` is where the mirror was filled from
+# and what the app falls back to if our own release cannot be reached. The
+# point of the mirror is that a model Concat offers stays reachable, stays
+# the bytes we tested, and stays under one roof, rather than depending on a
+# third party's bucket keeping a path alive.
+#
+# scripts/models.py --check # this table against the engine's
+# scripts/models.py mirror --out dir # fill a mirror from upstream
+# scripts/models.py release-manifest # the manifest.json a release carries
+#
+# Refreshing the mirror is .github/workflows/models.yml: it fetches every
+# `upstream`, checks each against `sha256`, records the digest when this
+# table has none yet, and uploads the lot to the `release` tag. Bump that
+# tag only when a model's bytes change - app releases point at it by name,
+# so an unchanged model is never re-uploaded.
+#
+# `sha256` is what the app verifies a finished download against. An empty
+# digest means the mirror has not been filled since the model was added;
+# the workflow prints what it computed, and that goes here and into the
+# engine's table beside it.
+
+# The release every model is mirrored on.
+release = "models-v1"
+
+# ── Cutout ───────────────────────────────────────────────────────────────
+# engine/crates/concat-vision/src/models.rs
+
+[[model]]
+id = "cutout-person"
+family = "cutout"
+file = "rvm-mobilenetv3.onnx"
+bytes = 14975696
+sha256 = "88d4531297118f595bf2fd60f6f566aec2e559393802d1f436c380f0cbbd2828"
+licence = "GPL-3.0"
+upstream = "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx"
+
+[[model]]
+id = "cutout-object"
+family = "cutout"
+file = "isnet-general-use.onnx"
+bytes = 178648008
+sha256 = ""
+licence = "Apache-2.0"
+upstream = "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx"
+
+[[model]]
+id = "cutout-brush-encoder"
+family = "cutout"
+file = "slimsam-77-encoder.onnx"
+bytes = 23276014
+sha256 = ""
+licence = "Apache-2.0"
+upstream = "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx"
+
+[[model]]
+id = "cutout-brush-decoder"
+family = "cutout"
+file = "slimsam-77-decoder.onnx"
+bytes = 16557892
+sha256 = ""
+licence = "Apache-2.0"
+upstream = "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx"
+
+# ── Text to speech ───────────────────────────────────────────────────────
+# engine/crates/concat-speech/src/tts.rs. Kokoro v1.0 multi-language, the
+# official sherpa-onnx conversions; the two differ only in quantisation.
+
+[[model]]
+id = "kokoro-int8-multi-lang-v1_0"
+family = "tts"
+file = "kokoro-int8-multi-lang-v1_0.tar.bz2"
+bytes = 131839838
+sha256 = ""
+licence = "Apache-2.0"
+upstream = "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-int8-multi-lang-v1_0.tar.bz2"
+
+[[model]]
+id = "kokoro-multi-lang-v1_0"
+family = "tts"
+file = "kokoro-multi-lang-v1_0.tar.bz2"
+bytes = 349418188
+sha256 = ""
+licence = "Apache-2.0"
+upstream = "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2"
+
+# ── Transcription ────────────────────────────────────────────────────────
+# engine/crates/concat-speech/src/transcribe.rs. whisper.cpp's GGML
+# conversions of OpenAI's whisper.
+
+[[model]]
+id = "whisper-tiny.en"
+family = "whisper"
+file = "ggml-tiny.en.bin"
+bytes = 77700000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin"
+
+[[model]]
+id = "whisper-tiny"
+family = "whisper"
+file = "ggml-tiny.bin"
+bytes = 77700000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin"
+
+[[model]]
+id = "whisper-base.en"
+family = "whisper"
+file = "ggml-base.en.bin"
+bytes = 147400000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"
+
+[[model]]
+id = "whisper-base"
+family = "whisper"
+file = "ggml-base.bin"
+bytes = 147500000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin"
+
+[[model]]
+id = "whisper-small.en"
+family = "whisper"
+file = "ggml-small.en.bin"
+bytes = 487600000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin"
+
+[[model]]
+id = "whisper-small"
+family = "whisper"
+file = "ggml-small.bin"
+bytes = 487600000
+sha256 = ""
+licence = "MIT"
+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"
diff --git a/scripts/models.py b/scripts/models.py
new file mode 100755
index 00000000..07e89341
--- /dev/null
+++ b/scripts/models.py
@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""The model mirror: the table, the fill, and the manifest a release carries.
+
+ scripts/models.py --check # the table against the engine
+ scripts/models.py mirror --out dir # fill a mirror from upstream
+ scripts/models.py release-manifest \\
+ --version 0.2.1 --tag v0.2.1 \\
+ --bundles dir --out manifest.json # what a release describes
+
+Concat fetches its cutout networks, its Kokoro voice bank and its whisper
+models on demand rather than carrying them in the bundle. models/manifest.toml
+says what they are and where they come from; this script is what fills the
+mirror from upstream, and what turns the table into the manifest.json every
+release ships beside its bundles. See that file for the shape of a row.
+"""
+import argparse
+import hashlib
+import json
+import pathlib
+import re
+import sys
+import tomllib
+import urllib.request
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+MANIFEST = ROOT / "models" / "manifest.toml"
+CRATES = ROOT / "engine" / "crates"
+
+# The repository the mirror lives on, and the engine constant that must
+# agree with this file about which release holds it.
+REPO = "jub0t/Concat"
+HOST_MODELS = CRATES / "concat-host" / "src" / "models.rs"
+
+# Where each family's table lives in the engine, for --check.
+TABLES = {
+ "cutout": CRATES / "concat-vision" / "src" / "models.rs",
+ "tts": CRATES / "concat-speech" / "src" / "tts.rs",
+ "whisper": CRATES / "concat-speech" / "src" / "transcribe.rs",
+}
+
+# The eight bundles a release publishes, as build-app.yml and mobile.yml
+# name them. Stem is formatted with the version.
+BUNDLES = [
+ ("macos", "arm64", "Concat-{v}-macos-arm64.dmg"),
+ ("macos", "x86_64", "Concat-{v}-macos-x86_64.dmg"),
+ ("linux", "x86_64", "Concat-{v}-linux-x86_64.tar.gz"),
+ ("linux", "aarch64", "Concat-{v}-linux-aarch64.tar.gz"),
+ ("windows", "x86_64", "Concat-{v}-windows-x86_64.zip"),
+ ("windows", "aarch64", "Concat-{v}-windows-aarch64.zip"),
+ ("android", "arm64", "Concat-{v}-android-arm64.apk"),
+ ("ios", "arm64", "Concat-{v}-ios-arm64.ipa"),
+]
+
+
+def table() -> dict:
+ return tomllib.loads(MANIFEST.read_text(encoding="utf-8"))
+
+
+def asset_url(release: str, file: str) -> str:
+ return f"https://github.com/{REPO}/releases/download/{release}/{file}"
+
+
+def engine_id(model: dict) -> str:
+ """What the engine calls this model.
+
+ The engine's own tables are keyed the way each family's upstream names
+ things - a cutout model by its file, a Kokoro bundle and a whisper model
+ by the id their archive name is built from - so the id is read back out
+ of the file rather than stored twice.
+ """
+ file = model["file"]
+ if model["family"] == "tts":
+ return file.removesuffix(".tar.bz2")
+ if model["family"] == "whisper":
+ return file.removeprefix("ggml-").removesuffix(".bin")
+ return file
+
+
+def digest(path: pathlib.Path) -> str:
+ sha = hashlib.sha256()
+ with path.open("rb") as handle:
+ for block in iter(lambda: handle.read(1024 * 1024), b""):
+ sha.update(block)
+ return sha.hexdigest()
+
+
+# ── check ────────────────────────────────────────────────────────────────
+
+
+def check() -> int:
+ data = table()
+ models = data.get("model", [])
+ release = data.get("release", "")
+ failed = 0
+
+ def bad(message: str) -> None:
+ nonlocal failed
+ print(f" {message}")
+ failed = 1
+
+ if not release:
+ bad("the table names no release")
+ if not models:
+ bad("the table has no models")
+
+ for field in ("id", "family", "file", "bytes", "sha256", "licence", "upstream"):
+ for model in models:
+ if field not in model:
+ bad(f"{model.get('id', '?')!r} has no {field}")
+
+ for key in ("id", "file"):
+ seen: dict[str, int] = {}
+ for model in models:
+ seen[model.get(key, "")] = seen.get(model.get(key, ""), 0) + 1
+ for value, count in seen.items():
+ if count > 1:
+ bad(f"{value!r} is the {key} of {count} models; a mirror is one flat release")
+
+ for model in models:
+ if model.get("family") not in TABLES:
+ bad(f"{model['id']!r} is in family {model.get('family')!r}, which no engine table owns")
+ if not str(model.get("upstream", "")).startswith("https://"):
+ bad(f"{model['id']!r} has no https upstream")
+ if model.get("bytes", 0) <= 0:
+ bad(f"{model['id']!r} has no size")
+ sha = model.get("sha256", "")
+ if sha and not re.fullmatch(r"[0-9a-f]{64}", sha):
+ bad(f"{model['id']!r} has a sha256 that is not 64 hex digits")
+
+ # The engine constant and this file must name the same release, or the
+ # app asks for a mirror that is not the one being filled.
+ source = HOST_MODELS.read_text(encoding="utf-8") if HOST_MODELS.exists() else ""
+ if f'pub const RELEASE: &str = "{release}";' not in source:
+ bad(f"{HOST_MODELS.relative_to(ROOT)} does not name the release {release!r}")
+ if f'"{REPO}"' not in source:
+ bad(f"{HOST_MODELS.relative_to(ROOT)} does not name the repository {REPO!r}")
+
+ # Every model this table carries must be one the engine offers, with the
+ # same digest: the table is what the mirror is filled from, the engine's
+ # constant is what a download is checked against, and a download checked
+ # against a digest nothing was mirrored under can never pass.
+ for family, path in TABLES.items():
+ rows = [model for model in models if model.get("family") == family]
+ text = path.read_text(encoding="utf-8").split("#[cfg(test)]")[0]
+ ids = set(re.findall(r'^\s*(?:pub )?id: "([^"]+)"', text, re.M))
+ if family == "cutout":
+ ids = set(re.findall(r'^\s*file: "([^"]+)"', text, re.M))
+ wanted = {engine_id(model) for model in rows}
+ for missing in sorted(wanted - ids):
+ bad(f"{path.relative_to(ROOT)} has no model {missing!r}")
+ for extra in sorted(ids - wanted):
+ bad(f"{extra!r} is in {path.relative_to(ROOT)} and not in the table")
+ for model in rows:
+ sha = model.get("sha256", "")
+ if sha and f'sha256: "{sha}"' not in text:
+ bad(f"{path.relative_to(ROOT)} does not carry the digest of {model['id']!r}")
+
+ empty = [model["id"] for model in models if not model.get("sha256")]
+ total = sum(model.get("bytes", 0) for model in models)
+ print(f"{len(models)} models, {total / 1e9:.2f} GB, mirrored on {release}")
+ if empty:
+ # Not a failure: a model added here is unverified until the mirror
+ # workflow has run once and reported what it fetched.
+ print(f" {len(empty)} awaiting a digest from the mirror: {', '.join(empty)}")
+ return failed
+
+
+# ── mirror ───────────────────────────────────────────────────────────────
+
+
+def fill(path: pathlib.Path, block: str, sha: str, what: str) -> bool:
+ """Fills one empty `sha256` inside one fenced block of `path`."""
+ text = path.read_text(encoding="utf-8")
+ updated, count = re.subn(block, lambda match: match.group(1) + sha + '"', text, count=1)
+ if not count:
+ print(f" could not record the digest in {what}", file=sys.stderr)
+ return False
+ path.write_text(updated, encoding="utf-8")
+ return True
+
+
+def record(model: dict, sha: str) -> None:
+ """Writes a digest into the table and into the engine's table beside it.
+
+ Both, because the table is what the mirror is filled from and the
+ engine's constant is what a download is checked against: a digest in
+ only one of them is a download that can never pass. Each match is fenced
+ to one model's own block, so the empty digest that gets filled is never
+ the next one down.
+ """
+ model_id = model["id"]
+ fill(
+ MANIFEST,
+ r'(\[\[model\]\]\nid = "'
+ + re.escape(model_id)
+ + r'"(?:(?!\[\[model\]\])[\s\S])*?sha256 = ")"',
+ sha,
+ str(MANIFEST.relative_to(ROOT)),
+ )
+ path = TABLES[model["family"]]
+ item, key = ("ModelSpec", "file") if model["family"] == "cutout" else ("KnownModel", "id")
+ fill(
+ path,
+ r"("
+ + item
+ + r' \{\n(?:(?!\n \},)[\s\S])*?'
+ + key
+ + r': "'
+ + re.escape(engine_id(model))
+ + r'"(?:(?!\n \},)[\s\S])*?sha256: ")"',
+ sha,
+ str(path.relative_to(ROOT)),
+ )
+
+
+def stream(url: str, into: pathlib.Path) -> tuple[int, str]:
+ sha = hashlib.sha256()
+ received = 0
+ request = urllib.request.Request(url, headers={"User-Agent": f"concat-models ({REPO})"})
+ with urllib.request.urlopen(request) as response, into.open("wb") as handle:
+ while True:
+ block = response.read(1024 * 1024)
+ if not block:
+ break
+ handle.write(block)
+ sha.update(block)
+ received += len(block)
+ return received, sha.hexdigest()
+
+
+def mirror(out: pathlib.Path, only: list[str]) -> int:
+ data = table()
+ models = [m for m in data["model"] if not only or m["family"] in only or m["id"] in only]
+ out.mkdir(parents=True, exist_ok=True)
+ failed = 0
+ for model in models:
+ target = out / model["file"]
+ print(f"{model['id']}: {model['upstream']}")
+ partial = target.with_suffix(target.suffix + ".part")
+ try:
+ received, sha = stream(model["upstream"], partial)
+ except Exception as error: # noqa: BLE001 - the report is the point
+ partial.unlink(missing_ok=True)
+ print(f" could not fetch: {error}", file=sys.stderr)
+ failed = 1
+ continue
+ expected = model.get("sha256", "")
+ if expected and expected != sha:
+ partial.unlink(missing_ok=True)
+ print(f" expected sha256 {expected}, got {sha}", file=sys.stderr)
+ failed = 1
+ continue
+ partial.replace(target)
+ print(f" {received} bytes, sha256 {sha}")
+ if not expected:
+ record(model, sha)
+ print(f" recorded in {MANIFEST.relative_to(ROOT)}")
+ if received != model["bytes"]:
+ # Not fatal - the table's size is for a progress bar before the
+ # server says - but a large drift means the row is stale.
+ print(f" note: table says {model['bytes']} bytes, upstream sent {received}")
+ return failed
+
+
+# ── release manifest ─────────────────────────────────────────────────────
+
+
+def release_manifest(version: str, tag: str, bundles: pathlib.Path | None) -> dict:
+ data = table()
+ release = data["release"]
+
+ binaries: dict[str, dict] = {}
+ for platform, arch, stem in BUNDLES:
+ file = stem.format(v=version)
+ entry = {
+ "file": file,
+ "url": asset_url(tag, file),
+ }
+ if bundles is not None:
+ path = bundles / file
+ if path.exists():
+ entry["bytes"] = path.stat().st_size
+ entry["sha256"] = digest(path)
+ else:
+ # A target that did not build is absent rather than a row
+ # promising a file that is not there.
+ continue
+ binaries.setdefault(platform, {})[arch] = entry
+
+ models = {}
+ for model in data["model"]:
+ models[model["id"]] = {
+ "family": model["family"],
+ "file": model["file"],
+ "bytes": model["bytes"],
+ "sha256": model["sha256"],
+ "licence": model["licence"],
+ "url": asset_url(release, model["file"]),
+ "upstream": model["upstream"],
+ }
+
+ return {
+ "schema": 1,
+ "product": "Concat",
+ "version": version,
+ "tag": tag,
+ "models_release": release,
+ "binaries": binaries,
+ "models": models,
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--check", action="store_true", help="the table against the engine")
+ sub = parser.add_subparsers(dest="command")
+
+ fill = sub.add_parser("mirror", help="fill a mirror from upstream")
+ fill.add_argument("--out", type=pathlib.Path, required=True)
+ fill.add_argument("--only", action="append", default=[], help="a family or an id; repeatable")
+
+ emit = sub.add_parser("release-manifest", help="the manifest.json a release carries")
+ emit.add_argument("--version", required=True)
+ emit.add_argument("--tag", required=True)
+ emit.add_argument("--bundles", type=pathlib.Path)
+ emit.add_argument("--out", type=pathlib.Path)
+
+ args = parser.parse_args()
+ if args.command == "mirror":
+ return mirror(args.out, args.only)
+ if args.command == "release-manifest":
+ manifest = release_manifest(args.version, args.tag, args.bundles)
+ text = json.dumps(manifest, indent=2) + "\n"
+ if args.out:
+ args.out.write_text(text, encoding="utf-8")
+ print(f"{args.out}: {len(manifest['binaries'])} platforms, {len(manifest['models'])} models")
+ else:
+ print(text, end="")
+ return 0
+ return check()
+
+
+if __name__ == "__main__":
+ sys.exit(main())