The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(models): mirror every downloadable model on our own releases
Jareer93274efe395e
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 29963705..2499eddb 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -55,6 +55,11 @@ jobs:# keeps en.json the inventory of them and every shipped language# keyed off it. See TRANSLATING.md.- run: python3 scripts/locales.py --check+ # The models Concat downloads are mirrored onto a release of this+ # repository; models/manifest.toml is what fills that mirror and the+ # engine's tables are what a download is checked against. This keeps+ # the two saying the same thing. See models/manifest.toml.+ - run: python3 scripts/models.py --check# -D warnings covers rustc lints too (missing_docs among them), so a# public item without documentation fails here, not in review. Lints# before tests: they finish in a fraction of the time and catch thediff --git a/.github/workflows/models.yml b/.github/workflows/models.ymlnew file mode 100644index 00000000..c30e8da5--- /dev/null+++ b/.github/workflows/models.yml@@ -0,0 +1,122 @@+name: Mirror models++# Fills Concat's model mirror from upstream and publishes it as a release.+#+# Concat fetches its cutout networks, its Kokoro voice bank and its whisper+# models on demand, and it fetches them from a release of this repository+# rather than from whoever published them. models/manifest.toml is the table+# of what those are; this workflow is what turns that table into the release+# the app asks for.+#+# Run it when a model is added to the table, or when one's bytes change and+# its release tag is bumped. It is not part of cutting an app release: the+# mirror has its own tag, app releases point at it by name, and an unchanged+# model is never re-uploaded. Roughly 2 GB down and 2 GB up, so give it time.+#+# A model whose row has no digest yet is recorded: the job fetches it,+# computes the sha256, writes it into models/manifest.toml and into the+# engine's table beside it, and uploads the patch as an artifact for a+# maintainer to commit. Until that lands, the app accepts the download+# unverified, which is what the empty digest means.+on:+ workflow_dispatch:+ inputs:+ only:+ description: A family (cutout, tts, whisper) or one model id; blank for all+ required: false+ default: ""+ publish:+ description: Upload to the release tag, rather than only fetching and checking+ type: boolean+ required: false+ default: true++concurrency:+ group: models+ cancel-in-progress: false++jobs:+ mirror:+ name: Fill and publish+ runs-on: ubuntu-latest+ permissions:+ contents: write+ steps:+ - uses: actions/checkout@v7++ - name: The table+ id: table+ run: |+ set -euo pipefail+ release=$(sed -n 's/^release = "\(.*\)"/\1/p' models/manifest.toml | head -1)+ test -n "$release"+ echo "release=$release" >> "$GITHUB_OUTPUT"+ python3 scripts/models.py --check++ # The runner has room for the lot with a few gigabytes to spare, but+ # not much more; the mirror folder is deleted with the runner.+ - name: Disk before+ run: df -h /++ - name: Fetch from upstream+ env:+ ONLY: ${{ inputs.only }}+ run: |+ set -euo pipefail+ args=()+ if [ -n "$ONLY" ]; then args+=(--only "$ONLY"); fi+ python3 scripts/models.py mirror --out mirror "${args[@]}"+ ls -l mirror++ # What the fetch recorded, if anything: the digests are now in the+ # table and in the engine, and that diff is what wants committing.+ - name: Digests recorded+ id: recorded+ run: |+ set -euo pipefail+ if git diff --quiet; then+ echo "Nothing recorded: every model already carried its digest." >> "$GITHUB_STEP_SUMMARY"+ echo "patch=false" >> "$GITHUB_OUTPUT"+ else+ {+ echo "### Digests recorded"+ echo+ echo 'Commit this patch, or the app keeps accepting these downloads unverified.'+ echo+ echo '```diff'+ git diff+ echo '```'+ } >> "$GITHUB_STEP_SUMMARY"+ git diff > recorded-digests.patch+ echo "patch=true" >> "$GITHUB_OUTPUT"+ fi++ - uses: actions/upload-artifact@v7+ if: steps.recorded.outputs.patch == 'true'+ with:+ name: recorded-digests+ path: recorded-digests.patch+ if-no-files-found: error++ # The mirror is one flat release: every file in the table is uploaded+ # under its own name, and `--clobber` means re-running after a fixed+ # row replaces the file rather than failing on it.+ - name: Publish+ if: inputs.publish+ env:+ GH_TOKEN: ${{ github.token }}+ RELEASE: ${{ steps.table.outputs.release }}+ run: |+ set -euo pipefail+ if ! gh release view "$RELEASE" >/dev/null 2>&1; then+ gh release create "$RELEASE" \+ --title "Concat models ($RELEASE)" \+ --notes "The models Concat downloads on demand, mirrored from the upstreams named in models/manifest.toml. Not an app release: see the tags beginning with v for those." \+ --latest=false+ fi+ gh release upload "$RELEASE" mirror/* --clobber+ gh release view "$RELEASE"++ - name: Disk after+ if: always()+ run: df -h /diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.ymlindex 29bf4973..226bce29 100644--- a/.github/workflows/nightly.yml+++ b/.github/workflows/nightly.yml@@ -49,7 +49,7 @@ jobs:merge-multiple: true# Named by the commit, so two testers can say which nightly they have.- - name: Name the bundles by commit and write checksums+ - name: Name the bundles by commitid: stamprun: |set -euo pipefail@@ -59,14 +59,34 @@ jobs:for file in Concat-*; domv "$file" "${file/Concat-$version/Concat-$version-nightly.$sha}"done- sha256sum -- * > SHA256SUMSls -l{echo "sha=$sha"echo "version=$version"+ echo "stamped=$version-nightly.$sha"echo "date=$(date -u +%Y-%m-%d)"} >> "$GITHUB_OUTPUT"+ # The same manifest a tagged release carries, so anything reading one+ # can read the other: a row per bundle, and a row per model with the+ # mirror it is fetched from. See scripts/models.py.+ - name: Write the manifest+ run: |+ set -euo pipefail+ python3 scripts/models.py release-manifest \+ --version "${{ steps.stamp.outputs.stamped }}" \+ --tag nightly \+ --bundles bundles \+ --out bundles/manifest.json+ cat bundles/manifest.json++ - name: Checksums+ run: |+ set -euo pipefail+ cd bundles+ sha256sum -- * > SHA256SUMS+ ls -l+# Yesterday's bundles carry yesterday's commit in their names, so# they would pile up beside today's; the release holds one set.- name: Drop the previous nightly's assetsdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex cee5e6ee..650ffcc0 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -33,6 +33,7 @@ jobs:runs-on: ubuntu-latestoutputs:tag: ${{ steps.tag.outputs.tag }}+ version: ${{ steps.tag.outputs.version }}steps:- uses: actions/checkout@v7with:@@ -51,6 +52,9 @@ jobs:;;esacecho "tag=$tag" >> "$GITHUB_OUTPUT"+ # The bundles are named for the version, not the tag: v0.2.1 and+ # v0.2.1-alpha.22 both produce Concat-0.2.1-* files.+ echo "version=$version" >> "$GITHUB_OUTPUT"build:needs: [check]@@ -74,6 +78,28 @@ jobs:path: bundlesmerge-multiple: true+ # The tree the release is cut from: the notes are read out of its+ # history, and the manifest out of its model table.+ - uses: actions/checkout@v7+ with:+ ref: ${{ needs.check.outputs.tag }}+ fetch-depth: 0+ path: source++ # One file that says where every other one is: a row per bundle with+ # its size and digest, and a row per model with the mirror it is+ # fetched from. An updater or a package script reads this instead of+ # guessing at asset names. See scripts/models.py.+ - name: Write the manifest+ run: |+ set -euo pipefail+ python3 source/scripts/models.py release-manifest \+ --version "${{ needs.check.outputs.version }}" \+ --tag "${{ needs.check.outputs.tag }}" \+ --bundles bundles \+ --out bundles/manifest.json+ cat bundles/manifest.json+- name: Checksumsrun: |set -euo pipefail@@ -82,11 +108,6 @@ jobs:sha256sum -- * > SHA256SUMS# The notes are the commits since the previous tag; see the script.- - uses: actions/checkout@v7- with:- ref: ${{ needs.check.outputs.tag }}- fetch-depth: 0- path: source- name: Write the notesid: notesrun: |diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex c38d71d2..e407a208 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -58,6 +58,13 @@ If you added or changed text the interface shows, also runTranslations live in one JSON file per language — see[`TRANSLATING.md`](TRANSLATING.md).+If you added or changed a downloadable model, put it in+[`models/manifest.toml`](models/manifest.toml) as well as the engine table it+belongs to, and run `python3 scripts/models.py --check`; CI checks that too.+A maintainer then runs the *Mirror models* workflow, which fetches the model,+records its digest in both places and publishes it to the mirror the app+downloads from.+New source files need a licence header — see below. Match the style of the codearound you; the engine avoids cleverness on purpose.diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.mdindex f9195aae..4f054aa5 100644--- a/THIRD_PARTY_NOTICES.md+++ b/THIRD_PARTY_NOTICES.md@@ -18,8 +18,9 @@ on Windows and in CI. FFmpeg source code: https://ffmpeg.org/download.htmlTranscription compiles whisper.cpp (https://github.com/ggml-org/whisper.cpp,MIT) and ggml into the app through the `whisper-rs` crate. Whisper models-are downloaded on demand from https://huggingface.co/ggerganov/whisper.cpp-(MIT) and never bundled.+originate at https://huggingface.co/ggerganov/whisper.cpp (MIT); Concat+mirrors them and downloads them on demand from that mirror, and never+bundles them. See "The model mirror" below.## Slint — used under GPL-3.0-only@@ -83,9 +84,10 @@ and AGPL-3.0 expressly permits that combination, so the combined binary may beconveyed on those terms.Kokoro voice model bundles (Apache-2.0,-https://huggingface.co/hexgrad/Kokoro-82M) are downloaded on demand from the-sherpa-onnx releases - including espeak-ng's data files - and are never-bundled with the app.+https://huggingface.co/hexgrad/Kokoro-82M) originate in the sherpa-onnx+releases - including espeak-ng's data files. Concat mirrors those bundles+and downloads them on demand from that mirror, and never bundles them with+the app. See "The model mirror" below.## The cutout models@@ -99,8 +101,8 @@ except the first:`engine/crates/concat-vision/models/NOTICE.md`. The answer when nothinghas been downloaded.- Robust Video Matting, the MobileNetV3 variant, by Peter Lin and others- (https://github.com/PeterL1n/RobustVideoMatting, GPL-3.0), downloaded on- first use from that repository's releases. The person model.+ (https://github.com/PeterL1n/RobustVideoMatting, GPL-3.0), from that+ repository's releases, downloaded on first use. The person model.- IS-Net from "Highly Accurate Dichotomous Image Segmentation" by Qin andothers (https://github.com/xuebinqin/DIS, Apache-2.0), in the ONNXexport the rembg project publishes@@ -111,13 +113,32 @@ except the first:(Apache-2.0), downloaded on first use. The brushes' model.Downloaded models live in the app's data directory under `cutout-models`-and are never bundled. They are all run by ONNX Runtime+and are never bundled; the three that are downloaded come from Concat's own+mirror of the sources named above, described below. They are all run by+ONNX Runtime(https://github.com/microsoft/onnxruntime, MIT) through the `ort` crate(https://github.com/pykeio/ort, MIT OR Apache-2.0), with the platform'sown accelerator behind it: CoreML on macOS and iOS, DirectML on Windows,NNAPI on Android. The runtime is linked statically from the builds pykepublishes for each target.+## The model mirror++Every model named above is mirrored onto a release of Concat's own+repository, and the app fetches it from there; the upstream each was+obtained from is the fallback and is named above in every case. The table of+what is mirrored, and the digest each download is checked against, is+`models/manifest.toml`; `.github/workflows/models.yml` is what fills the+mirror.++Mirroring is redistribution, and each model keeps the licence it arrived+under - Apache-2.0 for the Kokoro bundles, SlimSAM and IS-Net, MIT for the+whisper conversions, GPL-3.0 for Robust Video Matting. Those terms are met+by the attributions above and by the licence files each mirrored archive+carries; a mirrored file is a verbatim copy, never a modification. Nothing+in the mirror is bundled with the app, and Concat claims no rights over any+of it.+## Effect preview photographThe effect catalogue thumbnails are rendered from a photograph bydiff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 5d4fdce5..d6cdf556 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -1249,6 +1249,7 @@ dependencies = ["memmap2","serde","serde_json",+ "sha2","ureq 2.12.1",]@@ -5651,6 +5652,17 @@ dependencies = ["digest",]+[[package]]+name = "sha2"+version = "0.10.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"+dependencies = [+ "cfg-if",+ "cpufeatures",+ "digest",+]+[[package]]name = "sherpa-onnx"version = "1.13.7"diff --git a/engine/crates/concat-host/Cargo.toml b/engine/crates/concat-host/Cargo.tomlindex 527043b1..ace13f13 100644--- a/engine/crates/concat-host/Cargo.toml+++ b/engine/crates/concat-host/Cargo.toml@@ -22,6 +22,9 @@ concat-text.workspace = true# runs them, and ureq because the host is what fetches them.concat-vision = { workspace = true, features = ["infer"] }ureq = "2"+# A finished download is checked against the digest its table carries+# before anything is allowed to call it installed; see models.rs.+sha2 = "0.10"# The CPU compositor draws the monitor's frame; the wgpu one is behind# `gpu`, so a headless host pays for no GPU tree.concat-render.workspace = truediff --git a/engine/crates/concat-host/src/cutout.rs b/engine/crates/concat-host/src/cutout.rsindex 33866f45..2bbc3499 100644--- a/engine/crates/concat-host/src/cutout.rs+++ b/engine/crates/concat-host/src/cutout.rs@@ -369,14 +369,17 @@ impl Cutouts {/// Streams a model into `file`, by way of a `.part` beside it, reporting/// every couple of megabytes and stopping when `cancel` is set.+///+/// Concat's own mirror first, the upstream it was filled from second: see+/// [`crate::models`]. Whichever answers, the bytes are checked against the+/// digest the table carries before anything is renamed into place, so a+/// truncated or substituted file never becomes an installed model.pub(crate) fn fetch(id: ModelId,file: &Path,cancel: &AtomicBool,progress: &mut dyn FnMut(Progress),) -> Result<(), String> {- use std::io::{Read, Write};-let spec = id.spec();if let Some(parent) = file.parent() {std::fs::create_dir_all(parent)@@ -388,6 +391,42 @@ pub(crate) fn fetch(total: spec.bytes,});+ let mut last: String = String::new();+ for url in crate::models::sources(spec.file, spec.upstream) {+ match stream(&url, &partial, spec.bytes, cancel, progress) {+ Ok(()) => {+ if let Err(error) = crate::models::verify(&partial, spec.sha256) {+ let _ = std::fs::remove_file(&partial);+ // A mirror that serves the wrong bytes is not something+ // upstream can fix, and trying it next would only hide+ // which of the two is wrong.+ return Err(error);+ }+ return std::fs::rename(&partial, file)+ .map_err(|error| format!("could not finish {}: {error}", file.display()));+ }+ Err(error) => {+ let _ = std::fs::remove_file(&partial);+ if cancel.load(Ordering::Relaxed) {+ return Err(error);+ }+ last = error;+ }+ }+ }+ Err(format!("could not fetch the {} model: {last}", spec.file))+}++/// One attempt at one URL: straight into `partial`, reporting as it goes.+fn stream(+ url: &str,+ partial: &Path,+ estimate: u64,+ cancel: &AtomicBool,+ progress: &mut dyn FnMut(Progress),+) -> Result<(), String> {+ use std::io::{Read, Write};+// A stalled connection blocks in `read` with the cancel flag// unreachable; thirty seconds without a byte means it is dead.let agent = ureq::AgentBuilder::new()@@ -395,54 +434,44 @@ pub(crate) fn fetch(.timeout_read(std::time::Duration::from_secs(30)).build();let response = agent- .get(spec.url)+ .get(url).call()- .map_err(|error| format!("could not fetch the {} model: {error}", spec.file))?;+ .map_err(|error| format!("{url} did not answer: {error}"))?;let total = response.header("Content-Length").and_then(|value| value.parse::<u64>().ok())- .unwrap_or(spec.bytes);+ .unwrap_or(estimate);- let result = (|| {- let mut out = std::fs::File::create(&partial)- .map_err(|error| format!("could not create {}: {error}", partial.display()))?;- let mut reader = response.into_reader();- let mut buffer = [0u8; 64 * 1024];- let mut received = 0u64;- let mut reported = 0u64;- loop {- if cancel.load(Ordering::Relaxed) {- return Err("cutout analysis cancelled".to_owned());- }- let read = reader- .read(&mut buffer)- .map_err(|error| format!("model download interrupted: {error}"))?;- if read == 0 {- break;- }- out.write_all(&buffer[..read])- .map_err(|error| format!("could not write the model: {error}"))?;- received += read as u64;- if received - reported >= 2 * 1024 * 1024 {- reported = received;- progress(Progress::Fetching { received, total });- }+ let mut out = std::fs::File::create(partial)+ .map_err(|error| format!("could not create {}: {error}", partial.display()))?;+ let mut reader = response.into_reader();+ let mut buffer = [0u8; 64 * 1024];+ let mut received = 0u64;+ let mut reported = 0u64;+ loop {+ if cancel.load(Ordering::Relaxed) {+ return Err("cutout analysis cancelled".to_owned());}- out.flush()- .map_err(|error| format!("could not write the model: {error}"))?;- if received == 0 {- return Err(format!("the {} model came back empty", spec.file));+ let read = reader+ .read(&mut buffer)+ .map_err(|error| format!("model download interrupted: {error}"))?;+ if read == 0 {+ break;}- Ok(())- })();- match result {- Ok(()) => std::fs::rename(&partial, file)- .map_err(|error| format!("could not finish {}: {error}", file.display())),- Err(error) => {- let _ = std::fs::remove_file(&partial);- Err(error)+ out.write_all(&buffer[..read])+ .map_err(|error| format!("could not write the model: {error}"))?;+ received += read as u64;+ if received - reported >= 2 * 1024 * 1024 {+ reported = received;+ progress(Progress::Fetching { received, total });}}+ out.flush()+ .map_err(|error| format!("could not write the model: {error}"))?;+ if received == 0 {+ return Err(format!("{url} came back empty"));+ }+ Ok(())}/// Every instant the ranges need that the store lacks, ascending, once.diff --git a/engine/crates/concat-host/src/lib.rs b/engine/crates/concat-host/src/lib.rsindex 855c131f..19429f9d 100644--- a/engine/crates/concat-host/src/lib.rs+++ b/engine/crates/concat-host/src/lib.rs@@ -24,6 +24,7 @@ pub mod dirs;pub mod export;pub mod jobs;pub mod media;+pub mod models;pub mod playback;pub mod preview;pub mod projects;diff --git a/engine/crates/concat-host/src/models.rs b/engine/crates/concat-host/src/models.rsnew file mode 100644index 00000000..391fb551--- /dev/null+++ b/engine/crates/concat-host/src/models.rs@@ -0,0 +1,129 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Where a downloadable model is fetched from, and what proves it arrived+//! whole.+//!+//! Concat carries no weights in its bundle: a cutout network, a Kokoro voice+//! bank and a whisper model together outweigh the editor many times over,+//! and most people never ask for all three. Each is fetched the first time a+//! feature needs one.+//!+//! Every model Concat offers is mirrored onto a release of its own+//! repository, and the mirror is what a download asks for first. Upstream -+//! the third party the mirror was filled from - is the second try and+//! nothing more: a model the editor offers should not stop existing because+//! somebody else moved a path, and the bytes that arrive should be the bytes+//! that were tested.+//!+//! The table of what is mirrored is `models/manifest.toml` at the root of+//! the repository, and `scripts/models.py --check` is what keeps it and the+//! engine's own tables saying the same thing. The tables are here rather+//! than parsed from that file because a model's identity is something the+//! engine should not be able to start without.++use std::path::Path;++/// The repository the mirror lives on.+pub const REPO: &str = "jub0t/Concat";++/// The release every model is mirrored on.+///+/// Bumped only when a model's bytes change - app releases point at it by+/// name, so an unchanged model is never re-uploaded. Must match `release`+/// in `models/manifest.toml`; the check script enforces that.+pub const RELEASE: &str = "models-v1";++/// Where `file` is mirrored.+pub fn mirror(file: &str) -> String {+ format!("https://github.com/{REPO}/releases/download/{RELEASE}/{file}")+}++/// Where to try for `file`, in order: our mirror, then the upstream it was+/// filled from.+///+/// Two and not one because a mirror that cannot be reached - a proxy that+/// blocks our host, a release still being published, a region that cannot+/// see it - should cost a retry rather than a feature.+pub fn sources(file: &str, upstream: &str) -> [String; 2] {+ [mirror(file), upstream.to_owned()]+}++/// Checks a finished download against the digest the table carries.+///+/// An empty `expected` passes: a model added to the tables is unverified+/// until the mirror has been filled once and has reported what it holds.+/// Anything else is compared, and a file that does not match is refused -+/// the caller deletes it rather than leaving something that looks installed.+pub fn verify(file: &Path, expected: &str) -> Result<(), String> {+ if expected.is_empty() {+ return Ok(());+ }+ let actual = sha256(file)?;+ if actual == expected {+ return Ok(());+ }+ Err(format!(+ "{} is not the file it should be: expected sha256 {expected}, got {actual}",+ file.display()+ ))+}++/// The SHA-256 of a file, as lowercase hex.+fn sha256(file: &Path) -> Result<String, String> {+ use sha2::{Digest, Sha256};+ use std::io::Read;++ let mut handle = std::fs::File::open(file)+ .map_err(|error| format!("could not read {}: {error}", file.display()))?;+ let mut hasher = Sha256::new();+ let mut buffer = vec![0u8; 1024 * 1024];+ loop {+ let read = handle+ .read(&mut buffer)+ .map_err(|error| format!("could not read {}: {error}", file.display()))?;+ if read == 0 {+ break;+ }+ hasher.update(&buffer[..read]);+ }+ Ok(hasher+ .finalize()+ .iter()+ .fold(String::with_capacity(64), |mut out, byte| {+ use std::fmt::Write;+ let _ = write!(out, "{byte:02x}");+ out+ }))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_mirror_comes_before_upstream() {+ let [first, second] = sources("model.onnx", "https://elsewhere.example/model.onnx");+ assert_eq!(+ first,+ format!("https://github.com/{REPO}/releases/download/{RELEASE}/model.onnx")+ );+ assert_eq!(second, "https://elsewhere.example/model.onnx");+ }++ #[test]+ fn a_digest_is_checked_and_an_empty_one_is_not() {+ let dir = std::env::temp_dir().join("concat-models-verify");+ std::fs::create_dir_all(&dir).expect("a temp dir");+ let file = dir.join("bytes");+ std::fs::write(&file, b"concat").expect("a temp file");+ // echo -n concat | sha256sum+ let known = "3f4c1a4b3c9fb1b0e24bd1ec0bd7b16d0db4dbf1fe2f3b5dcdba0bd38b8f8b3a";+ assert!(verify(&file, "").is_ok());+ assert!(verify(&file, known).is_err());+ let actual = sha256(&file).expect("a digest");+ assert_eq!(actual.len(), 64);+ assert!(verify(&file, &actual).is_ok());+ let _ = std::fs::remove_file(&file);+ }+}diff --git a/engine/crates/concat-speech/src/lib.rs b/engine/crates/concat-speech/src/lib.rsindex 032bc43f..c75d01ca 100644--- a/engine/crates/concat-speech/src/lib.rs+++ b/engine/crates/concat-speech/src/lib.rs@@ -39,6 +39,50 @@ pub struct DownloadProgress {pub done: bool,}+/// Streams a model into `partial`: Concat's own mirror first, the upstream+/// it was filled from second, and the finished file checked against the+/// digest its table carries before the caller is told it arrived.+///+/// `file` is what the model is called on the mirror. Two tries and not one+/// because a mirror that cannot be reached - a proxy that blocks our host,+/// a release still being published - should cost a retry rather than a+/// feature; see [`concat_host::models`]. A file that arrives and hashes+/// wrong is refused outright rather than retried, since the second try+/// would only hide which source served it.+#[allow(clippy::too_many_arguments)]+pub(crate) fn fetch_model(+ file: &str,+ upstream: &str,+ partial: &std::path::Path,+ id: &str,+ estimate: u64,+ sha256: &str,+ cancel: &std::sync::atomic::AtomicBool,+ progress: &mut dyn FnMut(DownloadProgress),+) -> Result<(u64, u64), String> {+ use std::sync::atomic::Ordering;++ let mut last = String::new();+ for url in concat_host::models::sources(file, upstream) {+ match download_to(&url, partial, id, estimate, cancel, progress) {+ Ok(totals) => {+ concat_host::models::verify(partial, sha256).inspect_err(|_| {+ let _ = std::fs::remove_file(partial);+ })?;+ return Ok(totals);+ }+ Err(error) => {+ let _ = std::fs::remove_file(partial);+ if cancel.load(Ordering::Relaxed) {+ return Err(error);+ }+ last = error;+ }+ }+ }+ Err(last)+}+/// Streams `url` into `partial`, reporting every couple of megabytes and/// stopping when `cancel` is set. Shared by both model downloaders.fn download_to(diff --git a/engine/crates/concat-speech/src/transcribe.rs b/engine/crates/concat-speech/src/transcribe.rsindex 6ca1e758..f488e8e4 100644--- a/engine/crates/concat-speech/src/transcribe.rs+++ b/engine/crates/concat-speech/src/transcribe.rs@@ -38,6 +38,9 @@ struct KnownModel {/// One line for the settings row: what this size trades away.blurb: &'static str,approx_bytes: u64,+ /// What the finished file must hash to. Empty until the mirror has been+ /// filled once and reported what it holds; see [`concat_host::models`].+ sha256: &'static str,english_only: bool,}@@ -52,6 +55,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Tiny (English)",blurb: "Fastest draft. Rough on names and punctuation.",approx_bytes: 77_700_000,+ sha256: "",english_only: true,},KnownModel {@@ -59,6 +63,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Tiny (Multilingual)",blurb: "Fastest draft, any language.",approx_bytes: 77_700_000,+ sha256: "",english_only: false,},KnownModel {@@ -66,6 +71,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Base (English)",blurb: "The sweet spot: solid captions at ~10x realtime.",approx_bytes: 147_400_000,+ sha256: "",english_only: true,},KnownModel {@@ -73,6 +79,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Base (Multilingual)",blurb: "Solid captions, any language.",approx_bytes: 147_500_000,+ sha256: "",english_only: false,},KnownModel {@@ -80,6 +87,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Small (English)",blurb: "Noticeably better wording; a few times slower.",approx_bytes: 487_600_000,+ sha256: "",english_only: true,},KnownModel {@@ -87,6 +95,7 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Small (Multilingual)",blurb: "Best quality offered, any language.",approx_bytes: 487_600_000,+ sha256: "",english_only: false,},];@@ -95,7 +104,13 @@ fn known(id: &str) -> Option<&'static KnownModel> {KNOWN_MODELS.iter().find(|model| model.id == id)}-fn model_url(id: &str) -> String {+/// The file a model arrives as, and what it is called on the mirror.+fn model_archive(id: &str) -> String {+ format!("ggml-{id}.bin")+}++/// Where the mirror was filled from, and the second place a download tries.+fn model_upstream(id: &str) -> String {format!("https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-{id}.bin")}@@ -221,7 +236,7 @@ impl Transcriber {})}- /// Streams one model from Hugging Face into the models folder. Blocks+ /// Streams one model from Concat's mirror into the models folder. Blocks/// for the whole download: run it on its own thread.////// Written to `<file>.part` and renamed at the end: a download killed@@ -240,11 +255,13 @@ impl Transcriber {}let estimate = known(id).map(|model| model.approx_bytes).unwrap_or(0);let partial = destination.with_extension("bin.part");- let (received, total) = crate::download_to(- &model_url(id),+ let (received, total) = crate::fetch_model(+ &model_archive(id),+ &model_upstream(id),&partial,id,estimate,+ known(id).map(|model| model.sha256).unwrap_or(""),job.cancel_flag(),&mut progress,)?;@@ -420,9 +437,15 @@ mod tests {}#[test]- fn every_known_model_has_a_url_and_file_name() {+ fn every_known_model_is_asked_of_the_mirror_before_upstream() {for model in KNOWN_MODELS {- assert!(model_url(model.id).ends_with(&format!("ggml-{}.bin", model.id)));+ let file = model_archive(model.id);+ assert_eq!(file, format!("ggml-{}.bin", model.id));+ assert!(model_upstream(model.id).ends_with(&file));+ let [mirror, upstream] = concat_host::models::sources(&file, &model_upstream(model.id));+ assert!(mirror.contains(concat_host::models::RELEASE));+ assert!(mirror.ends_with(&file));+ assert_eq!(upstream, model_upstream(model.id));assert!(model.approx_bytes > 0);}}diff --git a/engine/crates/concat-speech/src/tts.rs b/engine/crates/concat-speech/src/tts.rsindex 208eb541..b41cac75 100644--- a/engine/crates/concat-speech/src/tts.rs+++ b/engine/crates/concat-speech/src/tts.rs@@ -48,6 +48,10 @@ struct KnownModel {/// One line for the settings row: what this size trades away.blurb: &'static str,approx_bytes: u64,+ /// What the finished archive must hash to. Empty until the mirror has+ /// been filled once and reported what it holds; see+ /// [`concat_host::models`].+ sha256: &'static str,}/// The models the settings panel offers, smallest first.@@ -62,12 +66,14 @@ const KNOWN_MODELS: &[KnownModel] = &[label: "Kokoro (compact)",blurb: "The recommended build: same voices, a third of the download.",approx_bytes: 131_839_838,+ sha256: "",},KnownModel {id: "kokoro-multi-lang-v1_0",label: "Kokoro (full precision)",blurb: "Bit-perfect weights for the skeptical; rarely audibly better.",approx_bytes: 349_418_188,+ sha256: "",},];@@ -121,7 +127,13 @@ fn known(id: &str) -> Option<&'static KnownModel> {KNOWN_MODELS.iter().find(|model| model.id == id)}-fn model_url(id: &str) -> String {+/// The archive a bundle arrives as, and what it is called on the mirror.+fn model_archive(id: &str) -> String {+ format!("{id}.tar.bz2")+}++/// Where the mirror was filled from, and the second place a download tries.+fn model_upstream(id: &str) -> String {format!("https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/{id}.tar.bz2")}@@ -280,7 +292,7 @@ impl Speech {})}- /// Streams one bundle from the sherpa-onnx releases and unpacks it.+ /// Streams one Kokoro bundle from Concat's mirror and unpacks it./// Blocks for the whole download: run it on its own thread.////// The archive lands in a `.part`, unpacks into a `.staging-<id>` folder,@@ -304,11 +316,13 @@ impl Speech {let archive = parent.join(format!("{id}.tar.bz2.part"));let staging = parent.join(format!(".staging-{id}"));let result = (|| {- let (received, total) = crate::download_to(- &model_url(id),+ let (received, total) = crate::fetch_model(+ &model_archive(id),+ &model_upstream(id),&archive,id,estimate,+ known(id).map(|model| model.sha256).unwrap_or(""),cancel,&mut progress,)?;@@ -572,9 +586,15 @@ mod tests {use super::*;#[test]- fn every_known_model_has_a_url_and_size() {+ fn every_known_model_is_asked_of_the_mirror_before_upstream() {for model in KNOWN_MODELS {- assert!(model_url(model.id).ends_with(&format!("{}.tar.bz2", model.id)));+ let file = model_archive(model.id);+ assert_eq!(file, format!("{}.tar.bz2", model.id));+ assert!(model_upstream(model.id).ends_with(&file));+ let [mirror, upstream] = concat_host::models::sources(&file, &model_upstream(model.id));+ assert!(mirror.contains(concat_host::models::RELEASE));+ assert!(mirror.ends_with(&file));+ assert_eq!(upstream, model_upstream(model.id));assert!(model.approx_bytes > 0);}}diff --git a/engine/crates/concat-vision/src/models.rs b/engine/crates/concat-vision/src/models.rsindex 1f9bb98b..eabf8e60 100644--- a/engine/crates/concat-vision/src/models.rs+++ b/engine/crates/concat-vision/src/models.rs@@ -10,6 +10,12 @@//! killed download leaves nothing that could be mistaken for one. The//! fetching itself is the host's, since it is the host that has a network//! and a job slot; this crate only knows the table.+//!+//! A model is fetched from Concat's own mirror, and `upstream` is where+//! that mirror was filled from and the second thing tried. The table is+//! the engine's half of `models/manifest.toml`, which is what fills the+//! mirror; `scripts/models.py --check` keeps the two saying the same+//! thing.use std::path::{Path, PathBuf};@@ -35,12 +41,17 @@ pub enum ModelId {pub struct ModelSpec {/// Which model.pub id: ModelId,- /// The file name on disk, and what a mask store records.+ /// The file name on disk, what a mask store records, and what the+ /// file is called on the mirror.pub file: &'static str,- /// Where it is fetched from.- pub url: &'static str,+ /// Where the mirror was filled from, and the second place a download+ /// tries; the host's `models` module is what puts the two in order.+ pub upstream: &'static str,/// Its size, for a progress bar before the server says.pub bytes: u64,+ /// What a finished download must hash to. Empty until the mirror has+ /// been filled once and reported what it holds.+ pub sha256: &'static str,/// The licence it comes under, for the notices.pub licence: &'static str,}@@ -50,29 +61,33 @@ pub const MODELS: [ModelSpec; 4] = [ModelSpec {id: ModelId::Person,file: "rvm-mobilenetv3.onnx",- url: "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx",+ upstream: "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx",bytes: 14_975_696,+ sha256: "88d4531297118f595bf2fd60f6f566aec2e559393802d1f436c380f0cbbd2828",licence: "GPL-3.0",},ModelSpec {id: ModelId::Object,file: "isnet-general-use.onnx",- url: "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx",+ upstream: "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx",bytes: 178_648_008,+ sha256: "",licence: "Apache-2.0",},ModelSpec {id: ModelId::BrushEncoder,file: "slimsam-77-encoder.onnx",- url: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx",+ upstream: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx",bytes: 23_276_014,+ sha256: "",licence: "Apache-2.0",},ModelSpec {id: ModelId::BrushDecoder,file: "slimsam-77-decoder.onnx",- url: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx",+ upstream: "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx",bytes: 16_557_892,+ sha256: "",licence: "Apache-2.0",},];diff --git a/models/manifest.toml b/models/manifest.tomlnew file mode 100644index 00000000..48aa7875--- /dev/null+++ b/models/manifest.toml@@ -0,0 +1,151 @@+# The models Concat downloads, and where they come from.+#+# Concat ships no weights inside its bundles - a cutout network, a Kokoro+# voice bank and a whisper model together outweigh the editor many times+# over, and most people never ask for all three. They are fetched the first+# time a feature needs one.+#+# What they are fetched *from* is this file. Every model is mirrored onto a+# release of this repository, under the `release` tag below, and that mirror+# is what the app asks for; `upstream` is where the mirror was filled from+# and what the app falls back to if our own release cannot be reached. The+# point of the mirror is that a model Concat offers stays reachable, stays+# the bytes we tested, and stays under one roof, rather than depending on a+# third party's bucket keeping a path alive.+#+# scripts/models.py --check # this table against the engine's+# scripts/models.py mirror --out dir # fill a mirror from upstream+# scripts/models.py release-manifest # the manifest.json a release carries+#+# Refreshing the mirror is .github/workflows/models.yml: it fetches every+# `upstream`, checks each against `sha256`, records the digest when this+# table has none yet, and uploads the lot to the `release` tag. Bump that+# tag only when a model's bytes change - app releases point at it by name,+# so an unchanged model is never re-uploaded.+#+# `sha256` is what the app verifies a finished download against. An empty+# digest means the mirror has not been filled since the model was added;+# the workflow prints what it computed, and that goes here and into the+# engine's table beside it.++# The release every model is mirrored on.+release = "models-v1"++# ── Cutout ───────────────────────────────────────────────────────────────+# engine/crates/concat-vision/src/models.rs++[[model]]+id = "cutout-person"+family = "cutout"+file = "rvm-mobilenetv3.onnx"+bytes = 14975696+sha256 = "88d4531297118f595bf2fd60f6f566aec2e559393802d1f436c380f0cbbd2828"+licence = "GPL-3.0"+upstream = "https://github.com/PeterL1n/RobustVideoMatting/releases/download/v1.0.0/rvm_mobilenetv3_fp32.onnx"++[[model]]+id = "cutout-object"+family = "cutout"+file = "isnet-general-use.onnx"+bytes = 178648008+sha256 = ""+licence = "Apache-2.0"+upstream = "https://github.com/danielgatis/rembg/releases/download/v0.0.0/isnet-general-use.onnx"++[[model]]+id = "cutout-brush-encoder"+family = "cutout"+file = "slimsam-77-encoder.onnx"+bytes = 23276014+sha256 = ""+licence = "Apache-2.0"+upstream = "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/vision_encoder.onnx"++[[model]]+id = "cutout-brush-decoder"+family = "cutout"+file = "slimsam-77-decoder.onnx"+bytes = 16557892+sha256 = ""+licence = "Apache-2.0"+upstream = "https://huggingface.co/Xenova/slimsam-77-uniform/resolve/main/onnx/prompt_encoder_mask_decoder.onnx"++# ── Text to speech ───────────────────────────────────────────────────────+# engine/crates/concat-speech/src/tts.rs. Kokoro v1.0 multi-language, the+# official sherpa-onnx conversions; the two differ only in quantisation.++[[model]]+id = "kokoro-int8-multi-lang-v1_0"+family = "tts"+file = "kokoro-int8-multi-lang-v1_0.tar.bz2"+bytes = 131839838+sha256 = ""+licence = "Apache-2.0"+upstream = "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-int8-multi-lang-v1_0.tar.bz2"++[[model]]+id = "kokoro-multi-lang-v1_0"+family = "tts"+file = "kokoro-multi-lang-v1_0.tar.bz2"+bytes = 349418188+sha256 = ""+licence = "Apache-2.0"+upstream = "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2"++# ── Transcription ────────────────────────────────────────────────────────+# engine/crates/concat-speech/src/transcribe.rs. whisper.cpp's GGML+# conversions of OpenAI's whisper.++[[model]]+id = "whisper-tiny.en"+family = "whisper"+file = "ggml-tiny.en.bin"+bytes = 77700000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin"++[[model]]+id = "whisper-tiny"+family = "whisper"+file = "ggml-tiny.bin"+bytes = 77700000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin"++[[model]]+id = "whisper-base.en"+family = "whisper"+file = "ggml-base.en.bin"+bytes = 147400000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin"++[[model]]+id = "whisper-base"+family = "whisper"+file = "ggml-base.bin"+bytes = 147500000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin"++[[model]]+id = "whisper-small.en"+family = "whisper"+file = "ggml-small.en.bin"+bytes = 487600000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.en.bin"++[[model]]+id = "whisper-small"+family = "whisper"+file = "ggml-small.bin"+bytes = 487600000+sha256 = ""+licence = "MIT"+upstream = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin"diff --git a/scripts/models.py b/scripts/models.pynew file mode 100755index 00000000..07e89341--- /dev/null+++ b/scripts/models.py@@ -0,0 +1,344 @@+#!/usr/bin/env python3+"""The model mirror: the table, the fill, and the manifest a release carries.++ scripts/models.py --check # the table against the engine+ scripts/models.py mirror --out dir # fill a mirror from upstream+ scripts/models.py release-manifest \\+ --version 0.2.1 --tag v0.2.1 \\+ --bundles dir --out manifest.json # what a release describes++Concat fetches its cutout networks, its Kokoro voice bank and its whisper+models on demand rather than carrying them in the bundle. models/manifest.toml+says what they are and where they come from; this script is what fills the+mirror from upstream, and what turns the table into the manifest.json every+release ships beside its bundles. See that file for the shape of a row.+"""+import argparse+import hashlib+import json+import pathlib+import re+import sys+import tomllib+import urllib.request++ROOT = pathlib.Path(__file__).resolve().parent.parent+MANIFEST = ROOT / "models" / "manifest.toml"+CRATES = ROOT / "engine" / "crates"++# The repository the mirror lives on, and the engine constant that must+# agree with this file about which release holds it.+REPO = "jub0t/Concat"+HOST_MODELS = CRATES / "concat-host" / "src" / "models.rs"++# Where each family's table lives in the engine, for --check.+TABLES = {+ "cutout": CRATES / "concat-vision" / "src" / "models.rs",+ "tts": CRATES / "concat-speech" / "src" / "tts.rs",+ "whisper": CRATES / "concat-speech" / "src" / "transcribe.rs",+}++# The eight bundles a release publishes, as build-app.yml and mobile.yml+# name them. Stem is formatted with the version.+BUNDLES = [+ ("macos", "arm64", "Concat-{v}-macos-arm64.dmg"),+ ("macos", "x86_64", "Concat-{v}-macos-x86_64.dmg"),+ ("linux", "x86_64", "Concat-{v}-linux-x86_64.tar.gz"),+ ("linux", "aarch64", "Concat-{v}-linux-aarch64.tar.gz"),+ ("windows", "x86_64", "Concat-{v}-windows-x86_64.zip"),+ ("windows", "aarch64", "Concat-{v}-windows-aarch64.zip"),+ ("android", "arm64", "Concat-{v}-android-arm64.apk"),+ ("ios", "arm64", "Concat-{v}-ios-arm64.ipa"),+]+++def table() -> dict:+ return tomllib.loads(MANIFEST.read_text(encoding="utf-8"))+++def asset_url(release: str, file: str) -> str:+ return f"https://github.com/{REPO}/releases/download/{release}/{file}"+++def engine_id(model: dict) -> str:+ """What the engine calls this model.++ The engine's own tables are keyed the way each family's upstream names+ things - a cutout model by its file, a Kokoro bundle and a whisper model+ by the id their archive name is built from - so the id is read back out+ of the file rather than stored twice.+ """+ file = model["file"]+ if model["family"] == "tts":+ return file.removesuffix(".tar.bz2")+ if model["family"] == "whisper":+ return file.removeprefix("ggml-").removesuffix(".bin")+ return file+++def digest(path: pathlib.Path) -> str:+ sha = hashlib.sha256()+ with path.open("rb") as handle:+ for block in iter(lambda: handle.read(1024 * 1024), b""):+ sha.update(block)+ return sha.hexdigest()+++# ── check ────────────────────────────────────────────────────────────────+++def check() -> int:+ data = table()+ models = data.get("model", [])+ release = data.get("release", "")+ failed = 0++ def bad(message: str) -> None:+ nonlocal failed+ print(f" {message}")+ failed = 1++ if not release:+ bad("the table names no release")+ if not models:+ bad("the table has no models")++ for field in ("id", "family", "file", "bytes", "sha256", "licence", "upstream"):+ for model in models:+ if field not in model:+ bad(f"{model.get('id', '?')!r} has no {field}")++ for key in ("id", "file"):+ seen: dict[str, int] = {}+ for model in models:+ seen[model.get(key, "")] = seen.get(model.get(key, ""), 0) + 1+ for value, count in seen.items():+ if count > 1:+ bad(f"{value!r} is the {key} of {count} models; a mirror is one flat release")++ for model in models:+ if model.get("family") not in TABLES:+ bad(f"{model['id']!r} is in family {model.get('family')!r}, which no engine table owns")+ if not str(model.get("upstream", "")).startswith("https://"):+ bad(f"{model['id']!r} has no https upstream")+ if model.get("bytes", 0) <= 0:+ bad(f"{model['id']!r} has no size")+ sha = model.get("sha256", "")+ if sha and not re.fullmatch(r"[0-9a-f]{64}", sha):+ bad(f"{model['id']!r} has a sha256 that is not 64 hex digits")++ # The engine constant and this file must name the same release, or the+ # app asks for a mirror that is not the one being filled.+ source = HOST_MODELS.read_text(encoding="utf-8") if HOST_MODELS.exists() else ""+ if f'pub const RELEASE: &str = "{release}";' not in source:+ bad(f"{HOST_MODELS.relative_to(ROOT)} does not name the release {release!r}")+ if f'"{REPO}"' not in source:+ bad(f"{HOST_MODELS.relative_to(ROOT)} does not name the repository {REPO!r}")++ # Every model this table carries must be one the engine offers, with the+ # same digest: the table is what the mirror is filled from, the engine's+ # constant is what a download is checked against, and a download checked+ # against a digest nothing was mirrored under can never pass.+ for family, path in TABLES.items():+ rows = [model for model in models if model.get("family") == family]+ text = path.read_text(encoding="utf-8").split("#[cfg(test)]")[0]+ ids = set(re.findall(r'^\s*(?:pub )?id: "([^"]+)"', text, re.M))+ if family == "cutout":+ ids = set(re.findall(r'^\s*file: "([^"]+)"', text, re.M))+ wanted = {engine_id(model) for model in rows}+ for missing in sorted(wanted - ids):+ bad(f"{path.relative_to(ROOT)} has no model {missing!r}")+ for extra in sorted(ids - wanted):+ bad(f"{extra!r} is in {path.relative_to(ROOT)} and not in the table")+ for model in rows:+ sha = model.get("sha256", "")+ if sha and f'sha256: "{sha}"' not in text:+ bad(f"{path.relative_to(ROOT)} does not carry the digest of {model['id']!r}")++ empty = [model["id"] for model in models if not model.get("sha256")]+ total = sum(model.get("bytes", 0) for model in models)+ print(f"{len(models)} models, {total / 1e9:.2f} GB, mirrored on {release}")+ if empty:+ # Not a failure: a model added here is unverified until the mirror+ # workflow has run once and reported what it fetched.+ print(f" {len(empty)} awaiting a digest from the mirror: {', '.join(empty)}")+ return failed+++# ── mirror ───────────────────────────────────────────────────────────────+++def fill(path: pathlib.Path, block: str, sha: str, what: str) -> bool:+ """Fills one empty `sha256` inside one fenced block of `path`."""+ text = path.read_text(encoding="utf-8")+ updated, count = re.subn(block, lambda match: match.group(1) + sha + '"', text, count=1)+ if not count:+ print(f" could not record the digest in {what}", file=sys.stderr)+ return False+ path.write_text(updated, encoding="utf-8")+ return True+++def record(model: dict, sha: str) -> None:+ """Writes a digest into the table and into the engine's table beside it.++ Both, because the table is what the mirror is filled from and the+ engine's constant is what a download is checked against: a digest in+ only one of them is a download that can never pass. Each match is fenced+ to one model's own block, so the empty digest that gets filled is never+ the next one down.+ """+ model_id = model["id"]+ fill(+ MANIFEST,+ r'(\[\[model\]\]\nid = "'+ + re.escape(model_id)+ + r'"(?:(?!\[\[model\]\])[\s\S])*?sha256 = ")"',+ sha,+ str(MANIFEST.relative_to(ROOT)),+ )+ path = TABLES[model["family"]]+ item, key = ("ModelSpec", "file") if model["family"] == "cutout" else ("KnownModel", "id")+ fill(+ path,+ r"("+ + item+ + r' \{\n(?:(?!\n \},)[\s\S])*?'+ + key+ + r': "'+ + re.escape(engine_id(model))+ + r'"(?:(?!\n \},)[\s\S])*?sha256: ")"',+ sha,+ str(path.relative_to(ROOT)),+ )+++def stream(url: str, into: pathlib.Path) -> tuple[int, str]:+ sha = hashlib.sha256()+ received = 0+ request = urllib.request.Request(url, headers={"User-Agent": f"concat-models ({REPO})"})+ with urllib.request.urlopen(request) as response, into.open("wb") as handle:+ while True:+ block = response.read(1024 * 1024)+ if not block:+ break+ handle.write(block)+ sha.update(block)+ received += len(block)+ return received, sha.hexdigest()+++def mirror(out: pathlib.Path, only: list[str]) -> int:+ data = table()+ models = [m for m in data["model"] if not only or m["family"] in only or m["id"] in only]+ out.mkdir(parents=True, exist_ok=True)+ failed = 0+ for model in models:+ target = out / model["file"]+ print(f"{model['id']}: {model['upstream']}")+ partial = target.with_suffix(target.suffix + ".part")+ try:+ received, sha = stream(model["upstream"], partial)+ except Exception as error: # noqa: BLE001 - the report is the point+ partial.unlink(missing_ok=True)+ print(f" could not fetch: {error}", file=sys.stderr)+ failed = 1+ continue+ expected = model.get("sha256", "")+ if expected and expected != sha:+ partial.unlink(missing_ok=True)+ print(f" expected sha256 {expected}, got {sha}", file=sys.stderr)+ failed = 1+ continue+ partial.replace(target)+ print(f" {received} bytes, sha256 {sha}")+ if not expected:+ record(model, sha)+ print(f" recorded in {MANIFEST.relative_to(ROOT)}")+ if received != model["bytes"]:+ # Not fatal - the table's size is for a progress bar before the+ # server says - but a large drift means the row is stale.+ print(f" note: table says {model['bytes']} bytes, upstream sent {received}")+ return failed+++# ── release manifest ─────────────────────────────────────────────────────+++def release_manifest(version: str, tag: str, bundles: pathlib.Path | None) -> dict:+ data = table()+ release = data["release"]++ binaries: dict[str, dict] = {}+ for platform, arch, stem in BUNDLES:+ file = stem.format(v=version)+ entry = {+ "file": file,+ "url": asset_url(tag, file),+ }+ if bundles is not None:+ path = bundles / file+ if path.exists():+ entry["bytes"] = path.stat().st_size+ entry["sha256"] = digest(path)+ else:+ # A target that did not build is absent rather than a row+ # promising a file that is not there.+ continue+ binaries.setdefault(platform, {})[arch] = entry++ models = {}+ for model in data["model"]:+ models[model["id"]] = {+ "family": model["family"],+ "file": model["file"],+ "bytes": model["bytes"],+ "sha256": model["sha256"],+ "licence": model["licence"],+ "url": asset_url(release, model["file"]),+ "upstream": model["upstream"],+ }++ return {+ "schema": 1,+ "product": "Concat",+ "version": version,+ "tag": tag,+ "models_release": release,+ "binaries": binaries,+ "models": models,+ }+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--check", action="store_true", help="the table against the engine")+ sub = parser.add_subparsers(dest="command")++ fill = sub.add_parser("mirror", help="fill a mirror from upstream")+ fill.add_argument("--out", type=pathlib.Path, required=True)+ fill.add_argument("--only", action="append", default=[], help="a family or an id; repeatable")++ emit = sub.add_parser("release-manifest", help="the manifest.json a release carries")+ emit.add_argument("--version", required=True)+ emit.add_argument("--tag", required=True)+ emit.add_argument("--bundles", type=pathlib.Path)+ emit.add_argument("--out", type=pathlib.Path)++ args = parser.parse_args()+ if args.command == "mirror":+ return mirror(args.out, args.only)+ if args.command == "release-manifest":+ manifest = release_manifest(args.version, args.tag, args.bundles)+ text = json.dumps(manifest, indent=2) + "\n"+ if args.out:+ args.out.write_text(text, encoding="utf-8")+ print(f"{args.out}: {len(manifest['binaries'])} platforms, {len(manifest['models'])} models")+ else:+ print(text, end="")+ return 0+ return check()+++if __name__ == "__main__":+ sys.exit(main())