Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

Cutout inference on ONNX Runtime, with the platform's accelerator

Jareerd23f0c7d8b01
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 30d81b06..fbf7ae48 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -78,9 +78,11 @@ concat-android ──► concat concat-cli ──► concat-core, con
because the document needs serde and core's zero-dependency rule is worth
more than the adjacency. Five crates carry no native library and build for
the web as well (`concat-core`, `concat-project`, `concat-effects`,
-`concat-render`, `concat-text`); CI keeps that true. `concat-vision` is
-pure Rust too, inference included: tract runs the model, so a cutout needs
-no runtime library on any platform.
+`concat-render`, `concat-text`); CI keeps that true. `concat-vision`
+links ONNX Runtime behind its `infer` feature, prebuilt for every target
+the app ships on, with the platform's accelerator registered ahead of the
+CPU; without the feature it is pure Rust, and that is how the renderer
+takes it.
---
@@ -421,8 +423,9 @@ look repaints.
A cutout takes a picture's background away without a key colour. The
model is MediaPipe's selfie segmentation, compiled into `concat-vision`
-and run by tract: a 256 × 256 picture in, a probability per pixel out,
-about twenty-five milliseconds a frame on one core. Masks are square
+and run by ONNX Runtime: a 256 × 256 picture in, a probability per pixel
+out, five milliseconds a frame on an Apple GPU and about twenty on a
+laptop's cores. Masks are square
whatever the picture's shape, and every position in one is a fraction of
the source picture, which is also how strokes are stored; a crop, a flip
or a change of output size changes nothing about either, because
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index 79be655d..1a005266 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -93,9 +93,12 @@ Remove background's automatic and custom modes run Google's MediaPipe
Selfie Segmentation model (Apache-2.0), in the ONNX conversion published by
the ONNX Community (https://huggingface.co/onnx-community/mediapipe_selfie_segmentation,
Apache-2.0). The model file is compiled into the `concat-vision` crate; see
-`engine/crates/concat-vision/models/NOTICE.md`. It is run by tract
-(https://github.com/sonos/tract, MIT OR Apache-2.0), in pure Rust, so no
-inference runtime is linked or shipped for it.
+`engine/crates/concat-vision/models/NOTICE.md`. It is run by ONNX Runtime
+(https://github.com/microsoft/onnxruntime, MIT) through the `ort` crate
+(https://github.com/pykeio/ort, MIT OR Apache-2.0), with the platform's
+own accelerator behind it: CoreML on macOS and iOS, DirectML on Windows,
+NNAPI on Android. The runtime is linked statically from the builds pyke
+publishes for each target.
## Effect preview photograph
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 90eb546b..f16caf51 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -140,7 +140,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher",
- "cpufeatures 0.2.17",
+ "cpufeatures",
]
[[package]]
@@ -359,12 +359,6 @@ version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
-[[package]]
-name = "anymap3"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9"
-
[[package]]
name = "arbitrary"
version = "1.4.2"
@@ -662,6 +656,18 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
+[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
[[package]]
name = "bincode"
version = "2.0.1"
@@ -698,16 +704,7 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd"
dependencies = [
- "bit-vec 0.9.1",
-]
-
-[[package]]
-name = "bit-set"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c"
-dependencies = [
- "bit-vec 0.10.1",
+ "bit-vec",
]
[[package]]
@@ -716,16 +713,6 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
-[[package]]
-name = "bit-vec"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269"
-dependencies = [
- "borsh",
- "serde",
-]
-
[[package]]
name = "bit_field"
version = "0.10.3"
@@ -802,24 +789,10 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
dependencies = [
- "borsh-derive",
"bytes",
"cfg_aliases",
]
-[[package]]
-name = "borsh-derive"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
-dependencies = [
- "once_cell",
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn 3.0.4",
-]
-
[[package]]
name = "bstr"
version = "1.13.1"
@@ -1024,17 +997,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "chacha20"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
-dependencies = [
- "cfg-if",
- "cpufeatures 0.3.1",
- "rand_core 0.10.1",
-]
-
[[package]]
name = "chrono"
version = "0.4.45"
@@ -1310,7 +1272,7 @@ dependencies = [
"serde_json",
"sherpa-onnx",
"tar",
- "ureq",
+ "ureq 2.12.1",
"whisper-rs",
]
@@ -1330,8 +1292,8 @@ version = "0.2.1"
dependencies = [
"concat-core",
"concat-project",
+ "ort",
"png 0.17.16",
- "tract-onnx",
]
[[package]]
@@ -1403,6 +1365,16 @@ dependencies = [
"libc",
]
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
@@ -1416,9 +1388,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081"
dependencies = [
"bitflags 1.3.2",
- "core-foundation",
+ "core-foundation 0.9.4",
"core-graphics-types",
- "foreign-types",
+ "foreign-types 0.5.0",
"libc",
]
@@ -1429,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [
"bitflags 1.3.2",
- "core-foundation",
+ "core-foundation 0.9.4",
"libc",
]
@@ -1497,15 +1469,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "cpufeatures"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "crc"
version = "3.4.0"
@@ -1620,21 +1583,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
[[package]]
-name = "deranged"
-version = "0.5.8"
+name = "der"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
+dependencies = [
+ "pem-rfc7468",
+ "zeroize",
+]
[[package]]
-name = "derive-new"
-version = "0.7.0"
+name = "deranged"
+version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
[[package]]
name = "derive_arbitrary"
@@ -1745,12 +1707,6 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
-[[package]]
-name = "downcast-rs"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
-
[[package]]
name = "dpi"
version = "0.1.2"
@@ -1803,24 +1759,6 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b"
-[[package]]
-name = "dyn-clone"
-version = "1.0.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
-
-[[package]]
-name = "dyn-eq"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c2d035d21af5cde1a6f5c7b444a5bf963520a9f142e5d06931178433d7d5388"
-
-[[package]]
-name = "dyn-hash"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fdab65db9274e0168143841eb8f864a0a21f8b1b8d2ba6812bbe6024346e99e"
-
[[package]]
name = "either"
version = "1.18.0"
@@ -1890,17 +1828,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-[[package]]
-name = "erased-serde"
-version = "0.4.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec"
-dependencies = [
- "serde",
- "serde_core",
- "typeid",
-]
-
[[package]]
name = "errno"
version = "0.3.14"
@@ -2084,12 +2011,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
-[[package]]
-name = "float-ord"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d"
-
[[package]]
name = "fnv"
version = "1.0.7"
@@ -2171,6 +2092,15 @@ dependencies = [
"yeslogic-fontconfig-sys",
]
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared 0.1.1",
+]
+
[[package]]
name = "foreign-types"
version = "0.5.0"
@@ -2178,7 +2108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
- "foreign-types-shared",
+ "foreign-types-shared 0.3.1",
]
[[package]]
@@ -2192,6 +2122,12 @@ dependencies = [
"syn 3.0.4",
]
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
@@ -2399,7 +2335,6 @@ dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
- "rand_core 0.10.1",
]
[[package]]
@@ -2595,8 +2530,6 @@ dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
- "serde",
- "serde_core",
]
[[package]]
@@ -2647,12 +2580,34 @@ dependencies = [
"digest",
]
+[[package]]
+name = "hmac-sha256"
+version = "1.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f"
+
[[package]]
name = "htmlparser"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48ce8546b993eaf241d69ded33b1be6d205dd9857ec879d9d18bd05d3676e144"
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
[[package]]
name = "i-slint-backend-android-activity"
version = "1.17.1"
@@ -3292,15 +3247,6 @@ dependencies = [
"syn 2.0.119",
]
-[[package]]
-name = "inventory"
-version = "0.3.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
-dependencies = [
- "rustversion",
-]
-
[[package]]
name = "io-lifetimes"
version = "1.0.11"
@@ -3336,15 +3282,6 @@ dependencies = [
"either",
]
-[[package]]
-name = "itertools"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
-dependencies = [
- "either",
-]
-
[[package]]
name = "itoa"
version = "1.0.18"
@@ -3702,6 +3639,12 @@ dependencies = [
"crc",
]
+[[package]]
+name = "lzma-rust2"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619"
+
[[package]]
name = "lzma-sys"
version = "0.1.20"
@@ -3722,12 +3665,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "maplit"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
-
[[package]]
name = "matrixmultiply"
version = "0.3.11"
@@ -3763,12 +3700,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "memo-map"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5449c8c750f1a07ea702bbd212bd999fceece9b3d1508b17023b3e174583124b"
-
[[package]]
name = "memoffset"
version = "0.9.1"
@@ -3778,16 +3709,6 @@ dependencies = [
"autocfg",
]
-[[package]]
-name = "minijinja"
-version = "2.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86886cf6dbf4e614b19c9a1eec9775f021869d7eadde0fc73921a81b90c9b4c9"
-dependencies = [
- "memo-map",
- "serde",
-]
-
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -3850,7 +3771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df"
dependencies = [
"arrayvec",
- "bit-set 0.9.1",
+ "bit-set",
"bitflags 2.13.1",
"cfg-if",
"cfg_aliases",
@@ -3869,6 +3790,23 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "native-tls"
+version = "0.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
[[package]]
name = "natord"
version = "1.0.9"
@@ -3989,15 +3927,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "nom-language"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29"
-dependencies = [
- "nom 8.0.0",
-]
-
[[package]]
name = "noop_proc_macro"
version = "0.3.0"
@@ -4544,6 +4473,49 @@ dependencies = [
"zbus",
]
+[[package]]
+name = "openssl"
+version = "0.10.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
+dependencies = [
+ "bitflags 2.13.1",
+ "cfg-if",
+ "foreign-types 0.3.2",
+ "libc",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "orbclient"
version = "0.3.55"
@@ -4573,6 +4545,30 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "ort"
+version = "2.0.0-rc.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b"
+dependencies = [
+ "ndarray",
+ "ort-sys",
+ "smallvec",
+ "tracing",
+ "ureq 3.4.1",
+]
+
+[[package]]
+name = "ort-sys"
+version = "2.0.0-rc.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872"
+dependencies = [
+ "hmac-sha256",
+ "lzma-rust2",
+ "ureq 3.4.1",
+]
+
[[package]]
name = "owned_ttf_parser"
version = "0.25.1"
@@ -4672,6 +4668,15 @@ dependencies = [
"hmac",
]
+[[package]]
+name = "pem-rfc7468"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
+dependencies = [
+ "base64ct",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -4903,15 +4908,6 @@ dependencies = [
"syn 2.0.119",
]
-[[package]]
-name = "primal-check"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
-dependencies = [
- "num-integer",
-]
-
[[package]]
name = "proc-macro-crate"
version = "3.5.0"
@@ -4949,29 +4945,6 @@ dependencies = [
"syn 2.0.119",
]
-[[package]]
-name = "prost"
-version = "0.14.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.14.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
-dependencies = [
- "anyhow",
- "itertools 0.14.0",
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
[[package]]
name = "pulldown-cmark"
version = "0.13.4"
@@ -5072,18 +5045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha",
- "rand_core 0.9.5",
-]
-
-[[package]]
-name = "rand"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
-dependencies = [
- "chacha20",
- "getrandom 0.4.3",
- "rand_core 0.10.1",
+ "rand_core",
]
[[package]]
@@ -5093,7 +5055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core 0.9.5",
+ "rand_core",
]
[[package]]
@@ -5105,22 +5067,6 @@ dependencies = [
"getrandom 0.3.4",
]
-[[package]]
-name = "rand_core"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
-
-[[package]]
-name = "rand_distr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8"
-dependencies = [
- "num-traits",
- "rand 0.10.2",
-]
-
[[package]]
name = "range-alloc"
version = "0.1.5"
@@ -5154,7 +5100,7 @@ dependencies = [
"num-traits",
"paste",
"profiling",
- "rand 0.9.5",
+ "rand",
"rand_chacha",
"simd_helpers",
"thiserror 2.0.20",
@@ -5447,20 +5393,6 @@ dependencies = [
"semver",
]
-[[package]]
-name = "rustfft"
-version = "6.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
-dependencies = [
- "num-complex",
- "num-integer",
- "num-traits",
- "primal-check",
- "strength_reduce",
- "transpose",
-]
-
[[package]]
name = "rustix"
version = "0.38.44"
@@ -5546,19 +5478,6 @@ dependencies = [
"unicode-script",
]
-[[package]]
-name = "safetensors"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79b079b829cb27a1c3c374341345ed2e8b2c0c839034522cee576c140bd7f846"
-dependencies = [
- "hashbrown 0.16.1",
- "libc",
- "serde",
- "serde_json",
- "tempfile",
-]
-
[[package]]
name = "same-file"
version = "1.0.6"
@@ -5569,12 +5488,12 @@ dependencies = [
]
[[package]]
-name = "scan_fmt"
-version = "0.2.6"
+name = "schannel"
+version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b53b0a5db882a8e2fdaae0a43f7b39e7e9082389e978398bdf223a55b581248"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
dependencies = [
- "regex",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -5608,6 +5527,29 @@ dependencies = [
"tiny-skia 0.11.4",
]
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags 2.13.1",
+ "core-foundation 0.10.1",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "semver"
version = "1.0.28"
@@ -5684,7 +5626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
- "cpufeatures 0.2.17",
+ "cpufeatures",
"digest",
]
@@ -5707,7 +5649,7 @@ checksum = "566aad07d0924a7ed897cc035cc67ad71cdfddb959717dbf445031d5d4b58d4d"
dependencies = [
"bzip2 0.4.4",
"tar",
- "ureq",
+ "ureq 2.12.1",
"zip",
]
@@ -6001,6 +5943,17 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "socks"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
+dependencies = [
+ "byteorder",
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "softbuffer"
version = "0.4.8"
@@ -6063,12 +6016,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-[[package]]
-name = "strength_reduce"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
-
[[package]]
name = "strict-num"
version = "0.1.1"
@@ -6078,16 +6025,6 @@ dependencies = [
"float-cmp",
]
-[[package]]
-name = "string-interner"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15"
-dependencies = [
- "hashbrown 0.16.1",
- "serde",
-]
-
[[package]]
name = "strsim"
version = "0.11.1"
@@ -6494,223 +6431,6 @@ dependencies = [
"once_cell",
]
-[[package]]
-name = "tract-core"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49647af4b6a0c5522c0c9b8120ce517146f2a7fd67797a7c6ab14fe91f01c5d5"
-dependencies = [
- "anyhow",
- "anymap3",
- "bit-set 0.11.1",
- "derive-new",
- "downcast-rs 2.0.2",
- "dyn-clone",
- "dyn-eq",
- "erased-serde",
- "inventory",
- "lazy_static",
- "log",
- "maplit",
- "ndarray",
- "num-complex",
- "num-integer",
- "num-traits",
- "pastey 0.2.3",
- "rustfft",
- "serde",
- "smallvec",
- "tract-data",
- "tract-linalg",
-]
-
-[[package]]
-name = "tract-data"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33bb72b6667b6a4a698a263044aec741dc2887425ada192874f7b80b8fb3f85c"
-dependencies = [
- "anyhow",
- "downcast-rs 2.0.2",
- "dyn-clone",
- "dyn-eq",
- "dyn-hash",
- "half",
- "inventory",
- "itertools 0.15.0",
- "lazy_static",
- "libm",
- "maplit",
- "ndarray",
- "nom 8.0.0",
- "nom-language",
- "num-integer",
- "num-traits",
- "parking_lot",
- "scan_fmt",
- "smallvec",
- "string-interner",
-]
-
-[[package]]
-name = "tract-extra"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdd571adedfa2108532688c12e94b95640c87793bdaffd3daef238fed5a071ad"
-dependencies = [
- "tract-nnef",
- "tract-pulse",
-]
-
-[[package]]
-name = "tract-hir"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fd20ade88f142bf0419d899c3ce56fe00ee7d208794d5eb9c86059649b89c"
-dependencies = [
- "derive-new",
- "log",
- "tract-core",
-]
-
-[[package]]
-name = "tract-linalg"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4aa27179d6809f4f5797ed2d0df8d0a79588c5ec7a78a68eb98c3aec85fa9f4"
-dependencies = [
- "byteorder",
- "cc",
- "derive-new",
- "downcast-rs 2.0.2",
- "dyn-clone",
- "dyn-eq",
- "dyn-hash",
- "half",
- "inventory",
- "lazy_static",
- "libc",
- "log",
- "minijinja",
- "num-traits",
- "pastey 0.2.3",
- "scan_fmt",
- "tract-data",
- "walkdir",
-]
-
-[[package]]
-name = "tract-nnef"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e746cd34227b14280dbd2ac176400224f6407ea304b6c16f87b5be757ce73b4"
-dependencies = [
- "byteorder",
- "erased-serde",
- "flate2",
- "log",
- "minijinja",
- "nom 8.0.0",
- "nom-language",
- "safetensors",
- "serde",
- "serde_json",
- "simd-adler32",
- "tar",
- "tract-core",
- "walkdir",
-]
-
-[[package]]
-name = "tract-onnx"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320e35b04af77879b05921ab0509a44b77f52fcd452e92162e8052f5d1f4a762"
-dependencies = [
- "bytes",
- "derive-new",
- "dyn-eq",
- "log",
- "memmap2",
- "nom 8.0.0",
- "nom-language",
- "num-integer",
- "num-rational",
- "num-traits",
- "prost",
- "smallvec",
- "tract-extra",
- "tract-hir",
- "tract-nnef",
- "tract-onnx-opl",
- "tract-transformers",
-]
-
-[[package]]
-name = "tract-onnx-opl"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e723cf0756822155aff3f4046cb85c62ccd8fffd0ea39290b1640b67ddbf9f"
-dependencies = [
- "dyn-eq",
- "getrandom 0.4.3",
- "log",
- "rand 0.10.2",
- "rand_distr",
- "rustfft",
- "tract-extra",
- "tract-nnef",
-]
-
-[[package]]
-name = "tract-pulse"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7af0b850fc74a39da37f6ace584cd54dce773be9b7ec71f45f4011cb0125db3c"
-dependencies = [
- "downcast-rs 2.0.2",
- "dyn-eq",
- "erased-serde",
- "lazy_static",
- "log",
- "serde",
- "tract-pulse-opl",
- "tract-transformers",
-]
-
-[[package]]
-name = "tract-pulse-opl"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0629fb3c289fb906d95a2aa1a71b4389511e8620f8bde3c711073e393414729"
-dependencies = [
- "downcast-rs 2.0.2",
- "dyn-eq",
- "lazy_static",
- "tract-nnef",
-]
-
-[[package]]
-name = "tract-transformers"
-version = "0.23.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a075c3a29baf4a859c67a69fd4643c43ccda51e9644d74af9481937a7e57f710"
-dependencies = [
- "float-ord",
- "rayon",
- "tract-nnef",
-]
-
-[[package]]
-name = "transpose"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
-dependencies = [
- "num-integer",
- "strength_reduce",
-]
-
[[package]]
name = "ttf-parser"
version = "0.25.1"
@@ -6730,12 +6450,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "typeid"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
-
[[package]]
name = "typenum"
version = "1.20.1"
@@ -6855,7 +6569,7 @@ version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
dependencies = [
- "base64",
+ "base64 0.22.1",
"flate2",
"log",
"once_cell",
@@ -6865,6 +6579,36 @@ dependencies = [
"webpki-roots 0.26.11",
]
+[[package]]
+name = "ureq"
+version = "3.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af5546be8f5378d5414f83733f5c9a2526f4645829edbc1c41790aeef1b38e8b"
+dependencies = [
+ "base64 0.23.1",
+ "der",
+ "log",
+ "native-tls",
+ "percent-encoding",
+ "rustls-pki-types",
+ "socks",
+ "ureq-proto",
+ "utf8-zero",
+ "webpki-root-certs",
+]
+
+[[package]]
+name = "ureq-proto"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fabc3e92916c89c95b20eef7b06b00b066bc217ef9ea3a4ac9bf1a7e35261e10"
+dependencies = [
+ "base64 0.23.1",
+ "http",
+ "httparse",
+ "log",
+]
+
[[package]]
name = "url"
version = "2.5.8"
@@ -6884,7 +6628,7 @@ version = "0.47.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d46cf96c5f498d36b7a9693bc6a7075c0bb9303189d61b2249b0dc3d309c07de"
dependencies = [
- "base64",
+ "base64 0.22.1",
"data-url",
"flate2",
"fontdb",
@@ -6906,6 +6650,12 @@ dependencies = [
"xmlwriter",
]
+[[package]]
+name = "utf8-zero"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
+
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -7062,7 +6812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078"
dependencies = [
"cc",
- "downcast-rs 1.2.1",
+ "downcast-rs",
"rustix 1.1.4",
"scoped-tls",
"smallvec",
@@ -7226,6 +6976,15 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "webpki-roots"
version = "0.26.11"
@@ -7287,8 +7046,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7"
dependencies = [
"arrayvec",
- "bit-set 0.9.1",
- "bit-vec 0.9.1",
+ "bit-set",
+ "bit-vec",
"bitflags 2.13.1",
"bytemuck",
"cfg_aliases",
@@ -7349,7 +7108,7 @@ dependencies = [
"android_system_properties",
"arrayvec",
"ash",
- "bit-set 0.9.1",
+ "bit-set",
"bitflags 2.13.1",
"block2 0.6.2",
"bytemuck",
@@ -7440,6 +7199,22 @@ dependencies = [
"semver",
]
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
[[package]]
name = "winapi-util"
version = "0.1.11"
@@ -7449,6 +7224,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
[[package]]
name = "windows"
version = "0.54.0"
@@ -7826,7 +7607,7 @@ dependencies = [
"calloop 0.13.0",
"cfg_aliases",
"concurrent-queue",
- "core-foundation",
+ "core-foundation 0.9.4",
"core-graphics",
"cursor-icon",
"dpi",
diff --git a/engine/Cargo.toml b/engine/Cargo.toml
index e8c62d8d..d1f2da92 100644
--- a/engine/Cargo.toml
+++ b/engine/Cargo.toml
@@ -33,6 +33,11 @@ clap = { version = "4", features = ["derive"] }
wgpu = "29"
pollster = "0.4"
+# ONNX Runtime, for the cutout models. The binaries come prebuilt for every
+# target the app ships on, with the platform's own accelerator behind them:
+# CoreML on macOS and iOS, DirectML on Windows, NNAPI on Android.
+ort = "=2.0.0-rc.13"
+
[workspace.lints.rust]
missing_docs = "warn"
diff --git a/engine/crates/concat-vision/Cargo.toml b/engine/crates/concat-vision/Cargo.toml
index 55c257aa..cdaae5d5 100644
--- a/engine/crates/concat-vision/Cargo.toml
+++ b/engine/crates/concat-vision/Cargo.toml
@@ -11,7 +11,7 @@ description = "What the engine sees in a picture: the person mask behind a cutou
# them is inference, and inference is a dependency tree the export path
# should not carry, so it sits behind `infer` and the host turns it on.
[features]
-infer = ["dep:tract-onnx"]
+infer = ["dep:ort"]
[dependencies]
concat-core.workspace = true
@@ -19,9 +19,18 @@ concat-project.workspace = true
# Masks on disk are eight-bit grey PNGs: a tenth the size of raw bytes, and
# a file anyone can open to see what the model saw.
png = "0.17"
-# ONNX inference in pure Rust. No runtime library to ship, the same
-# binary on every platform, and the model travels inside the crate.
-tract-onnx = { version = "0.23", optional = true }
+# ONNX Runtime. The platform's accelerator is a feature per target below;
+# the CPU provider is always there behind it.
+ort = { workspace = true, optional = true }
+
+[target.'cfg(target_vendor = "apple")'.dependencies]
+ort = { workspace = true, optional = true, features = ["coreml"] }
+
+[target.'cfg(target_os = "windows")'.dependencies]
+ort = { workspace = true, optional = true, features = ["directml"] }
+
+[target.'cfg(target_os = "android")'.dependencies]
+ort = { workspace = true, optional = true, features = ["nnapi"] }
[lints]
workspace = true
diff --git a/engine/crates/concat-vision/src/lib.rs b/engine/crates/concat-vision/src/lib.rs
index 8639744e..9da8abfa 100644
--- a/engine/crates/concat-vision/src/lib.rs
+++ b/engine/crates/concat-vision/src/lib.rs
@@ -20,9 +20,10 @@
//! its waveforms, so a cutout is found once and travels with the edit.
//!
//! Finding the mask is [`segment`], behind the `infer` feature: the model
-//! is compiled in and run by tract, in pure Rust, so a cutout needs no
-//! download and no runtime library, on a desk or a phone. The renderer
-//! reads masks and never infers; the host infers and writes them.
+//! is compiled in and run by ONNX Runtime through [`runtime`], on the
+//! platform's accelerator where there is one, so a cutout needs no
+//! download, on a desk or a phone. The renderer reads masks and never
+//! infers; the host infers and writes them.
//!
//! Masks are square at the model's resolution whatever the picture's
//! shape, and every position in them is a fraction of the source picture:
@@ -35,6 +36,8 @@
pub mod apply;
pub mod mask;
#[cfg(feature = "infer")]
+pub mod runtime;
+#[cfg(feature = "infer")]
pub mod segment;
pub mod store;
pub mod strokes;
diff --git a/engine/crates/concat-vision/src/runtime.rs b/engine/crates/concat-vision/src/runtime.rs
new file mode 100644
index 00000000..128dd092
--- /dev/null
+++ b/engine/crates/concat-vision/src/runtime.rs
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! One ONNX model, loaded and ready to run.
+//!
+//! A thin coat over ONNX Runtime: a session with the platform's
+//! accelerator registered ahead of the CPU - CoreML on Apple, DirectML on
+//! Windows, NNAPI on Android - and one call that takes named `f32`
+//! tensors and gives named `f32` tensors back. Every model this crate
+//! runs is a picture in and a picture out, so that is all the surface it
+//! needs; what each model's tensors mean lives beside the model.
+
+use ort::session::Session;
+use ort::value::Tensor;
+
+/// An `f32` tensor by name: its dimensions and its values, row-major.
+pub struct Input<'a> {
+ /// The graph's input name.
+ pub name: &'a str,
+ /// The tensor's shape.
+ pub dims: Vec<usize>,
+ /// The values, `dims` product of them.
+ pub data: Vec<f32>,
+}
+
+/// An `f32` tensor a model answered with.
+#[derive(Clone, Debug)]
+pub struct Output {
+ /// The tensor's shape.
+ pub dims: Vec<usize>,
+ /// The values, row-major.
+ pub data: Vec<f32>,
+}
+
+/// A loaded model.
+pub struct Model {
+ session: Session,
+}
+
+impl Model {
+ /// Loads a model from its bytes.
+ pub fn from_bytes(bytes: &[u8]) -> Result<Model, String> {
+ let mut builder = Session::builder().map_err(|error| format!("onnx runtime: {error}"))?;
+ builder = builder
+ .with_execution_providers(accelerators())
+ .map_err(|error| format!("onnx runtime: {error}"))?;
+ // Every core on one frame at a time: the models are convolutional
+ // and a frame fills a machine on its own.
+ let threads = std::thread::available_parallelism()
+ .map(|count| count.get())
+ .unwrap_or(1)
+ .clamp(1, 8);
+ builder = builder
+ .with_intra_threads(threads)
+ .map_err(|error| format!("onnx runtime: {error}"))?;
+ let session = builder
+ .commit_from_memory(bytes)
+ .map_err(|error| format!("cutout model: {error}"))?;
+ Ok(Model { session })
+ }
+
+ /// Loads a model from a file.
+ pub fn from_file(path: &std::path::Path) -> Result<Model, String> {
+ let bytes = std::fs::read(path)
+ .map_err(|error| format!("could not read {}: {error}", path.display()))?;
+ Model::from_bytes(&bytes)
+ }
+
+ /// Runs the model on `inputs` and returns the named outputs, in the
+ /// order asked for.
+ pub fn run(&mut self, inputs: Vec<Input<'_>>, outputs: &[&str]) -> Result<Vec<Output>, String> {
+ let mut fed: Vec<(std::borrow::Cow<'static, str>, ort::session::SessionInputValue<'_>)> =
+ Vec::with_capacity(inputs.len());
+ for input in inputs {
+ let tensor = Tensor::from_array((input.dims, input.data))
+ .map_err(|error| format!("cutout input {}: {error}", input.name))?;
+ fed.push((input.name.to_owned().into(), tensor.into()));
+ }
+ let answered = self
+ .session
+ .run(fed)
+ .map_err(|error| format!("cutout: {error}"))?;
+ outputs
+ .iter()
+ .map(|name| {
+ let value = answered
+ .get(*name)
+ .ok_or_else(|| format!("cutout: the model has no output {name:?}"))?;
+ let (shape, data) = value
+ .try_extract_tensor::<f32>()
+ .map_err(|error| format!("cutout output {name}: {error}"))?;
+ Ok(Output {
+ dims: shape.iter().map(|&d| d.max(0) as usize).collect(),
+ data: data.to_vec(),
+ })
+ })
+ .collect()
+ }
+}
+
+/// The platform's accelerators, ahead of the CPU. A provider the machine
+/// cannot supply is skipped, not an error, so the same build runs on a
+/// box without one.
+fn accelerators() -> Vec<ort::ep::ExecutionProviderDispatch> {
+ let mut providers = Vec::new();
+ #[cfg(target_vendor = "apple")]
+ providers.push(ort::ep::CoreML::default().build());
+ #[cfg(target_os = "windows")]
+ providers.push(ort::ep::DirectML::default().build());
+ #[cfg(target_os = "android")]
+ providers.push(ort::ep::NNAPI::default().build());
+ providers.push(ort::ep::CPU::default().build());
+ providers
+}
diff --git a/engine/crates/concat-vision/src/segment.rs b/engine/crates/concat-vision/src/segment.rs
index 95490db0..e4dab2f6 100644
--- a/engine/crates/concat-vision/src/segment.rs
+++ b/engine/crates/concat-vision/src/segment.rs
@@ -4,52 +4,37 @@
//! Finding the person in a picture.
//!
//! The model is MediaPipe's selfie segmentation, compiled into the binary
-//! (see `models/NOTICE.md`) and run by tract. It takes a 256 × 256 RGB
-//! picture and answers with a probability per pixel; a source of any shape
-//! is squashed to the square on the way in and the mask is read as covering
-//! the whole source on the way out, which is the convention every other
-//! part of this crate keeps.
+//! (see `models/NOTICE.md`) and run by ONNX Runtime. It takes a 256 × 256
+//! RGB picture and answers with a probability per pixel; a source of any
+//! shape is squashed to the square on the way in and the mask is read as
+//! covering the whole source on the way out, which is the convention every
+//! other part of this crate keeps.
//!
-//! One inference is about twenty-five milliseconds on a laptop core, and a
-//! loaded [`Segmenter`] is shared between threads, so a host that has
-//! several cores analyses several frames at once against one model.
+//! One inference is a few milliseconds with the platform's accelerator and
+//! about twenty on a laptop's cores without.
+
+use std::sync::Mutex;
use concat_core::frame::{BYTES_PER_PIXEL, Frame};
-use tract_onnx::prelude::*;
+use crate::runtime::{Input, Model};
use crate::{MODEL_SIZE, Mask};
/// The model as the crate ships it.
const MODEL: &[u8] = include_bytes!("../models/selfie-segmentation.onnx");
-type Plan = std::sync::Arc<TypedRunnableModel>;
-
-/// A loaded model, ready to run. `Send + Sync`: one per process is enough.
+/// A loaded model, ready to run. `Send + Sync`: one per process is enough,
+/// and the runtime's own threads take a frame across the cores.
pub struct Segmenter {
- plan: Plan,
+ model: Mutex<Model>,
}
impl Segmenter {
- /// Loads and optimises the compiled-in model. A few tens of
- /// milliseconds, once.
+ /// Loads the compiled-in model. A few tens of milliseconds, once.
pub fn load() -> Result<Segmenter, String> {
- let onnx = tract_onnx::onnx();
- let mut proto = onnx
- .proto_model_for_read(&mut std::io::Cursor::new(MODEL))
- .map_err(|error| format!("cutout model: {error}"))?;
- // The conversion annotates every tensor with a symbolic batch, and
- // the batch here is always one; the annotations would only argue.
- if let Some(graph) = proto.graph.as_mut() {
- graph.value_info.clear();
- }
- let size = MODEL_SIZE as usize;
- let plan = onnx
- .model_for_proto_model(&proto)
- .and_then(|model| model.with_input_fact(0, f32::fact([1, 3, size, size]).into()))
- .and_then(|model| model.into_optimized())
- .and_then(|model| model.into_runnable())
- .map_err(|error| format!("cutout model: {error}"))?;
- Ok(Segmenter { plan })
+ Ok(Segmenter {
+ model: Mutex::new(Model::from_bytes(MODEL)?),
+ })
}
/// The person mask of `frame`, at the model's resolution. A frame of
@@ -62,23 +47,31 @@ impl Segmenter {
return Err("cutout: an empty frame".to_owned());
}
let pixels = frame.pixels();
- let input = tract_ndarray::Array4::from_shape_fn((1, 3, size, size), |(_, c, y, x)| {
- // Nearest sample: the decoder already scaled to the square in
- // the analysis path, and a still is fine at this resolution.
- let sx = (x * width / size).min(width - 1);
- let sy = (y * height / size).min(height - 1);
- f32::from(pixels[(sy * width + sx) * BYTES_PER_PIXEL + c]) / 255.0
- });
+ let mut data = vec![0f32; 3 * size * size];
+ for c in 0..3 {
+ for y in 0..size {
+ // Nearest sample: the decoder already scaled to the square
+ // in the analysis path, and a still is fine at this size.
+ let sy = (y * height / size).min(height - 1);
+ for x in 0..size {
+ let sx = (x * width / size).min(width - 1);
+ data[(c * size + y) * size + x] =
+ f32::from(pixels[(sy * width + sx) * BYTES_PER_PIXEL + c]) / 255.0;
+ }
+ }
+ }
+ let input = Input {
+ name: "pixel_values",
+ dims: vec![1, 3, size, size],
+ data,
+ };
let outputs = self
- .plan
- .run(tvec!(input.into_tensor().into()))
- .map_err(|error| format!("cutout: {error}"))?;
- let view = outputs
- .first()
- .ok_or_else(|| "cutout: the model answered nothing".to_owned())?
- .to_plain_array_view::<f32>()
- .map_err(|error| format!("cutout: {error}"))?;
- let data: Vec<u8> = view
+ .model
+ .lock()
+ .map_err(|_| "cutout model poisoned".to_owned())?
+ .run(vec![input], &["alphas"])?;
+ let data: Vec<u8> = outputs[0]
+ .data
.iter()
.take(size * size)
.map(|&p: &f32| (p.clamp(0.0, 1.0) * 255.0 + 0.5) as u8)
diff --git a/flake.nix b/flake.nix
index e040a5ba..452732a9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -19,6 +19,9 @@
# - sherpa-onnx (text to speech) links prebuilt static libraries that its
# sys crate would download at build time. They are fetched here as fixed-
# output derivations instead and handed over with SHERPA_ONNX_ARCHIVE_DIR.
+ # - ONNX Runtime (the cutout models) is likewise downloaded by its sys
+ # crate. nixpkgs' onnxruntime is linked instead, through ORT_LIB_LOCATION,
+ # dynamically, so the wrapper's rpath finds it.
#
# The window is built with the FemtoVG-over-wgpu renderer (`--features
# wgpu`) rather than Skia: skia-bindings also downloads its binaries at
@@ -91,6 +94,8 @@
# 8, by name: the engine needs 7 or newer, and nixpkgs' unversioned
# `ffmpeg` is whichever major the distribution defaults to.
ffmpeg_8
+ # The cutout models' runtime; see ORT_LIB_LOCATION above.
+ onnxruntime
alsa-lib
fontconfig
freetype
@@ -124,6 +129,8 @@
doCheck = false;
env.SHERPA_ONNX_ARCHIVE_DIR = sherpaArchiveDir pkgs;
+ env.ORT_LIB_LOCATION = "${pkgs.onnxruntime}/lib";
+ env.ORT_PREFER_DYNAMIC_LINK = "1";
nativeBuildInputs = (nativeInputs pkgs) ++ [
pkgs.wrapGAppsHook3
@@ -180,6 +187,8 @@
]);
env.SHERPA_ONNX_ARCHIVE_DIR = sherpaArchiveDir pkgs;
+ env.ORT_LIB_LOCATION = "${pkgs.onnxruntime}/lib";
+ env.ORT_PREFER_DYNAMIC_LINK = "1";
LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath (runtimeLibs pkgs);
shellHook = ''