Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

perf(captions): transcription runs AVX2 kernels again on CPUs that have them, and still starts on a Sandy Bridge (#388)

Jareer1eb9a33007ea
diff --git a/.github/workflows/build-app.yml b/.github/workflows/build-app.yml
index a0b7e778..b3aa89df 100644
--- a/.github/workflows/build-app.yml
+++ b/.github/workflows/build-app.yml
@@ -100,20 +100,11 @@ env:
# ggml (whisper.cpp's kernels) tunes itself to the machine it is built
# on unless told otherwise, and these bundles run on other machines: a
# runner with AVX-512 or SVE would leave a binary that faults on a CPU
- # without them. Off, ggml does not probe the runner - but its own
- # instruction-set switches still default to on, which on x86_64 means
- # AVX2 with FMA, F16C and BMI2: the whole CPU backend is compiled with
- # /arch:AVX2 (or -mavx2) and the editor dies at launch with an illegal
- # instruction on a Sandy Bridge or Ivy Bridge machine, which has AVX
- # and nothing newer (#193). The baseline is AVX: every x86_64 CPU since
- # 2011 runs it, and whisper on the CPU is somewhat slower for the rest.
- # NEON on ARM, where these switches are not read. whisper-rs-sys
- # forwards every GGML_* variable to CMake as a define.
+ # without them. Off, ggml does not probe the runner. whisper-rs-sys
+ # forwards every GGML_* variable to CMake as a define. The instruction
+ # sets each target's kernels are built for are the job's, by its `ggml`
+ # row in the matrix; see the job's `env`.
GGML_NATIVE: "OFF"
- GGML_AVX2: "OFF"
- GGML_FMA: "OFF"
- GGML_F16C: "OFF"
- GGML_BMI2: "OFF"
jobs:
app:
@@ -134,6 +125,10 @@ jobs:
os: macos-15-intel
target: x86_64-apple-darwin
sherpa: osx-x64
+ # No `ggml` row: built as 0.2.7 was, with no AVX2. This bundle
+ # runs on macOS 12 again, and so on the 2013 Mac Pro, whose Ivy
+ # Bridge has AVX and nothing newer; Intel Macs are about one
+ # download in a hundred, and slower captions there beat a crash.
# 22.04, not latest: the binary links the build machine's glibc,
# so the oldest supported runner is what keeps it runnable on
# older distributions.
@@ -146,6 +141,7 @@ jobs:
ort: x64
nfpm: amd64
appimage: x86_64
+ ggml: variants
- platform: linux
arch: aarch64
os: ubuntu-22.04-arm
@@ -166,6 +162,7 @@ jobs:
vc: x64
inno: x64compatible
wix: x64
+ ggml: variants
# No Windows on ARM until its program starts: v0.2.7-rc.4's
# concat.exe died with an access violation on `--version` (#326).
# Its row was platform windows, arch aarch64, os windows-11-arm,
@@ -177,6 +174,42 @@ jobs:
# at most 155 minutes between them on any one target, so a stuck step
# is named by its own timeout; this is for a job stuck anywhere else.
timeout-minutes: 190
+ env:
+ # What whisper's CPU kernels are built for on x86_64, by the matrix's
+ # `ggml` row. ggml's own switches default to AVX2 with FMA, F16C and
+ # BMI2, compiled into the whole CPU backend - and the editor died at
+ # launch with an illegal instruction on an Ivy Bridge, which has AVX
+ # and nothing newer (#193). 0.2.6 and 0.2.7 built with all four off,
+ # so those machines start, and every other one transcribed some
+ # eight times slower: 324 s for 43 s of sound on a Ryzen 5 2600, 40 s
+ # with AVX2 (#388). Two answers, by target:
+ #
+ # - `variants` (Windows and Linux x86_64): whisper and ggml are built
+ # as shared libraries and the CPU backend as one module per x86_64
+ # generation - ggml-cpu-x64, -sse42, -sandybridge, -haswell,
+ # -skylakex, -icelake, -alderlake and more - shipped beside the
+ # program. At the first transcription ggml scores each against the
+ # CPU's CPUID and loads the best it can run (concat-speech's
+ # `cpu_backend`), so a Haswell or a Ryzen gets AVX2 and FMA, and a
+ # Sandy Bridge the AVX kernels it had. The modules are what the
+ # whisper.cpp and llama.cpp projects ship their own x86 builds as.
+ # The published whisper-rs-sys can only build static libraries;
+ # src/patches/whisper-rs-sys adds the shared build, chosen by
+ # GGML_BACKEND_DL. Nothing else in the bundle may hold AVX2: the
+ # instruction-set scan allows it in the Haswell-and-newer modules
+ # alone (scripts/ci/isa-allow.txt).
+ # - none (macOS x86_64): all four off, as before. It declares macOS 12,
+ # which the 2013 Mac Pro runs, and that is an Ivy Bridge.
+ #
+ # Everything else is ARM, which reads none of these. The `haswell`
+ # answer - built for Haswell outright - is kept for a target that
+ # only runs on AVX2 machines.
+ GGML_BACKEND_DL: ${{ matrix.ggml == 'variants' && 'ON' || 'OFF' }}
+ GGML_CPU_ALL_VARIANTS: ${{ matrix.ggml == 'variants' && 'ON' || 'OFF' }}
+ GGML_AVX2: ${{ matrix.ggml == 'haswell' && 'ON' || 'OFF' }}
+ GGML_FMA: ${{ matrix.ggml == 'haswell' && 'ON' || 'OFF' }}
+ GGML_F16C: ${{ matrix.ggml == 'haswell' && 'ON' || 'OFF' }}
+ GGML_BMI2: ${{ matrix.ggml == 'haswell' && 'ON' || 'OFF' }}
steps:
# The ref release.yml names, which is its tag: without one a run
# started by hand builds main and publishes it under the tag's name.
@@ -698,6 +731,24 @@ jobs:
cp "src/target/${{ matrix.target }}/app/concat" "stage/$name/concat"
cp -a ffmpeg-dev/lib/*.so* "stage/$name/lib/"
cp -a "$ORT_DIR"/lib/libonnxruntime.so* "stage/$name/lib/"
+ # whisper's shared libraries and its CPU modules, one per x86_64
+ # generation (the job's `env` says why), from where whisper-rs-sys
+ # installed them: libraries in lib/, modules in bin/. Its build
+ # also copies them beside the binary, for `cargo run`, but a build
+ # the cache satisfied does not run that script again, and the
+ # cache does not promise to keep loose files at the top of a
+ # profile. In lib/ the rpath finds the libraries and
+ # concat-speech's `cpu_backend` the modules.
+ if [ "${{ matrix.ggml }}" = variants ]; then
+ out=$(ls -dt "src/target/${{ matrix.target }}/app/build"/whisper-rs-sys-*/out/bin | head -1)
+ out=${out%/bin}
+ echo "whisper's build: $out"
+ cp -a "$out"/lib*/libwhisper.so* "$out"/lib*/libggml*.so* "$out"/bin/libggml-cpu-*.so "stage/$name/lib/"
+ for module in x64 sandybridge haswell; do
+ test -f "stage/$name/lib/libggml-cpu-$module.so" \
+ || { echo "libggml-cpu-$module.so is missing from the build" >&2; exit 1; }
+ done
+ fi
cp assets/icons/concat_logo_256.png "stage/$name/concat.png"
cat > "stage/$name/concat.desktop" <<DESKTOP
[Desktop Entry]
@@ -923,6 +974,21 @@ jobs:
# The sherpa-onnx runtime, set aside by the download step above.
Copy-Item "$env:SHERPA_DLL_DIR\*.dll" $dir
Copy-Item "$env:ORT_DIR\lib\*.dll" $dir
+ # whisper's DLLs and its CPU modules, one per x86_64 generation
+ # (the job's `env` says why), from the bin\ whisper-rs-sys
+ # installed them to; see the Linux stage for why not from beside
+ # the program. Beside concat.exe Windows finds the DLLs and
+ # concat-speech's `cpu_backend` the modules.
+ if ("${{ matrix.ggml }}" -eq "variants") {
+ $built = Get-ChildItem "src\target\${{ matrix.target }}\app\build\whisper-rs-sys-*\out\bin\ggml-cpu-x64.dll" |
+ Sort-Object LastWriteTime -Descending | Select-Object -First 1
+ if (-not $built) { throw "whisper-rs-sys installed no ggml-cpu-x64.dll" }
+ Write-Host "whisper's build: $($built.DirectoryName)"
+ Copy-Item "$($built.DirectoryName)\whisper.dll", "$($built.DirectoryName)\ggml*.dll" $dir
+ foreach ($module in "x64", "sandybridge", "haswell") {
+ if (-not (Test-Path "$dir\ggml-cpu-$module.dll")) { throw "ggml-cpu-$module.dll is missing from the build" }
+ }
+ }
Copy-Item LICENSE, THIRD_PARTY_NOTICES.md $dir
# The Visual C++ runtime DLLs anything in the folder imports, from
# the newest redistributable folder Visual Studio carries.
diff --git a/scripts/ci/isa-allow.txt b/scripts/ci/isa-allow.txt
index d0ff2fc3..6feddbc0 100644
--- a/scripts/ci/isa-allow.txt
+++ b/scripts/ci/isa-allow.txt
@@ -44,6 +44,24 @@ vcruntime140*.dll any
msvcp140*.dll any
concrt140.dll any
+# whisper's CPU kernels, one module per x86_64 generation, of which ggml
+# loads only the best one the CPU's CPUID says it can run (build-app.yml,
+# GGML_BACKEND_DL; concat-speech's `cpu_backend`). Those for Haswell and
+# newer, and AMD's Piledriver, are built for their generation as a whole
+# and never load on an older CPU. The ones an Ivy Bridge or a Sandy Bridge
+# loads - x64, sse42, sandybridge, ivybridge - have no row, so they may
+# hold none of it, and neither may whisper, ggml and ggml-base themselves.
+*ggml-cpu-piledriver.* any
+*ggml-cpu-haswell.* any
+*ggml-cpu-skylakex.* any
+*ggml-cpu-cannonlake.* any
+*ggml-cpu-cascadelake.* any
+*ggml-cpu-icelake.* any
+*ggml-cpu-cooperlake.* any
+*ggml-cpu-zen4.* any
+*ggml-cpu-alderlake.* any
+*ggml-cpu-sapphirerapids.* any
+
# Concat itself. It holds dispatched code of its own - memchr, aho-corasick
# and ring choose AVX2 or BMI2 paths at run time (ring's hand-written
# cryptography is where mulx and rorx come from), and Skia carries its
diff --git a/scripts/ci/isa-scan.sh b/scripts/ci/isa-scan.sh
index b74d2768..170b4678 100755
--- a/scripts/ci/isa-scan.sh
+++ b/scripts/ci/isa-scan.sh
@@ -25,7 +25,9 @@
# Not every one of those is a fault waiting to happen. A library that asks
# the CPU what it has at run time and picks its kernels by the answer
# carries AVX2 code it never runs on an older machine: FFmpeg's codecs and
-# scalers, ONNX Runtime's MLAS, sherpa-onnx. And a statically linked binary
+# scalers, ONNX Runtime's MLAS, sherpa-onnx. whisper's CPU kernels are one
+# module per CPU generation, of which ggml loads the one the CPU can run,
+# so the Haswell-and-newer modules are allowed what the rest are not. And a statically linked binary
# carries such code too - memchr, aho-corasick, ring and Skia all dispatch
# this way - so a count of zero is not what to expect from concat itself.
# scripts/ci/isa-allow.txt says, per file, how much of each class is
diff --git a/scripts/ci/smoke-linux.sh b/scripts/ci/smoke-linux.sh
index 5e2c32ba..6b8a6a7f 100755
--- a/scripts/ci/smoke-linux.sh
+++ b/scripts/ci/smoke-linux.sh
@@ -12,7 +12,9 @@
# to open the NVIDIA driver's CUDA or NVENC libraries, which nothing
# should touch before a person asks for a hardware export. With
# --sde, `--version` again under Intel's emulator posing as an Ivy
-# Bridge (AVX, no AVX2), the oldest CPU the x86_64 bundle promises.
+# Bridge (AVX, no AVX2), the oldest CPU the x86_64 bundle promises,
+# and `--whisper-cpu`, loading whisper's CPU kernels, posing as a
+# Sandy Bridge.
#
# smoke-linux.sh --container IMAGE DEB APPIMAGE
# The packages, in a clean IMAGE (ubuntu:22.04, debian:12) through
@@ -104,6 +106,14 @@ host() {
# The emulator stops the program at the first instruction the chip
# it poses as does not have, and says which.
HOME=$(fresh_home) expect_ok "concat --version under SDE -ivb" 600 "$sde" -ivb -- "$bin" --version
+
+ # whisper's CPU kernels, one module per CPU generation: ggml opens
+ # every one to score it against the CPU before it keeps the best, so
+ # the Haswell and Ice Lake modules are loaded here too, on a Sandy
+ # Bridge - AVX without even F16C, like #318's i7-2600 - which must
+ # come out with the sandybridge module and no illegal instruction.
+ say "concat --whisper-cpu on an emulated Sandy Bridge"
+ HOME=$(fresh_home) expect_ok "concat --whisper-cpu under SDE -snb" 600 "$sde" -snb -- "$bin" --whisper-cpu
fi
}
diff --git a/scripts/ci/smoke-windows.ps1 b/scripts/ci/smoke-windows.ps1
index 01e68ce1..64c9d83c 100644
--- a/scripts/ci/smoke-windows.ps1
+++ b/scripts/ci/smoke-windows.ps1
@@ -18,7 +18,8 @@
# the first frame of the window, then the program leaves.
# 4. With -Sde, `--version` again under Intel's emulator posing as an Ivy
# Bridge (AVX, nothing newer), which stops the program at the first
-# instruction that chip does not have. x86_64 only.
+# instruction that chip does not have; then `--whisper-cpu`, loading
+# whisper's CPU kernels, posing as a Sandy Bridge. x86_64 only.
#
# concat.exe is a GUI program: it has no console, so it is judged by its
# exit code and not by what it prints. Windows' error dialogs are turned
@@ -179,6 +180,14 @@ if ($Sde) {
# The emulator's own folder too: it loads its pieces from beside itself.
$path = (Split-Path $sdeExe) + ";" + $bare
Expect-Ok "concat.exe --version under SDE -ivb" $sdeExe @("-ivb", "--", $exe, "--version") $path 600
+
+ # whisper's CPU kernels, one DLL per CPU generation: ggml loads every
+ # one to score it against the CPU before it keeps the best, so the
+ # Haswell and Ice Lake DLLs are loaded here too, on a Sandy Bridge - AVX
+ # without even F16C, like #318's i7-2600 - which must come out with the
+ # sandybridge DLL and no illegal instruction.
+ Say "concat.exe --whisper-cpu on an emulated Sandy Bridge"
+ Expect-Ok "concat.exe --whisper-cpu under SDE -snb" $sdeExe @("-snb", "--", $exe, "--whisper-cpu") $path 600
}
Say "ok"
diff --git a/src/Cargo.toml b/src/Cargo.toml
index 4a26396e..cd95d77b 100644
--- a/src/Cargo.toml
+++ b/src/Cargo.toml
@@ -142,6 +142,8 @@ incremental = false
# bump it with Slint, re-applying the change.
[patch.crates-io]
i-slint-renderer-skia = { path = "patches/i-slint-renderer-skia" }
-# whisper.cpp's Rust build, as published, less the GPU backends no build
-# of Concat compiles. 0.15.0 exactly; bump it with whisper-rs.
+# whisper.cpp's Rust build, as published, able to build ggml as shared
+# libraries with one CPU backend per x86_64 generation chosen at run time
+# (see patches/README.md, and GGML_BACKEND_DL in build-app.yml). 0.15.0
+# exactly; bump it with whisper-rs, re-applying the change.
whisper-rs-sys = { path = "patches/whisper-rs-sys" }
diff --git a/src/crates/concat-speech/Cargo.toml b/src/crates/concat-speech/Cargo.toml
index dbbfd008..43846de3 100644
--- a/src/crates/concat-speech/Cargo.toml
+++ b/src/crates/concat-speech/Cargo.toml
@@ -40,8 +40,10 @@ tokenizers = { version = "0.23", optional = true, default-features = false, feat
log = { version = "0.4", features = ["std"] }
# Transcription: whisper.cpp in-process. Built from source by cmake at build
-# time; on macOS with Metal so the encoder runs on the GPU.
-whisper-rs = "0.16"
+# time; on macOS with Metal so the encoder runs on the GPU. `raw-api` for
+# ggml's backend registry: where whisper.cpp is a shared build, its CPU
+# kernels are loaded by hand (transcribe.rs, `cpu_backend`).
+whisper-rs = { version = "0.16", features = ["raw-api"] }
# Text to speech: the official k2-fsa bindings. Linked statically on the
# desktop Unixes, so the binary carries no extra dylibs.
diff --git a/src/crates/concat-speech/src/transcribe.rs b/src/crates/concat-speech/src/transcribe.rs
index a925c88c..b856087e 100644
--- a/src/crates/concat-speech/src/transcribe.rs
+++ b/src/crates/concat-speech/src/transcribe.rs
@@ -134,6 +134,90 @@ unsafe extern "C" fn abort_when_set(user_data: *mut std::ffi::c_void) -> bool {
unsafe { (*(user_data as *const AtomicBool)).load(Ordering::Relaxed) }
}
+/// Makes sure whisper has a CPU to run on, and names that CPU.
+///
+/// A build linked whisper statically - every one but the x86_64 Windows and
+/// Linux bundles - has its CPU backend from the start. Those two link it as
+/// shared libraries, with the kernels in one module per x86_64 generation
+/// (ggml-cpu-sandybridge, -haswell, -icelake, ...), and ggml registers
+/// nothing until it is told where to look: it then scores every module it
+/// finds against the CPU and loads the best one. That is how the same
+/// bundle runs AVX2, FMA and F16C on a Haswell or a Ryzen - eight times as
+/// fast as the AVX-only build on a Ryzen 5 2600 (#388) - and still runs on
+/// a Sandy Bridge (#193, #318).
+///
+/// The modules are beside the program on Windows and in its lib/ folder on
+/// Linux; a `cargo test` binary is one folder further down, in deps/.
+/// Only those folders are searched - not the working directory, which ggml
+/// would otherwise also look in. ggml logs which module it chose.
+///
+/// Called once per process; later calls answer from the first.
+pub fn cpu_backend() -> Result<String, String> {
+ use whisper_rs::whisper_rs_sys as sys;
+
+ fn cpu() -> Option<String> {
+ // SAFETY: a lookup in ggml's registry of devices, which ggml
+ // guards itself; the description is a C string ggml owns for the
+ // life of the process.
+ unsafe {
+ let device = sys::ggml_backend_dev_by_type(
+ sys::ggml_backend_dev_type_GGML_BACKEND_DEVICE_TYPE_CPU,
+ );
+ if device.is_null() {
+ return None;
+ }
+ let description = sys::ggml_backend_dev_description(device);
+ Some(if description.is_null() {
+ "CPU".to_owned()
+ } else {
+ std::ffi::CStr::from_ptr(description)
+ .to_string_lossy()
+ .into_owned()
+ })
+ }
+ }
+
+ static LOADED: std::sync::OnceLock<Result<String, String>> = std::sync::OnceLock::new();
+ LOADED
+ .get_or_init(|| {
+ if let Some(cpu) = cpu() {
+ return Ok(cpu);
+ }
+ let program = std::env::current_exe()
+ .map_err(|error| format!("could not find the program's folder: {error}"))?;
+ let beside = program.parent().unwrap_or(std::path::Path::new("."));
+ let mut folders = vec![beside.to_path_buf(), beside.join("lib")];
+ if beside.file_name().is_some_and(|name| name == "deps")
+ && let Some(profile) = beside.parent()
+ {
+ folders.push(profile.to_path_buf());
+ }
+ for folder in &folders {
+ // ggml reads the path as UTF-8 on every platform.
+ let Some(text) = folder.to_str() else {
+ continue;
+ };
+ let Ok(path) = std::ffi::CString::new(text) else {
+ continue;
+ };
+ // SAFETY: a NUL-terminated path that outlives the call.
+ unsafe { sys::ggml_backend_load_all_from_path(path.as_ptr()) };
+ if let Some(cpu) = cpu() {
+ log::info!(
+ "whisper: CPU kernels loaded from {}: {cpu}",
+ folder.display()
+ );
+ return Ok(cpu);
+ }
+ }
+ Err(format!(
+ "whisper's CPU kernels (ggml-cpu-*) are missing from {} - reinstall Concat",
+ folders[0].display()
+ ))
+ })
+ .clone()
+}
+
fn model_file(dirs: &AppDirs, id: &str) -> Result<PathBuf, String> {
// Ids come from our own table; anything else is a bug, not input.
known(id).ok_or_else(|| format!("unknown model {id:?}"))?;
@@ -373,6 +457,7 @@ impl Transcriber {
if loaded.as_ref().map(|model| model.id.as_str()) != Some(request.model_id.as_str()) {
// Load before overwriting: a failed load keeps the old model.
progress(JobStage::Loading);
+ cpu_backend()?;
let context =
WhisperContext::new_with_params(&model_path, WhisperContextParameters::default())
.map_err(|error| format!("could not load the model: {error}"))?;
diff --git a/src/crates/concat/src/launch.rs b/src/crates/concat/src/launch.rs
index 201d5e28..229ebe4c 100644
--- a/src/crates/concat/src/launch.rs
+++ b/src/crates/concat/src/launch.rs
@@ -6,7 +6,8 @@
use std::ffi::OsString;
use std::path::PathBuf;
-const USAGE: &str = "usage: concat [--project <PROJECT_DIRECTORY> | --version | --smoke]";
+const USAGE: &str =
+ "usage: concat [--project <PROJECT_DIRECTORY> | --version | --smoke | --whisper-cpu]";
/// What one run of the binary was asked to do.
#[derive(Debug, PartialEq, Eq)]
@@ -21,6 +22,14 @@ pub enum Launch {
/// the window's models, and exit instead of entering the event loop.
/// For CI, which has no one to close a window.
Smoke,
+ /// Load whisper's CPU kernels the way the first transcription does,
+ /// print the CPU they run on and exit, before logging, the GPU or a
+ /// window. For CI on an emulated older CPU: where the kernels are one
+ /// module per CPU generation, ggml opens every one of them to ask it
+ /// whether it suits, and a module whose loading alone used an
+ /// instruction the CPU lacks would crash here, not at somebody's first
+ /// transcription (#388).
+ WhisperCpu,
}
/// No arguments preserves the launch screen; a project names its directory.
@@ -33,6 +42,8 @@ pub fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Launch, String>
Launch::Version
} else if option == "--smoke" {
Launch::Smoke
+ } else if option == "--whisper-cpu" {
+ Launch::WhisperCpu
} else if option == "--project" {
let path = args
.next()
@@ -73,8 +84,10 @@ mod tests {
fn version_and_smoke_take_no_arguments() {
assert_eq!(parse_strs(&["--version"]).unwrap(), Launch::Version);
assert_eq!(parse_strs(&["--smoke"]).unwrap(), Launch::Smoke);
+ assert_eq!(parse_strs(&["--whisper-cpu"]).unwrap(), Launch::WhisperCpu);
for args in [
vec!["--version", "--smoke"],
+ vec!["--whisper-cpu", "--version"],
vec!["--smoke", "--project", "one"],
vec!["--version", "extra"],
vec!["--project", "one", "--smoke"],
diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rs
index f85c97ea..0e6ba6bc 100644
--- a/src/crates/concat/src/lib.rs
+++ b/src/crates/concat/src/lib.rs
@@ -2031,6 +2031,23 @@ fn start(project: Option<std::path::PathBuf>, smoke: bool) -> Result<(), slint::
// loop's end takes below, for the same reason - the engine's threads
// are not waited on.
if smoke {
+ // whisper's CPU kernels, which the x86_64 Windows and Linux bundles
+ // carry as modules beside the program, one per CPU generation, for
+ // ggml to choose from (#388). A bundle without them starts and
+ // fails at its first transcription, so the smoke run loads them.
+ // Not elsewhere: there they are linked in, and asking ggml for its
+ // devices would only bring Metal up on a Mac runner for nothing.
+ #[cfg(all(
+ target_arch = "x86_64",
+ any(target_os = "windows", target_os = "linux")
+ ))]
+ match concat_speech::transcribe::cpu_backend() {
+ Ok(cpu) => log::info!("smoke: whisper runs on {cpu}"),
+ Err(error) => {
+ log::error!("smoke: {error}");
+ leave(1);
+ }
+ }
log::info!("smoke: ok");
leave(0);
}
diff --git a/src/crates/concat/src/main.rs b/src/crates/concat/src/main.rs
index 564c8e43..8fec20c9 100644
--- a/src/crates/concat/src/main.rs
+++ b/src/crates/concat/src/main.rs
@@ -27,6 +27,14 @@ fn main() -> Result<(), slint::PlatformError> {
let _ = writeln!(std::io::stdout(), "concat {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
+ // Before the log and the window too: what this proves is that whisper's
+ // CPU kernels load on this CPU, and nothing else should be in the way.
+ if launch == Ok(Launch::WhisperCpu) {
+ use std::io::Write;
+ let cpu = concat_speech::transcribe::cpu_backend().map_err(slint::PlatformError::Other)?;
+ let _ = writeln!(std::io::stdout(), "whisper runs on {cpu}");
+ return Ok(());
+ }
// A desktop's console is its standard error, which a packaged build
// has wired to nowhere; hence the file underneath it.
concat::open_logging(None);
@@ -47,6 +55,6 @@ fn main() -> Result<(), slint::PlatformError> {
log::error!("smoke: could not start: {error}");
}),
// Answered above, before the log was opened.
- Launch::Version => Ok(()),
+ Launch::Version | Launch::WhisperCpu => Ok(()),
}
}
diff --git a/src/patches/README.md b/src/patches/README.md
index 4249e8b9..0d019d4c 100644
--- a/src/patches/README.md
+++ b/src/patches/README.md
@@ -38,3 +38,29 @@ And one for Android, marked the same way:
it, the way `create_default_surface` falls back on every other platform.
Nothing else differs from the published crate.
+
+## whisper-rs-sys 0.15.0
+
+whisper.cpp's Rust build. The published build script builds whisper and
+ggml as static libraries only, so ggml's CPU kernels are compiled for one
+instruction set: AVX2 crashed Sandy and Ivy Bridge machines at launch
+(jub0t/concat#193), and AVX alone made transcription some eight times
+slower everywhere else (#388). One change, marked `Concat:` in `build.rs`:
+
+- `GGML_BACKEND_DL=ON` in the environment builds whisper, ggml and
+ ggml-base as shared libraries and every backend as a module ggml loads
+ at run time, links the three libraries rather than the static archives,
+ and copies the runtime files into the profile directory for `cargo run`.
+ With `GGML_CPU_ALL_VARIANTS=ON` the CPU backend is one module per x86_64
+ generation, and ggml loads the best one the CPU can run; concat-speech's
+ `cpu_backend` tells it where to look. build-app.yml turns both on for the
+ x86_64 Windows and Linux bundles. Unset, the build is the published one.
+
+And the copy is smaller than the published crate: ggml's GPU backends that
+no build of Concat compiles (CANN, CUDA, Hexagon, HIP, MUSA, OpenCL, RPC,
+SYCL, Vulkan, WebGPU, zDNN, ZenDNN) are left out of
+`whisper.cpp/ggml/src`, so the crate's `cuda`, `hipblas`, `vulkan` and
+`intel-sycl` features no longer build. CMake only enters a backend's folder
+when that backend is turned on. `Cargo.lock` and the crate's registry
+metadata files are left out too. Nothing else differs from the published
+crate.
diff --git a/src/patches/whisper-rs-sys/build.rs b/src/patches/whisper-rs-sys/build.rs
index 64499164..12d83c09 100644
--- a/src/patches/whisper-rs-sys/build.rs
+++ b/src/patches/whisper-rs-sys/build.rs
@@ -102,6 +102,23 @@ fn main() {
println!("cargo:rerun-if-changed=wrapper.h");
+ // Concat: GGML_BACKEND_DL=ON in the environment builds whisper and ggml
+ // as shared libraries and each CPU backend as a module ggml loads at run
+ // time, picking the best one the CPU can run - with
+ // GGML_CPU_ALL_VARIANTS=ON, one per x86_64 generation. That is how one
+ // bundle runs AVX2 kernels on a Haswell and still starts on a Sandy
+ // Bridge (jub0t/concat#388, #193). The published build script only
+ // builds static libraries, so the switch was not there to turn.
+ // Watched, because a library built the other way would link wrong.
+ println!("cargo:rerun-if-env-changed=GGML_BACKEND_DL");
+ println!("cargo:rerun-if-env-changed=GGML_CPU_ALL_VARIANTS");
+ let backend_dl = env::var("GGML_BACKEND_DL").is_ok_and(|value| {
+ matches!(
+ value.to_ascii_uppercase().as_str(),
+ "ON" | "1" | "TRUE" | "YES"
+ )
+ });
+
let out = PathBuf::from(env::var("OUT_DIR").unwrap());
let whisper_root = out.join("whisper.cpp");
@@ -182,7 +199,7 @@ fn main() {
config
.profile("Release")
- .define("BUILD_SHARED_LIBS", "OFF")
+ .define("BUILD_SHARED_LIBS", if backend_dl { "ON" } else { "OFF" })
.define("WHISPER_ALL_WARNINGS", "OFF")
.define("WHISPER_ALL_WARNINGS_3RD_PARTY", "OFF")
.define("WHISPER_BUILD_TESTS", "OFF")
@@ -190,6 +207,18 @@ fn main() {
.very_verbose(true)
.pic(true);
+ // Concat: a module build cannot tune itself to the machine it is built
+ // on; ggml refuses the two together. The variants are the tuning.
+ // Off said outright otherwise: the forwarding below only passes what
+ // the environment holds, and an OUT_DIR configured with them on keeps
+ // them in its CMake cache, which then refuses the static build.
+ if backend_dl {
+ config.define("GGML_NATIVE", "OFF");
+ } else {
+ config.define("GGML_BACKEND_DL", "OFF");
+ config.define("GGML_CPU_ALL_VARIANTS", "OFF");
+ }
+
if cfg!(target_os = "windows") {
config.cxxflag("/utf-8");
println!("cargo:rustc-link-lib=advapi32");
@@ -303,6 +332,88 @@ fn main() {
add_link_search_path(&out.join("build")).unwrap();
println!("cargo:rustc-link-search=native={}", destination.display());
+ if backend_dl {
+ link_shared(&destination, &out);
+ } else {
+ link_static();
+ }
+
+ println!(
+ "cargo:WHISPER_CPP_VERSION={}",
+ get_whisper_cpp_version(&whisper_root)
+ .expect("Failed to read whisper.cpp CMake config")
+ .expect("Could not find whisper.cpp version declaration"),
+ );
+
+ // for whatever reason this file is generated during build and triggers cargo complaining
+ _ = std::fs::remove_file("bindings/javascript/package.json");
+}
+
+// Concat: the shared build's linking. Only whisper, ggml and ggml-base are
+// linked; every backend, the CPU's variants among them, is a module ggml
+// loads itself and is never linked. The import libraries are in lib/, and
+// on Windows the DLLs in bin/ - cargo puts native search paths inside the
+// target directory on PATH for `cargo run` and `cargo test`, so naming bin/
+// is what lets those find them.
+fn link_shared(destination: &std::path::Path, out: &std::path::Path) {
+ for dir in ["lib", "lib64", "bin"] {
+ if destination.join(dir).is_dir() {
+ println!("cargo:rustc-link-search=native={}", destination.join(dir).display());
+ }
+ }
+ println!("cargo:rustc-link-lib=dylib=whisper");
+ println!("cargo:rustc-link-lib=dylib=ggml");
+ println!("cargo:rustc-link-lib=dylib=ggml-base");
+ copy_runtime(destination, out);
+}
+
+// Concat: every runtime library of the shared build - whisper, ggml,
+// ggml-base and the backend modules - copied into the profile directory
+// (target/[<triple>/]<profile>/), beside the programs cargo builds there,
+// so a `cargo run` program finds its CPU backends next to itself, where
+// they are looked for. A release bundle is staged from OUT_DIR's own
+// install instead (build-app.yml): a build CI's cache satisfies does not
+// run this script again to make the copies. Symbolic links
+// (libwhisper.so.1 -> libwhisper.so.1.8.3) are copied as links.
+fn copy_runtime(destination: &std::path::Path, out: &std::path::Path) {
+ // OUT_DIR is <profile>/build/whisper-rs-sys-<hash>/out.
+ let Some(profile) = out.ancestors().nth(3) else {
+ println!("cargo:warning=no profile directory above {}", out.display());
+ return;
+ };
+ for dir in ["bin", "lib", "lib64"] {
+ let Ok(entries) = std::fs::read_dir(destination.join(dir)) else {
+ continue;
+ };
+ for entry in entries.flatten() {
+ let name = entry.file_name();
+ let text = name.to_string_lossy();
+ let runtime = text.ends_with(".dll")
+ || text.ends_with(".dylib")
+ || text.ends_with(".so")
+ || text.contains(".so.");
+ let ours = text.contains("whisper") || text.contains("ggml");
+ if !runtime || !ours {
+ continue;
+ }
+ let from = entry.path();
+ let to = profile.join(&name);
+ let _ = std::fs::remove_file(&to);
+ let copied = match std::fs::read_link(&from) {
+ #[cfg(unix)]
+ Ok(target) => std::os::unix::fs::symlink(target, &to),
+ _ => std::fs::copy(&from, &to).map(|_| ()),
+ };
+ if let Err(error) = copied {
+ panic!("could not copy {} to {}: {error}", from.display(), to.display());
+ }
+ }
+ }
+}
+
+// The published crate's linking, unchanged, moved out of `main` so the
+// shared build can take its place (Concat).
+fn link_static() {
if cfg!(feature = "intel-sycl") {
println!("cargo:rustc-link-lib=whisper");
println!("cargo:rustc-link-lib=ggml");
@@ -344,16 +455,6 @@ fn main() {
if cfg!(feature = "intel-sycl") {
println!("cargo:rustc-link-lib=ggml-sycl");
}
-
- println!(
- "cargo:WHISPER_CPP_VERSION={}",
- get_whisper_cpp_version(&whisper_root)
- .expect("Failed to read whisper.cpp CMake config")
- .expect("Could not find whisper.cpp version declaration"),
- );
-
- // for whatever reason this file is generated during build and triggers cargo complaining
- _ = std::fs::remove_file("bindings/javascript/package.json");
}
// From https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2462