Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

feat(stt): re-time whisper's words on a CTC forced aligner

EtienneLescot14bf04b237f1
diff --git a/electron/native/whisper-stt/CMakeLists.txt b/electron/native/whisper-stt/CMakeLists.txt
index 0f6f358da..a22d02eed 100644
--- a/electron/native/whisper-stt/CMakeLists.txt
+++ b/electron/native/whisper-stt/CMakeLists.txt
@@ -179,9 +179,10 @@ if(NOT "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp" IN_LIST osc_whi
endif()
set_property(TARGET whisper PROPERTY SOURCES ${osc_whisper_srcs})
-add_executable(whisper-stt-server src/main.cpp)
+add_executable(whisper-stt-server src/main.cpp src/ctc_aligner.cpp)
target_link_libraries(whisper-stt-server PRIVATE
whisper
+ ggml
httplib::httplib
nlohmann_json::nlohmann_json
)
diff --git a/electron/native/whisper-stt/src/ctc_aligner.cpp b/electron/native/whisper-stt/src/ctc_aligner.cpp
new file mode 100644
index 000000000..18d514596
--- /dev/null
+++ b/electron/native/whisper-stt/src/ctc_aligner.cpp
@@ -0,0 +1,433 @@
+// wav2vec2-for-CTC forward pass on ggml (HuggingFace Wav2Vec2ForCTC, both the
+// "group norm, post-LN" base layout and the "layer norm, pre-LN" large one).
+// Weights come from scripts/convert-wav2vec2-gguf.mjs.
+//
+// Layout: activations are [channels, time] (ne0 = channels) for the linear
+// layers, and transposed to [time, channels] only where a convolution or a
+// per-channel norm needs time contiguous.
+
+#include "ctc_aligner.h"
+
+#include <ggml.h>
+#include <ggml-alloc.h>
+#include <ggml-backend.h>
+#include <gguf.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+
+namespace {
+
+constexpr int kGraphSize = 8192;
+// Longest input one graph sees; longer ones run in windows. Attention is
+// quadratic in time and the first convolutions are wide, so this bounds memory
+// (about 265 MB of compute buffers at 20 s, for either model: the first
+// convolutions dominate).
+constexpr int kWindowFrames = 1000; // 20 s
+constexpr int kContextFrames = 50; // 1 s of context kept on each side of a window
+
+struct Layer {
+ ggml_tensor *q_w, *q_b, *k_w, *k_b, *v_w, *v_b, *o_w, *o_b;
+ ggml_tensor *ln1_w, *ln1_b, *ff1_w, *ff1_b, *ff2_w, *ff2_b, *ln2_w, *ln2_b;
+};
+
+} // namespace
+
+struct CtcModel {
+ CtcModelInfo info;
+ int hidden = 0, n_heads = 0, pos_k = 0, pos_groups = 0;
+ float eps = 1e-5f;
+ bool group_norm = false, stable = false;
+ std::vector<int> conv_k, conv_s;
+
+ std::vector<ggml_tensor*> conv_w, conv_b, conv_ln_w, conv_ln_b;
+ ggml_tensor *fp_ln_w = nullptr, *fp_ln_b = nullptr, *fp_w = nullptr, *fp_b = nullptr;
+ ggml_tensor *pos_w = nullptr, *pos_b = nullptr, *enc_ln_w = nullptr, *enc_ln_b = nullptr;
+ ggml_tensor *head_w = nullptr, *head_b = nullptr;
+ std::vector<Layer> layers;
+
+ ggml_context* wctx = nullptr;
+ gguf_context* gctx = nullptr;
+ ggml_backend_buffer_t wbuf = nullptr;
+ ggml_backend_t gpu = nullptr, cpu = nullptr;
+ ggml_backend_sched_t sched = nullptr;
+ std::string device = "CPU";
+
+ ~CtcModel() {
+ if (sched) ggml_backend_sched_free(sched);
+ if (wbuf) ggml_backend_buffer_free(wbuf);
+ if (gpu) ggml_backend_free(gpu);
+ if (cpu) ggml_backend_free(cpu);
+ if (wctx) ggml_free(wctx);
+ if (gctx) gguf_free(gctx);
+ }
+};
+
+void CtcModelDeleter::operator()(CtcModel* m) const { delete m; }
+
+const CtcModelInfo& ctc_info(const CtcModel& model) { return model.info; }
+std::string ctc_device(const CtcModel& model) { return model.device; }
+
+namespace {
+
+int64_t key(gguf_context* g, const char* k, std::string& err) {
+ const int64_t id = gguf_find_key(g, k);
+ if (id < 0 && err.empty()) err = std::string("missing metadata ") + k;
+ return id;
+}
+
+std::vector<std::string> str_array(gguf_context* g, int64_t id) {
+ std::vector<std::string> out;
+ if (id < 0) return out;
+ const size_t n = gguf_get_arr_n(g, id);
+ for (size_t i = 0; i < n; ++i) out.emplace_back(gguf_get_arr_str(g, id, i));
+ return out;
+}
+
+std::vector<int> u32_array(gguf_context* g, int64_t id) {
+ std::vector<int> out;
+ if (id < 0) return out;
+ const auto* p = static_cast<const uint32_t*>(gguf_get_arr_data(g, id));
+ out.assign(p, p + gguf_get_arr_n(g, id));
+ return out;
+}
+
+ggml_tensor* layer_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, float eps) {
+ return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b);
+}
+
+ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b) {
+ return ggml_add(ctx, ggml_mul_mat(ctx, w, x), b);
+}
+
+// 1-D convolution of `x` [time, in] with `w` [k, in, out] -> [out, time']. The
+// columns are F16, as in whisper.cpp's encoder: half the memory traffic of F32,
+// which made the CPU path 1.7x slower, and `w` must be F16 to match.
+ggml_tensor* conv1d(ggml_context* ctx, ggml_tensor* w, ggml_tensor* x, int stride, int pad) {
+ ggml_tensor* cols = ggml_im2col(ctx, w, x, stride, 0, pad, 0, 1, 0, false, GGML_TYPE_F16);
+ // cols: [k * in, time']
+ cols = ggml_reshape_2d(ctx, cols, cols->ne[0], cols->ne[1]);
+ ggml_tensor* w2 = ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]);
+ return ggml_mul_mat(ctx, w2, cols);
+}
+
+ggml_tensor* build(CtcModel& m, ggml_context* ctx, ggml_tensor* wave) {
+ // ---- feature encoder ----
+ ggml_tensor* x = wave; // [time, 1]
+ ggml_tensor* y = nullptr;
+ for (size_t i = 0; i < m.conv_w.size(); ++i) {
+ y = conv1d(ctx, m.conv_w[i], x, m.conv_s[i], 0); // [512, t]
+ if (m.conv_b[i]) y = ggml_add(ctx, y, m.conv_b[i]);
+ if (m.group_norm) {
+ // Time-major for the next convolution. Only the first layer is normed:
+ // GroupNorm with one group per channel, i.e. each channel over time.
+ ggml_tensor* t = ggml_cont(ctx, ggml_transpose(ctx, y)); // [t, 512]
+ if (i == 0) {
+ t = ggml_norm(ctx, t, m.eps);
+ t = ggml_add(ctx, ggml_mul(ctx, t, ggml_reshape_2d(ctx, m.conv_ln_w[0], 1, t->ne[1])),
+ ggml_reshape_2d(ctx, m.conv_ln_b[0], 1, t->ne[1]));
+ }
+ x = ggml_gelu_erf(ctx, t);
+ } else {
+ y = layer_norm(ctx, y, m.conv_ln_w[i], m.conv_ln_b[i], m.eps);
+ x = ggml_cont(ctx, ggml_transpose(ctx, ggml_gelu_erf(ctx, y))); // [t, 512]
+ }
+ }
+ // x: [t, 512] -> [512, t]
+ ggml_tensor* h = ggml_cont(ctx, ggml_transpose(ctx, x));
+ const int64_t T = h->ne[1];
+
+ // ---- feature projection ----
+ h = layer_norm(ctx, h, m.fp_ln_w, m.fp_ln_b, m.eps);
+ h = linear(ctx, h, m.fp_w, m.fp_b); // [H, T]
+
+ // ---- positional convolution (grouped, same padding, drop the last frame) ----
+ {
+ ggml_tensor* ht = ggml_cont(ctx, ggml_transpose(ctx, h)); // [T, H]
+ const int64_t cg = m.hidden / m.pos_groups;
+ ggml_tensor* pos = nullptr;
+ for (int g = 0; g < m.pos_groups; ++g) {
+ ggml_tensor* xin = ggml_view_2d(ctx, ht, T, cg, ht->nb[1], g * cg * ht->nb[1]);
+ ggml_tensor* wg = ggml_view_3d(ctx, m.pos_w, m.pos_k, cg, cg, m.pos_w->nb[1],
+ m.pos_w->nb[2], g * cg * m.pos_w->nb[2]);
+ ggml_tensor* part = conv1d(ctx, wg, ggml_cont(ctx, xin), 1, m.pos_k / 2); // [cg, T+1]
+ pos = pos ? ggml_concat(ctx, pos, part, 0) : part;
+ }
+ pos = ggml_view_2d(ctx, pos, m.hidden, T, pos->nb[1], 0);
+ pos = ggml_gelu_erf(ctx, ggml_add(ctx, pos, m.pos_b));
+ h = ggml_add(ctx, h, pos);
+ }
+ if (!m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps);
+
+ // ---- transformer ----
+ const int64_t dh = m.hidden / m.n_heads;
+ const float scale = 1.0f / std::sqrt(static_cast<float>(dh));
+ for (const Layer& L : m.layers) {
+ ggml_tensor* res = h;
+ ggml_tensor* a = m.stable ? layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps) : h;
+ ggml_tensor* q = ggml_reshape_3d(ctx, linear(ctx, a, L.q_w, L.q_b), dh, m.n_heads, T);
+ ggml_tensor* k = ggml_reshape_3d(ctx, linear(ctx, a, L.k_w, L.k_b), dh, m.n_heads, T);
+ ggml_tensor* v = ggml_reshape_3d(ctx, linear(ctx, a, L.v_w, L.v_b), dh, m.n_heads, T);
+ q = ggml_permute(ctx, q, 0, 2, 1, 3); // [dh, T, nh]
+ k = ggml_permute(ctx, k, 0, 2, 1, 3); // [dh, T, nh]
+ v = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, dh, nh]
+ ggml_tensor* kq = ggml_mul_mat(ctx, k, q); // [Tk, Tq, nh]
+ kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f);
+ ggml_tensor* o = ggml_mul_mat(ctx, v, kq); // [dh, Tq, nh]
+ o = ggml_cont(ctx, ggml_permute(ctx, o, 0, 2, 1, 3)); // [dh, nh, T]
+ o = linear(ctx, ggml_reshape_2d(ctx, o, m.hidden, T), L.o_w, L.o_b);
+ h = ggml_add(ctx, res, o);
+ if (m.stable) {
+ ggml_tensor* f = layer_norm(ctx, h, L.ln2_w, L.ln2_b, m.eps);
+ f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, f, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b);
+ h = ggml_add(ctx, h, f);
+ } else {
+ h = layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps);
+ ggml_tensor* f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, h, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b);
+ h = layer_norm(ctx, ggml_add(ctx, h, f), L.ln2_w, L.ln2_b, m.eps);
+ }
+ }
+ if (m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps);
+ return linear(ctx, h, m.head_w, m.head_b); // [V, T]
+}
+
+int frames_for(const CtcModel& m, int64_t n) {
+ for (size_t i = 0; i < m.conv_k.size(); ++i) {
+ if (n < m.conv_k[i]) return 0;
+ n = (n - m.conv_k[i]) / m.conv_s[i] + 1;
+ }
+ return static_cast<int>(n);
+}
+
+// Logits for one window of already-normalized samples.
+bool run_window(CtcModel& m, const float* pcm, int64_t n, std::vector<float>& logits, int& frames,
+ std::string& err) {
+ frames = frames_for(m, n);
+ if (frames <= 0) return true;
+ ggml_init_params p = {ggml_tensor_overhead() * kGraphSize + ggml_graph_overhead_custom(kGraphSize, false),
+ nullptr, true};
+ ggml_context* ctx = ggml_init(p);
+ ggml_tensor* wave = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1);
+ ggml_set_input(wave);
+ ggml_tensor* out = build(m, ctx, wave);
+ ggml_set_output(out);
+ ggml_cgraph* gf = ggml_new_graph_custom(ctx, kGraphSize, false);
+ ggml_build_forward_expand(gf, out);
+ ggml_backend_sched_reset(m.sched);
+ bool ok = ggml_backend_sched_alloc_graph(m.sched, gf);
+ if (ok) {
+ ggml_backend_tensor_set(wave, pcm, 0, n * sizeof(float));
+ ok = ggml_backend_sched_graph_compute(m.sched, gf) == GGML_STATUS_SUCCESS;
+ }
+ if (ok) {
+ if (out->ne[1] != frames) {
+ err = "unexpected frame count";
+ ok = false;
+ } else {
+ logits.resize(static_cast<size_t>(out->ne[0]) * frames);
+ ggml_backend_tensor_get(out, logits.data(), 0, logits.size() * sizeof(float));
+ }
+ } else if (err.empty()) {
+ err = "ggml graph allocation or compute failed";
+ }
+ ggml_free(ctx);
+ return ok;
+}
+
+} // namespace
+
+CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err) {
+ CtcModelPtr m(new CtcModel());
+ gguf_init_params gp = {true, &m->wctx};
+ m->gctx = gguf_init_from_file(path.c_str(), gp);
+ if (!m->gctx) {
+ err = "cannot read " + path;
+ return nullptr;
+ }
+ gguf_context* g = m->gctx;
+ CtcModelInfo& info = m->info;
+ info.vocab = str_array(g, key(g, "w2v.vocab", err));
+ info.languages = str_array(g, key(g, "w2v.languages", err));
+ int64_t id;
+ if ((id = key(g, "w2v.blank_id", err)) >= 0) info.blank = gguf_get_val_u32(g, id);
+ if ((id = key(g, "w2v.hidden_size", err)) >= 0) m->hidden = gguf_get_val_u32(g, id);
+ if ((id = key(g, "w2v.num_attention_heads", err)) >= 0) m->n_heads = gguf_get_val_u32(g, id);
+ if ((id = key(g, "w2v.layer_norm_eps", err)) >= 0) m->eps = gguf_get_val_f32(g, id);
+ if ((id = key(g, "w2v.num_conv_pos_embeddings", err)) >= 0) m->pos_k = gguf_get_val_u32(g, id);
+ if ((id = key(g, "w2v.num_conv_pos_embedding_groups", err)) >= 0) m->pos_groups = gguf_get_val_u32(g, id);
+ if ((id = key(g, "w2v.feat_extract_norm", err)) >= 0) m->group_norm = std::string(gguf_get_val_str(g, id)) == "group";
+ if ((id = key(g, "w2v.stable_layer_norm", err)) >= 0) m->stable = gguf_get_val_bool(g, id);
+ m->conv_k = u32_array(g, key(g, "w2v.conv_kernel", err));
+ m->conv_s = u32_array(g, key(g, "w2v.conv_stride", err));
+ int n_layers = 0;
+ if ((id = key(g, "w2v.num_hidden_layers", err)) >= 0) n_layers = gguf_get_val_u32(g, id);
+ if (!err.empty()) return nullptr;
+ info.stride = 1;
+ for (int s : m->conv_s) info.stride *= s;
+ info.receptive_field = 1;
+ for (int i = static_cast<int>(m->conv_k.size()) - 1; i >= 0; --i)
+ info.receptive_field = (info.receptive_field - 1) * m->conv_s[i] + m->conv_k[i];
+
+ auto T = [&](const std::string& name, bool required = true) -> ggml_tensor* {
+ ggml_tensor* t = ggml_get_tensor(m->wctx, name.c_str());
+ if (!t && required && err.empty()) err = "missing tensor " + name;
+ return t;
+ };
+ for (size_t i = 0; i < m->conv_k.size(); ++i) {
+ const std::string p = "feature_extractor.conv_layers." + std::to_string(i) + ".";
+ m->conv_w.push_back(T(p + "conv.weight"));
+ m->conv_b.push_back(T(p + "conv.bias", false));
+ const bool has_ln = !m->group_norm || i == 0;
+ m->conv_ln_w.push_back(has_ln ? T(p + "layer_norm.weight") : nullptr);
+ m->conv_ln_b.push_back(has_ln ? T(p + "layer_norm.bias") : nullptr);
+ }
+ m->fp_ln_w = T("feature_projection.layer_norm.weight");
+ m->fp_ln_b = T("feature_projection.layer_norm.bias");
+ m->fp_w = T("feature_projection.projection.weight");
+ m->fp_b = T("feature_projection.projection.bias");
+ m->pos_w = T("encoder.pos_conv_embed.conv.weight");
+ m->pos_b = T("encoder.pos_conv_embed.conv.bias");
+ m->enc_ln_w = T("encoder.layer_norm.weight");
+ m->enc_ln_b = T("encoder.layer_norm.bias");
+ m->head_w = T("lm_head.weight");
+ m->head_b = T("lm_head.bias");
+ for (int l = 0; l < n_layers; ++l) {
+ const std::string p = "encoder.layers." + std::to_string(l) + ".";
+ m->layers.push_back({
+ T(p + "attention.q_proj.weight"), T(p + "attention.q_proj.bias"),
+ T(p + "attention.k_proj.weight"), T(p + "attention.k_proj.bias"),
+ T(p + "attention.v_proj.weight"), T(p + "attention.v_proj.bias"),
+ T(p + "attention.out_proj.weight"), T(p + "attention.out_proj.bias"),
+ T(p + "layer_norm.weight"), T(p + "layer_norm.bias"),
+ T(p + "feed_forward.intermediate_dense.weight"), T(p + "feed_forward.intermediate_dense.bias"),
+ T(p + "feed_forward.output_dense.weight"), T(p + "feed_forward.output_dense.bias"),
+ T(p + "final_layer_norm.weight"), T(p + "final_layer_norm.bias"),
+ });
+ }
+ if (!err.empty()) return nullptr;
+ // conv1d() multiplies them with F16 columns; anything else aborts inside ggml.
+ for (const ggml_tensor* w : m->conv_w) {
+ if (w->type != GGML_TYPE_F16) err = "convolution weights must be F16";
+ }
+ if (m->pos_w->type != GGML_TYPE_F16) err = "convolution weights must be F16";
+ if (!err.empty()) return nullptr;
+ if (static_cast<int>(info.vocab.size()) != m->head_w->ne[1]) {
+ err = "vocabulary does not match lm_head";
+ return nullptr;
+ }
+
+ // ---- backends ----
+ if (use_gpu) {
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+ const auto type = ggml_backend_dev_type(dev);
+ if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU) continue;
+ m->gpu = ggml_backend_dev_init(dev, nullptr);
+ if (m->gpu) {
+ m->device = ggml_backend_dev_name(dev);
+ break;
+ }
+ }
+ }
+ m->cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
+ if (!m->cpu) {
+ err = "no CPU backend";
+ return nullptr;
+ }
+ {
+ ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(m->cpu));
+ using set_threads_t = void (*)(ggml_backend_t, int);
+ auto set_threads = reinterpret_cast<set_threads_t>(
+ ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
+ if (set_threads) set_threads(m->cpu, threads);
+ }
+ ggml_backend_t weights_on = m->gpu ? m->gpu : m->cpu;
+ m->wbuf = ggml_backend_alloc_ctx_tensors(m->wctx, weights_on);
+ if (!m->wbuf) {
+ err = "cannot allocate the model's weights";
+ return nullptr;
+ }
+ {
+ FILE* f = std::fopen(path.c_str(), "rb");
+ if (!f) {
+ err = "cannot open " + path;
+ return nullptr;
+ }
+ std::vector<uint8_t> buf;
+ const size_t data_off = gguf_get_data_offset(g);
+ for (int64_t i = 0; i < gguf_get_n_tensors(g); ++i) {
+ ggml_tensor* t = ggml_get_tensor(m->wctx, gguf_get_tensor_name(g, i));
+ const size_t nbytes = ggml_nbytes(t);
+ buf.resize(nbytes);
+#ifdef _WIN32
+ _fseeki64(f, static_cast<int64_t>(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET);
+#else
+ fseeko(f, static_cast<off_t>(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET);
+#endif
+ if (std::fread(buf.data(), 1, nbytes, f) != nbytes) {
+ std::fclose(f);
+ err = "truncated model file " + path;
+ return nullptr;
+ }
+ ggml_backend_tensor_set(t, buf.data(), 0, nbytes);
+ }
+ std::fclose(f);
+ }
+ std::vector<ggml_backend_t> backends;
+ if (m->gpu) backends.push_back(m->gpu);
+ backends.push_back(m->cpu);
+ m->sched = ggml_backend_sched_new(backends.data(), nullptr, static_cast<int>(backends.size()),
+ kGraphSize, false, true);
+ if (!m->sched) {
+ err = "cannot create the ggml scheduler";
+ return nullptr;
+ }
+ return m;
+}
+
+bool ctc_emissions(CtcModel& m, const float* pcm, size_t n, std::vector<float>& out, int& frames,
+ std::string& err) {
+ const int V = static_cast<int>(m.info.vocab.size());
+ const int64_t stride = m.info.stride;
+ frames = frames_for(m, static_cast<int64_t>(n));
+ out.assign(static_cast<size_t>(std::max(frames, 0)) * V, 0.0f);
+ if (frames <= 0) return true;
+
+ // Wav2Vec2FeatureExtractor(do_normalize=True): zero mean, unit variance, per input.
+ double mean = 0.0, var = 0.0;
+ for (size_t i = 0; i < n; ++i) mean += pcm[i];
+ mean /= static_cast<double>(n);
+ for (size_t i = 0; i < n; ++i) var += (pcm[i] - mean) * (pcm[i] - mean);
+ var /= static_cast<double>(n);
+ const double inv = 1.0 / std::sqrt(var + 1e-7);
+ std::vector<float> x(n);
+ for (size_t i = 0; i < n; ++i) x[i] = static_cast<float>((pcm[i] - mean) * inv);
+
+ // Windows of kWindowFrames, stepping by kWindowFrames - 2 * kContextFrames; each
+ // keeps its frames away from the edges it shares with a neighbour.
+ const int64_t win_samples = kWindowFrames * stride + (m.info.receptive_field - stride);
+ const int hop = kWindowFrames - 2 * kContextFrames;
+ std::vector<float> logits;
+ for (int f0 = 0;; f0 += hop) {
+ const int64_t s0 = static_cast<int64_t>(f0) * stride;
+ const int64_t len = std::min<int64_t>(win_samples, static_cast<int64_t>(n) - s0);
+ int wf = 0;
+ if (!run_window(m, x.data() + s0, len, logits, wf, err)) return false;
+ const bool last = s0 + len >= static_cast<int64_t>(n);
+ const int keep_from = f0 == 0 ? 0 : kContextFrames;
+ const int keep_to = last ? wf : std::min(wf, kWindowFrames - kContextFrames);
+ for (int i = keep_from; i < keep_to && f0 + i < frames; ++i) {
+ const float* row = logits.data() + static_cast<size_t>(i) * V;
+ float mx = row[0];
+ for (int v = 1; v < V; ++v) mx = std::max(mx, row[v]);
+ double sum = 0.0;
+ for (int v = 0; v < V; ++v) sum += std::exp(row[v] - mx);
+ const float lse = mx + static_cast<float>(std::log(sum));
+ float* dst = out.data() + static_cast<size_t>(f0 + i) * V;
+ for (int v = 0; v < V; ++v) dst[v] = row[v] - lse;
+ }
+ if (last) break;
+ }
+ return true;
+}
diff --git a/electron/native/whisper-stt/src/ctc_aligner.h b/electron/native/whisper-stt/src/ctc_aligner.h
new file mode 100644
index 000000000..80997505f
--- /dev/null
+++ b/electron/native/whisper-stt/src/ctc_aligner.h
@@ -0,0 +1,37 @@
+// CTC acoustic model (wav2vec2 fine-tuned for CTC) run on ggml, for the word
+// aligner's second pass. The helper only computes the per-frame log-probs; the
+// forced alignment of whisper's words over them lives on the Node side
+// (electron/stt/ctcAlign.ts). See transcription-and-captions.md § Word-level alignment.
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+struct CtcModel;
+
+struct CtcModelInfo {
+ std::vector<std::string> vocab; // token id -> text; "|" is the word delimiter
+ std::vector<std::string> languages; // what the model was fine-tuned on
+ int blank = 0;
+ int stride = 320; // samples per output frame (20 ms at 16 kHz)
+ int receptive_field = 400; // samples the first frame sees
+};
+
+struct CtcModelDeleter { void operator()(CtcModel* m) const; };
+using CtcModelPtr = std::unique_ptr<CtcModel, CtcModelDeleter>;
+
+// Loads a GGUF written by scripts/convert-wav2vec2-gguf.mjs. On the GPU when
+// `use_gpu` and a GPU device is registered, else on the CPU. Null + `err` on failure.
+CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err);
+
+const CtcModelInfo& ctc_info(const CtcModel& model);
+
+// Name of the device the weights live on ("Vulkan0", "MTL0", "CPU").
+std::string ctc_device(const CtcModel& model);
+
+// Log-softmax emissions for 16 kHz mono `pcm`: `frames` rows of vocab-size
+// floats. Frame i sees samples [i * stride, i * stride + receptive_field).
+// Long inputs run in overlapping windows, so memory stays bounded.
+bool ctc_emissions(CtcModel& model, const float* pcm, size_t n, std::vector<float>& out,
+ int& frames, std::string& err);
diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp
index 037230a57..77440e54d 100644
--- a/electron/native/whisper-stt/src/main.cpp
+++ b/electron/native/whisper-stt/src/main.cpp
@@ -36,6 +36,7 @@
// this is a belt-and-braces guarantee against a future bug or parallel invoker).
#include "whisper.h"
+#include "ctc_aligner.h"
#include <algorithm>
#include <atomic>
@@ -259,6 +260,21 @@ double to_original_sec(int64_t cs, const std::vector<Kept>& kept) {
return (kept.back().from + kept.back().len) / 16000.0;
}
+std::string base64(const void* data, size_t n) {
+ static const char* abc = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+ const auto* p = static_cast<const unsigned char*>(data);
+ std::string out;
+ out.reserve((n + 2) / 3 * 4);
+ for (size_t i = 0; i < n; i += 3) {
+ const uint32_t v = (p[i] << 16) | (i + 1 < n ? p[i + 1] << 8 : 0) | (i + 2 < n ? p[i + 2] : 0);
+ out += abc[(v >> 18) & 63];
+ out += abc[(v >> 12) & 63];
+ out += i + 1 < n ? abc[(v >> 6) & 63] : '=';
+ out += i + 2 < n ? abc[v & 63] : '=';
+ }
+ return out;
+}
+
} // namespace
int main(int argc, char** argv) {
@@ -376,13 +392,15 @@ int main(int argc, char** argv) {
res.set_content(probe.dump(), "application/json");
});
- // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`.
- svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) {
+ // The upload of /inference and /emissions: a 16 kHz mono PCM16 WAV in the
+ // multipart field `file`. False after answering 400.
+ const auto read_upload = [](const httplib::Request& req, httplib::Response& res,
+ std::vector<float>& pcm) -> bool {
auto it = req.files.find("file");
if (it == req.files.end()) {
res.status = 400;
res.set_content(R"({"error":"missing 'file' form field"})", "application/json");
- return;
+ return false;
}
const auto& file_entry = it->second;
@@ -404,7 +422,6 @@ int main(int argc, char** argv) {
out.write(file_entry.content.data(),
static_cast<std::streamsize>(file_entry.content.size()));
}
- std::vector<float> pcm;
int sample_rate = 0, channels = 0;
const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels);
std::error_code ec;
@@ -412,15 +429,22 @@ int main(int argc, char** argv) {
if (!ok) {
res.status = 400;
res.set_content(R"({"error":"failed to parse WAV"})", "application/json");
- return;
+ return false;
}
if (sample_rate != 16000 || channels != 1) {
res.status = 400;
res.set_content(
R"({"error":"expected 16 kHz mono PCM16 WAV"})",
"application/json");
- return;
+ return false;
}
+ return true;
+ };
+
+ // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`.
+ svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) {
+ std::vector<float> pcm;
+ if (!read_upload(req, res, pcm)) return;
// language param
std::string language = "auto";
@@ -669,6 +693,77 @@ int main(int argc, char** argv) {
res.set_content(reply.dump(), "application/json");
});
+ // POST /emissions — the CTC aligner's acoustic pass (issue #948, phase 3).
+ // Multipart form: `file` (the same WAV as /inference), `model` (path of a
+ // wav2vec2 GGUF, see ctc_aligner.h) and `regions` (JSON [[start_s, end_s], ...]).
+ // Answers the model's vocabulary and, per region, base64 float32 log-probs
+ // [frames x vocab]; frame i of a region sees the audio from
+ // `start + i * stride_s` for `receptive_s`. The forced alignment itself runs
+ // on the Node side (electron/stt/ctcAlign.ts). The model stays loaded until a
+ // request names another one.
+ CtcModelPtr aligner;
+ std::string aligner_path;
+ svr.Post("/emissions", [&](const httplib::Request& req, httplib::Response& res) {
+ std::vector<float> pcm;
+ if (!read_upload(req, res, pcm)) return;
+ const std::string model = req.get_file_value("model").content;
+ nlohmann::json regions = nlohmann::json::parse(req.get_file_value("regions").content, nullptr, false);
+ if (model.empty() || !regions.is_array()) {
+ res.status = 400;
+ res.set_content(R"({"error":"need 'model' and a JSON 'regions' array"})", "application/json");
+ return;
+ }
+ const std::lock_guard<std::mutex> lk(infer_mu);
+ const auto t0 = std::chrono::steady_clock::now();
+ if (!aligner || aligner_path != model) {
+ aligner.reset();
+ std::string err;
+ aligner = ctc_load(model, cparams.use_gpu, threads, err);
+ if (!aligner) {
+ log("aligner: " + err);
+ res.status = 500;
+ res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json");
+ return;
+ }
+ aligner_path = model;
+ log("aligner loaded on " + ctc_device(*aligner) + ": " + model);
+ }
+ const CtcModelInfo& info = ctc_info(*aligner);
+ nlohmann::json out_regions = nlohmann::json::array();
+ const int64_t n_pcm = static_cast<int64_t>(pcm.size());
+ for (const auto& r : regions) {
+ if (!r.is_array() || r.size() != 2 || !r[0].is_number() || !r[1].is_number()) continue;
+ const int64_t from = std::clamp<int64_t>(std::llround(r[0].get<double>() * 16000.0), 0, n_pcm);
+ const int64_t to = std::clamp<int64_t>(std::llround(r[1].get<double>() * 16000.0), from, n_pcm);
+ std::vector<float> lp;
+ int frames = 0;
+ std::string err;
+ if (!ctc_emissions(*aligner, pcm.data() + from, static_cast<size_t>(to - from), lp, frames, err)) {
+ log("aligner: " + err);
+ res.status = 500;
+ res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json");
+ return;
+ }
+ out_regions.push_back({
+ {"start", from / 16000.0},
+ {"frames", frames},
+ {"logprobs", base64(lp.data(), lp.size() * sizeof(float))},
+ });
+ }
+ const double elapsed_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
+ nlohmann::json reply = {
+ {"vocab", info.vocab},
+ {"blank", info.blank},
+ {"languages", info.languages},
+ {"stride_s", info.stride / 16000.0},
+ {"receptive_s", info.receptive_field / 16000.0},
+ {"device", ctc_device(*aligner)},
+ {"elapsed_s", elapsed_s},
+ {"regions", std::move(out_regions)},
+ };
+ res.set_content(reply.dump(), "application/json");
+ });
+
// ---- bind + listen ----
int bound_port = port;
if (bound_port == 0) {
diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts
new file mode 100644
index 000000000..99238205f
--- /dev/null
+++ b/electron/stt/ctcAlign.test.ts
@@ -0,0 +1,212 @@
+import { describe, expect, it } from "vitest";
+import {
+ type AlignableWord,
+ alignWordsOnEmissions,
+ type CtcEmissions,
+ ctcViterbi,
+ END_OFFSET_SEC,
+ emissionRegions,
+ PAUSE_END_OFFSET_SEC,
+ PAUSE_START_OFFSET_SEC,
+ parseEmissions,
+ START_OFFSET_SEC,
+ wordTokens,
+} from "./ctcAlign";
+
+const VOCAB = ["<pad>", "|", "a", "b", "c", "é", "'"];
+const index = new Map(VOCAB.map((t, i) => [t, i]));
+const id = (t: string) => index.get(t) as number;
+
+/** Log-probs where frame t is sure of `tokens[t]` (a vocab entry, "" for blank). */
+function logprobs(tokens: string[]): Float32Array {
+ const V = VOCAB.length;
+ const out = new Float32Array(tokens.length * V).fill(-12);
+ tokens.forEach((t, f) => {
+ out[f * V + (t ? id(t) : 0)] = -0.01;
+ });
+ return out;
+}
+
+const STRIDE = 0.02;
+const emissions = (tokens: string[], startSec = 0): CtcEmissions => ({
+ vocab: VOCAB,
+ blank: 0,
+ strideSec: STRIDE,
+ regions: [{ startSec, frames: tokens.length, logprobs: logprobs(tokens) }],
+});
+/** `n` blank frames. */
+const gap = (n: number) => Array<string>(n).fill("");
+const word = (
+ text: string,
+ startSec: number,
+ endSec: number,
+ anchorSec = startSec,
+): AlignableWord => ({
+ word: text,
+ startSec,
+ endSec,
+ anchorSec,
+});
+const r = (x: number) => Math.round(x * 1000) / 1000;
+
+describe("wordTokens", () => {
+ it("spells a word in the vocabulary's case, dropping silent punctuation", () => {
+ const upper = new Map(["<pad>", "|", "A", "B", "C", "'"].map((t, i) => [t, i]));
+ expect(wordTokens("Cab,", upper)).toEqual([4, 2, 3]);
+ expect(wordTokens("ab’c", upper)).toEqual([2, 3, 5, 4]);
+ expect(wordTokens("cab", index)).toEqual([id("c"), id("a"), id("b")]);
+ });
+
+ it("keeps an accented letter the vocabulary has and strips the accent otherwise", () => {
+ expect(wordTokens("É", index)).toEqual([id("é")]);
+ expect(wordTokens("àbc", index)).toEqual([id("a"), id("b"), id("c")]);
+ });
+
+ it("gives nothing to say for punctuation, and no spelling for digits and symbols", () => {
+ expect(wordTokens("?", index)).toEqual([]);
+ expect(wordTokens("...", index)).toEqual([]);
+ expect(wordTokens("1080p", index)).toBeNull();
+ expect(wordTokens("€", index)).toBeNull();
+ expect(wordTokens("日本", index)).toBeNull();
+ });
+});
+
+describe("ctcViterbi", () => {
+ it("finds the frames each label is spoken on", () => {
+ const tokens = ["", "a", "a", "", "b", ""];
+ expect(ctcViterbi(logprobs(tokens), 6, VOCAB.length, 0, [id("a"), id("b")])).toEqual([
+ [1, 2],
+ [4, 4],
+ ]);
+ });
+
+ it("needs a blank between two repeats of a label, and fails when the frames are too few", () => {
+ const aba = logprobs(["a", "", "a"]);
+ expect(ctcViterbi(aba, 3, VOCAB.length, 0, [id("a"), id("a")])).toEqual([
+ [0, 0],
+ [2, 2],
+ ]);
+ expect(ctcViterbi(logprobs(["a", "a"]), 2, VOCAB.length, 0, [id("a"), id("a")])).toBeNull();
+ });
+
+ it("scores a wildcard as whatever is spoken", () => {
+ const spans = ctcViterbi(logprobs(["a", "", "c", "c", "", "b"]), 6, VOCAB.length, 0, [
+ id("a"),
+ -1,
+ id("b"),
+ ]);
+ expect(spans?.[1]).toEqual([2, 3]);
+ });
+});
+
+describe("emissionRegions", () => {
+ it("pads each stretch of speech and merges the ones that touch", () => {
+ expect(
+ emissionRegions(
+ [
+ { startSec: 0.1, endSec: 1 },
+ { startSec: 1.4, endSec: 2 },
+ { startSec: 5, endSec: 6 },
+ ],
+ 6.1,
+ ).map(([a, b]) => [r(a), r(b)]),
+ ).toEqual([
+ [0, 2.3],
+ [4.7, 6.1],
+ ]);
+ });
+});
+
+describe("alignWordsOnEmissions", () => {
+ const speech = [{ startSec: 0.2, endSec: 1 }];
+
+ it("times words on their letters, calibrated outward, meeting mid-way inside speech", () => {
+ // "ab" on frames 20-21, the delimiter on 22, "c" on 24: continuous speech.
+ const tokens = [...gap(20), "a", "b", "|", "", "c", ...gap(40)];
+ const out = alignWordsOnEmissions(
+ [word("ab", 0.3, 0.5, 0.45), word("c", 0.5, 0.7, 0.6)],
+ speech,
+ emissions(tokens),
+ );
+ const abEnd = 22 * STRIDE + END_OFFSET_SEC;
+ const cStart = 24 * STRIDE + START_OFFSET_SEC;
+ const boundary = abEnd > cStart ? (abEnd + cStart) / 2 : null;
+ expect(r(out[0].startSec)).toBe(r(20 * STRIDE + PAUSE_START_OFFSET_SEC));
+ expect(r(out[0].endSec)).toBe(r(boundary ?? abEnd));
+ expect(r(out[1].startSec)).toBe(r(boundary ?? cStart));
+ expect(r(out[1].endSec)).toBe(r(25 * STRIDE + PAUSE_END_OFFSET_SEC));
+ expect(out[0].anchorSec).toBe(0.45);
+ });
+
+ it("keeps a pause between two words as a pause", () => {
+ // "a" on frame 12, "c" on frame 40: 0.54 s apart.
+ const tokens = [...gap(12), "a", ...gap(27), "c", ...gap(40)];
+ const out = alignWordsOnEmissions(
+ [word("a", 0.2, 0.4, 0.3), word("c", 0.4, 0.9, 0.85)],
+ speech,
+ emissions(tokens),
+ );
+ expect(r(out[0].endSec)).toBe(r(13 * STRIDE + PAUSE_END_OFFSET_SEC));
+ expect(r(out[1].startSec)).toBe(r(40 * STRIDE + PAUSE_START_OFFSET_SEC));
+ });
+
+ it("gives a word it cannot spell the speech between its neighbours", () => {
+ const tokens = [...gap(15), "a", "|", "c", "b", "c", "|", "b", ...gap(40)];
+ const out = alignWordsOnEmissions(
+ [word("a", 0.3, 0.32, 0.31), word("42", 0.32, 0.4, 0.35), word("b", 0.4, 0.5, 0.42)],
+ speech,
+ emissions(tokens),
+ );
+ expect(out[1].startSec).toBeGreaterThanOrEqual(out[0].endSec);
+ expect(out[1].endSec).toBeLessThanOrEqual(out[2].startSec);
+ expect(r(out[1].startSec)).toBeLessThan(0.34);
+ expect(r(out[2].startSec)).toBeLessThan(0.42);
+ });
+
+ it("puts punctuation on the end of the word before it", () => {
+ const tokens = [...gap(15), "a", "b", ...gap(40)];
+ const out = alignWordsOnEmissions(
+ [word("ab", 0.3, 0.5, 0.35), word("!", 0.5, 0.6, 0.55)],
+ speech,
+ emissions(tokens),
+ );
+ expect(out[1].startSec).toBe(out[0].endSec);
+ expect(out[1].endSec).toBe(out[0].endSec);
+ });
+
+ it("keeps the helper's times where it cannot align", () => {
+ const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)];
+ // The second word is outside every stretch; the first one's letters do not fit.
+ const out = alignWordsOnEmissions(words, speech, emissions(gap(2)));
+ expect(out).toEqual(words);
+ // No region covers the stretch at all.
+ expect(alignWordsOnEmissions(words, speech, emissions(gap(60), 5))).toEqual(words);
+ });
+});
+
+describe("parseEmissions", () => {
+ const floats = new Float32Array([-0.1, -2, -3, -4, -5, -6, -7]);
+ const logprobs = Buffer.from(floats.buffer).toString("base64");
+
+ it("decodes the helper's base64 log-probs", () => {
+ const parsed = parseEmissions({
+ vocab: VOCAB,
+ blank: 0,
+ stride_s: 0.02,
+ regions: [{ start: 1.5, frames: 1, logprobs }],
+ });
+ expect(parsed?.regions[0].startSec).toBe(1.5);
+ expect([...(parsed?.regions[0].logprobs ?? [])]).toEqual([...floats]);
+ });
+
+ it("refuses a reply whose sizes do not add up", () => {
+ expect(
+ parseEmissions({
+ vocab: VOCAB,
+ blank: 0,
+ stride_s: 0.02,
+ regions: [{ start: 0, frames: 2, logprobs }],
+ }),
+ ).toBeNull();
+ });
+});
diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts
new file mode 100644
index 000000000..f243dfe88
--- /dev/null
+++ b/electron/stt/ctcAlign.ts
@@ -0,0 +1,291 @@
+// Second alignment pass: re-times whisper's words on a CTC acoustic model
+// (issue #948, phase 3).
+//
+// whisper's DTW times words within ~30 ms median inside continuous speech, but
+// its 20 ms frames and diffuse cross-attention cap it there (P90 ~110 ms). A
+// wav2vec2 model fine-tuned for CTC scores every 20 ms frame against every
+// letter, so a Viterbi pass that forces whisper's own words through those
+// scores puts each letter where the audio has it. The helper computes the
+// scores (`/emissions`, electron/native/whisper-stt/src/ctc_aligner.cpp); this
+// file spells whisper's words in the model's letters, aligns them, and maps the
+// letters back to word times.
+//
+// Words the model cannot spell (digits, symbols, another script) become one
+// wildcard token that matches any speech, so they hold their place and their
+// neighbours stay aligned. A stretch the model cannot fit keeps the times it
+// came with, and so does every word when there is no aligner for the language.
+//
+// Self-contained on purpose: tools/stt-eval/word-timing imports this file
+// straight into Node, which resolves no extensionless import.
+
+import type { SttVadSegment } from "./transcriptionContract";
+
+/** What `/emissions` answers, decoded. */
+export interface CtcEmissions {
+ vocab: string[];
+ blank: number;
+ /** Seconds between two frames. */
+ strideSec: number;
+ /** Per requested region: its first frame's time and `frames * vocab` log-probs. */
+ regions: Array<{ startSec: number; frames: number; logprobs: Float32Array }>;
+}
+
+/** The fields of a helper word this pass reads and rewrites. */
+export interface AlignableWord {
+ word: string;
+ startSec: number;
+ endSec: number;
+ /** Always inside the word; decides which stretch of speech owns it. */
+ anchorSec: number;
+}
+
+// Calibration, fitted on tools/stt-eval/word-timing (TTS and LibriSpeech agree).
+// The model is sure of a letter a little after the sound starts and before it
+// ends, so CTC shrinks words: the edges move outward. Inside continuous speech
+// the word boundary lies between the last letter of one word and the first of
+// the next; where the two estimates cross, the boundary is their midpoint.
+/** Added to the frame of a word's first letter. */
+export const START_OFFSET_SEC = -0.045;
+/** Added to the end of the frame of a word's last letter. */
+export const END_OFFSET_SEC = 0.025;
+/** A gap between two words' letters this long is a pause, not a boundary. */
+export const PAUSE_SEC = 0.1;
+/** After a pause, a word's first letter trails its onset further (breath, closure, soft attack). */
+export const PAUSE_START_OFFSET_SEC = -0.06;
+/** Before a pause, a word decays past its last letter; early there is audible, late is silence. */
+export const PAUSE_END_OFFSET_SEC = 0.15;
+
+/** Audio kept on each side of a stretch of speech, so its first and last letters have context. */
+export const REGION_MARGIN_SEC = 0.3;
+
+/** As in snapWordBoundaries.ts: a stretch owns the words anchored before its end plus this tail. */
+const TAIL_SEC = 0.1;
+
+const WILDCARD = -1;
+
+/** Seconds of the upload the helper should score: the speech plus a margin, merged. */
+export function emissionRegions(
+ speech: SttVadSegment[],
+ durationSec: number,
+): Array<[number, number]> {
+ const out: Array<[number, number]> = [];
+ for (const s of speech) {
+ const a = Math.max(0, s.startSec - REGION_MARGIN_SEC);
+ const b = Math.min(durationSec, s.endSec + REGION_MARGIN_SEC);
+ const last = out[out.length - 1];
+ if (last && a <= last[1]) last[1] = Math.max(last[1], b);
+ else if (b > a) out.push([a, b]);
+ }
+ return out;
+}
+
+/**
+ * A word as the model's token ids: `[]` when there is nothing to say
+ * (punctuation), `null` when a letter is not in the vocabulary (digits, symbols,
+ * another script). Case follows the vocabulary, an accented letter the
+ * vocabulary lacks falls back to its base letter, and the apostrophe is
+ * normalised.
+ */
+export function wordTokens(word: string, index: Map<string, number>): number[] | null {
+ const upper = index.has("A") && !index.has("a");
+ let text = word.normalize("NFC").replace(/[’‘`´]/g, "'");
+ text = upper ? text.toUpperCase() : text.toLowerCase();
+ const out: number[] = [];
+ for (const ch of text) {
+ const id = index.get(ch);
+ if (id !== undefined) {
+ out.push(id);
+ continue;
+ }
+ if (/\p{S}/u.test(ch)) return null; // "%", "+", "€": said aloud, but not spelled so
+ if (/[\p{P}\s]/u.test(ch)) continue; // silent, apostrophes and hyphens included
+ const base = [...ch.normalize("NFD").replace(/\p{M}/gu, "")];
+ const ids = base.map((c) => index.get(c));
+ if (ids.length === 0 || ids.some((x) => x === undefined)) return null;
+ out.push(...(ids as number[]));
+ }
+ return out;
+}
+
+/**
+ * CTC Viterbi forced alignment of `labels` over `T` frames of `V` log-probs.
+ * `WILDCARD` scores as the frame's best non-blank token. Returns, per label, the
+ * first and last frame the best path spends on it, or null when the labels
+ * cannot fit in the frames.
+ */
+export function ctcViterbi(
+ logprobs: Float32Array,
+ T: number,
+ V: number,
+ blank: number,
+ labels: number[],
+): Array<[number, number]> | null {
+ const L = labels.length;
+ const S = 2 * L + 1;
+ if (L === 0 || T === 0) return null;
+ const wild = new Float32Array(T);
+ if (labels.includes(WILDCARD)) {
+ for (let t = 0; t < T; t++) {
+ let best = Number.NEGATIVE_INFINITY;
+ for (let v = 0; v < V; v++) if (v !== blank) best = Math.max(best, logprobs[t * V + v]);
+ wild[t] = best;
+ }
+ }
+ const lab = (s: number) => (s % 2 === 0 ? blank : labels[(s - 1) >> 1]);
+ const em = (t: number, s: number) => {
+ const l = lab(s);
+ return l === WILDCARD ? wild[t] : logprobs[t * V + l];
+ };
+ const NEG = Number.NEGATIVE_INFINITY;
+ let prev = new Float64Array(S).fill(NEG);
+ let cur = new Float64Array(S);
+ const back = new Uint8Array(T * S); // how far back the best predecessor is: 0, 1 or 2
+ prev[0] = em(0, 0);
+ prev[1] = em(0, 1);
+ for (let t = 1; t < T; t++) {
+ for (let s = 0; s < S; s++) {
+ let best = prev[s];
+ let from = 0;
+ if (s >= 1 && prev[s - 1] > best) {
+ best = prev[s - 1];
+ from = 1;
+ }
+ // A label may follow the one before the blank directly, unless it repeats it.
+ if (s >= 2 && s % 2 === 1 && lab(s) !== lab(s - 2) && prev[s - 2] > best) {
+ best = prev[s - 2];
+ from = 2;
+ }
+ cur[s] = best === NEG ? NEG : best + em(t, s);
+ back[t * S + s] = from;
+ }
+ [prev, cur] = [cur, prev];
+ }
+ let s = prev[S - 2] > prev[S - 1] ? S - 2 : S - 1;
+ if (prev[s] === NEG) return null;
+ const spans: Array<[number, number]> = labels.map(() => [-1, -1]);
+ for (let t = T - 1; t >= 0; t--) {
+ if (s % 2 === 1) {
+ const k = (s - 1) >> 1;
+ spans[k][0] = t;
+ if (spans[k][1] < 0) spans[k][1] = t;
+ }
+ s -= back[t * S + s];
+ }
+ return spans;
+}
+
+/**
+ * Re-time `words` on `emissions`. Each speech stretch owns the words anchored
+ * in it (the rule of snapWordBoundaries.ts); they are aligned on the frames
+ * between the neighbouring stretches, with the vocabulary's word delimiter
+ * between words when it has one. Words outside every stretch, stretches the
+ * model cannot fit, and a vocabulary with no word in it keep the helper's
+ * times. Punctuation collapses onto the end of the word before it.
+ */
+export function alignWordsOnEmissions<W extends AlignableWord>(
+ words: W[],
+ speech: SttVadSegment[],
+ emissions: CtcEmissions,
+): W[] {
+ const out = words.map((w) => ({ ...w }));
+ const index = new Map(emissions.vocab.map((t, i) => [t, i]));
+ const delimiter = index.get("|");
+ const V = emissions.vocab.length;
+ const { strideSec } = emissions;
+ let k = 0;
+ for (let i = 0; i < speech.length; i++) {
+ const { startSec: onset, endSec: offset } = speech[i];
+ const until = Math.min(offset + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY);
+ const owned: number[] = [];
+ for (; k < out.length && words[k].anchorSec < until; k++) owned.push(k);
+ const region = emissions.regions.find(
+ (r) => r.startSec <= onset && r.startSec + r.frames * strideSec >= offset,
+ );
+ if (!region || owned.length === 0) continue;
+ // The frames of this stretch: its margin, stopping at the neighbours' speech.
+ const lo = Math.max(
+ region.startSec,
+ (speech[i - 1]?.endSec ?? Number.NEGATIVE_INFINITY) + TAIL_SEC,
+ onset - REGION_MARGIN_SEC,
+ );
+ const hi = Math.min(
+ region.startSec + region.frames * strideSec,
+ speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY,
+ offset + REGION_MARGIN_SEC,
+ );
+ const f0 = Math.max(0, Math.ceil((lo - region.startSec) / strideSec));
+ const f1 = Math.min(region.frames, Math.floor((hi - region.startSec) / strideSec));
+ if (f1 <= f0) continue;
+
+ const labels: number[] = [];
+ const timed: Array<{ j: number; first: number; last: number }> = [];
+ for (const j of owned) {
+ const toks = wordTokens(out[j].word, index);
+ if (toks?.length === 0) continue;
+ if (labels.length > 0 && delimiter !== undefined) labels.push(delimiter);
+ timed.push({ j, first: labels.length, last: labels.length + (toks?.length ?? 1) - 1 });
+ labels.push(...(toks ?? [WILDCARD]));
+ }
+ const spans = ctcViterbi(
+ region.logprobs.subarray(f0 * V, f1 * V),
+ f1 - f0,
+ V,
+ emissions.blank,
+ labels,
+ );
+ if (!spans) continue;
+ const at = (f: number) => region.startSec + (f0 + f) * strideSec;
+ const s = timed.map((w) => at(spans[w.first][0]));
+ const e = timed.map((w) => at(spans[w.last][1] + 1));
+ timed.forEach(({ j }, q) => {
+ const prevEnd = e[q - 1] ?? Number.NEGATIVE_INFINITY;
+ const nextStart = s[q + 1] ?? Number.POSITIVE_INFINITY;
+ out[j].startSec =
+ s[q] - prevEnd >= PAUSE_SEC
+ ? Math.max(s[q] + PAUSE_START_OFFSET_SEC, (prevEnd + s[q]) / 2)
+ : s[q] + START_OFFSET_SEC;
+ out[j].endSec =
+ nextStart - e[q] >= PAUSE_SEC
+ ? Math.min(e[q] + PAUSE_END_OFFSET_SEC, (e[q] + nextStart) / 2)
+ : e[q] + END_OFFSET_SEC;
+ const before = q > 0 ? out[timed[q - 1].j] : null;
+ if (before && before.endSec > out[j].startSec) {
+ const mid = (before.endSec + out[j].startSec) / 2;
+ before.endSec = mid;
+ out[j].startSec = mid;
+ }
+ });
+ for (let n = 1; n < owned.length; n++) {
+ const j = owned[n];
+ if (timed.some((w) => w.j === j)) continue;
+ out[j].startSec = out[owned[n - 1]].endSec;
+ out[j].endSec = out[j].startSec;
+ }
+ }
+ return out;
+}
+
+/** The helper's `/emissions` JSON. */
+export interface EmissionsJson {
+ vocab: string[];
+ blank: number;
+ stride_s: number;
+ regions: Array<{ start: number; frames: number; logprobs: string }>;
+ /** Seconds the helper spent, model load included. */
+ elapsed_s?: number;
+}
+
+/** Decodes the helper's `/emissions` JSON; null when it is not one. */
+export function parseEmissions(json: EmissionsJson): CtcEmissions | null {
+ if (!Array.isArray(json?.vocab) || !Array.isArray(json.regions) || !(json.stride_s > 0))
+ return null;
+ const V = json.vocab.length;
+ const regions: CtcEmissions["regions"] = [];
+ for (const r of json.regions) {
+ // Copied: a pooled Buffer can start on an offset Float32Array refuses.
+ const bytes = Uint8Array.from(Buffer.from(r.logprobs, "base64"));
+ if (bytes.byteLength !== r.frames * V * 4) return null;
+ regions.push({ startSec: r.start, frames: r.frames, logprobs: new Float32Array(bytes.buffer) });
+ }
+ return { vocab: json.vocab, blank: json.blank, strideSec: json.stride_s, regions };
+}
diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts
index 69ef1723b..8b6d26c53 100644
--- a/electron/stt/index.test.ts
+++ b/electron/stt/index.test.ts
@@ -34,6 +34,7 @@ vi.mock("./whisperServer", () => {
vi.mock("./modelManager", () => ({
ensureModels: vi.fn(async () => undefined),
+ ensureAligner: vi.fn(async ({ language }: { language: string }) => `/fake/${language}.gguf`),
modelPaths: (base: string) => ({
whisper: `${base}/whisper-ggml/ggml-small-q8_0.bin`,
}),
@@ -470,6 +471,41 @@ describe("SttManager", () => {
expect(fakeWhisperServer.start).toHaveBeenCalledOnce();
});
+ it("hands the chunks an aligner, tried once per run when its download fails", async () => {
+ const { ensureAligner } = await import("./modelManager");
+ const mocked = vi.mocked(ensureAligner);
+ mocked.mockClear();
+ mocked.mockRejectedValueOnce(new Error("offline"));
+ // Each chunk asks for the aligner of the language it detected, as the helper does.
+ const asked: Array<string | null> = [];
+ fakeWhisperServer.transcribe.mockImplementation(
+ async (opts: { alignerFor?: (language: string) => Promise<string | null> }) => {
+ asked.push((await opts.alignerFor?.("fr")) ?? null);
+ return {
+ segments: [],
+ wordSegments: [],
+ detectedLanguage: "fr",
+ backend: "whispercpp-cpu" as const,
+ };
+ },
+ );
+ const mgr = new SttManager();
+ await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" });
+ const long = new Float32Array(16_000 * 400);
+ await mgr.transcribe({ samples: long, language: "fr" });
+ // Failed once: the rest of the run keeps whisper's times without retrying.
+ expect(asked.length).toBeGreaterThan(1);
+ expect(asked.every((x) => x === null)).toBe(true);
+ expect(mocked).toHaveBeenCalledOnce();
+
+ // The next run tries again, and a success is kept for the session.
+ asked.length = 0;
+ await mgr.transcribe({ samples: long, language: "fr" });
+ await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" });
+ expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true);
+ expect(mocked).toHaveBeenCalledTimes(2);
+ });
+
it("fans status out to every sink, and detaching one leaves the others", async () => {
const mgr = new SttManager();
const a = vi.fn<(e: SttStatusEvent) => void>();
diff --git a/electron/stt/index.ts b/electron/stt/index.ts
index 4ee2b24c9..eae7f8a97 100644
--- a/electron/stt/index.ts
+++ b/electron/stt/index.ts
@@ -2,7 +2,7 @@ import path from "node:path";
import { app, type IpcMain } from "electron";
import { planChunks } from "./chunking";
import { extractMono16kPcm } from "./extractAudio";
-import { ensureModels, modelPaths } from "./modelManager";
+import { ensureAligner, ensureModels, modelPaths } from "./modelManager";
import type {
SttPhraseSegment,
SttStatusEvent,
@@ -105,6 +105,38 @@ export class SttManager {
*/
private cancelEpoch = 0;
+ /**
+ * The word aligner per language (ctcAlign.ts), resolved once per session: its
+ * file is hashed on the way in, and that is not free at 350 MB. A language
+ * whose download failed is left out for the rest of the run rather than
+ * retried per chunk, and tried again on the next one.
+ */
+ private readonly aligners = new Map<string, Promise<string | null>>();
+ private readonly alignersFailed = new Set<string>();
+
+ private readonly alignerFor = (language: string): Promise<string | null> => {
+ if (this.alignersFailed.has(language)) return Promise.resolve(null);
+ let resolved = this.aligners.get(language);
+ if (!resolved) {
+ resolved = ensureAligner({
+ baseDir: this.getModelsDir(),
+ language,
+ onProgress: (downloadedBytes, totalBytes) =>
+ this.emit({ phase: "model", model: "ctc-aligner", downloadedBytes, totalBytes }),
+ }).catch((error: unknown) => {
+ console.warn(
+ `[stt] no word aligner for "${language}", keeping whisper's word times: ` +
+ `${error instanceof Error ? error.message : String(error)}`,
+ );
+ this.alignersFailed.add(language);
+ this.aligners.delete(language);
+ return null;
+ });
+ this.aligners.set(language, resolved);
+ }
+ return resolved;
+ };
+
/**
* The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is
* checked between chunks — so a cancel during the decode left ffmpeg running to
@@ -235,7 +267,7 @@ export class SttManager {
for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) {
if (this.shuttingDown) throw cancelledError();
try {
- return await this.server.transcribe({ samples, language });
+ return await this.server.transcribe({ samples, language, alignerFor: this.alignerFor });
} catch (error) {
lastError = error;
if (this.shuttingDown) throw cancelledError();
@@ -273,6 +305,7 @@ export class SttManager {
await this.init();
const epoch = this.cancelEpoch;
+ this.alignersFailed.clear();
// Extraction is part of the run, and on a long file it is the part the user used
// to watch the editor freeze through. Doing it here means the renderer hands over
// a path and gets segments back, holding none of the audio. No new status phase:
diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.ts
index 441906234..0676b1a83 100644
--- a/electron/stt/modelManager.test.ts
+++ b/electron/stt/modelManager.test.ts
@@ -4,7 +4,15 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { areModelsPresent, ensureModels, modelPaths, STT_MODELS } from "./modelManager";
+import {
+ alignerPath,
+ areModelsPresent,
+ CTC_ALIGNERS,
+ ensureAligner,
+ ensureModels,
+ modelPaths,
+ STT_MODELS,
+} from "./modelManager";
describe("modelManager", () => {
let dir: string;
@@ -254,4 +262,52 @@ describe("modelManager", () => {
STT_MODELS.whisper.files[0].expectedSha256 = originalSha;
}
});
+
+ describe("CTC aligners", () => {
+ it("pins every aligner to a digest and names its upstream revision", () => {
+ for (const file of Object.values(CTC_ALIGNERS)) {
+ expect(file.expectedSha256).toMatch(/^[0-9a-f]{64}$/);
+ expect(file.source).toMatch(/@[0-9a-f]{40}$/);
+ // Hosted under a `v0.0.0-*` release tag, never a moving one.
+ expect(file.url).toMatch(/\/releases\/download\/v0\.0\.0-[^/]+\//);
+ }
+ });
+
+ it("has none for a language it does not cover, prototype keys included", async () => {
+ expect(alignerPath(dir, "de")).toBeNull();
+ expect(alignerPath(dir, "constructor")).toBeNull();
+ expect(await ensureAligner({ baseDir: dir, language: "de" })).toBeNull();
+ });
+
+ it("downloads, verifies and returns the aligner of a covered language", async () => {
+ const bytes = Buffer.from("gguf weights");
+ const original = CTC_ALIGNERS.fr.expectedSha256;
+ CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex");
+ const progress: number[] = [];
+ try {
+ const file = await ensureAligner({
+ baseDir: dir,
+ language: "fr",
+ fetcher: async () => new Response(bytes, { status: 200 }),
+ onProgress: (done) => progress.push(done),
+ });
+ expect(file).toBe(alignerPath(dir, "fr"));
+ expect(await readFile(file as string)).toEqual(bytes);
+ expect(progress[progress.length - 1]).toBe(bytes.length);
+ } finally {
+ CTC_ALIGNERS.fr.expectedSha256 = original;
+ }
+ });
+
+ it("refuses a download whose digest does not match", async () => {
+ await expect(
+ ensureAligner({
+ baseDir: dir,
+ language: "en",
+ fetcher: async () => new Response("tampered", { status: 200 }),
+ }),
+ ).rejects.toThrow(/SHA-256 mismatch/);
+ expect(existsSync(alignerPath(dir, "en") as string)).toBe(false);
+ });
+ });
});
diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts
index bc37a0359..05d6c5c8e 100644
--- a/electron/stt/modelManager.ts
+++ b/electron/stt/modelManager.ts
@@ -22,7 +22,9 @@ import { pipeline } from "node:stream/promises";
*
* Word timestamps come from whisper.cpp's native DTW token timestamps. The
* Silero VAD model only decides which audio whisper decodes, and gives the
- * speech edges phrases are anchored on. See `technical-documentation/architecture/transcription-and-captions.md`.
+ * speech edges phrases are anchored on. The CTC aligners (`CTC_ALIGNERS`) re-time
+ * the words for the languages that have one, and are fetched only when a
+ * transcription detects such a language. See `technical-documentation/architecture/transcription-and-captions.md`.
*/
export type SttModelId = "whisper" | "silero-vad";
@@ -94,6 +96,71 @@ export const STT_MODELS: Record<SttModelId, SttModelDescriptor> = {
},
};
+/**
+ * The CTC aligners that re-time whisper's words (electron/stt/ctcAlign.ts), per
+ * language whisper reports. A language missing here keeps whisper's DTW times.
+ *
+ * Each file is a wav2vec2 CTC model converted to GGUF by
+ * `scripts/convert-wav2vec2-gguf.mjs` (deterministic: re-running it on the source
+ * named in `source` reproduces the digest below), then published under a `v0.0.0-*`
+ * release tag like the other binaries that need a permanent URL but are not a
+ * product version (see scripts/fetch-onnxruntime.mjs). Apache-2.0, both of them.
+ */
+const ALIGNER_RELEASE =
+ "https://github.com/getopenscreen/openscreen/releases/download/v0.0.0-ctc-aligners-1";
+
+export interface CtcAlignerFile extends SttModelFile {
+ /** HuggingFace repo and revision the file was converted from. */
+ source: string;
+}
+
+export const CTC_ALIGNERS: Record<string, CtcAlignerFile> = {
+ en: {
+ name: "w2v-en-base-q8_0.gguf",
+ url: `${ALIGNER_RELEASE}/w2v-en-base-q8_0.gguf`,
+ expectedSha256: "b7f21a97208f368d3505bd9a7bc9ff3795b1169d8e3b036028082eb25eb464ae",
+ approximateBytes: 109_040_064,
+ source: "facebook/wav2vec2-base-960h@22aad52d435eb6dbaf354bdad9b0da84ce7d6156",
+ },
+ fr: {
+ name: "w2v-fr-large-q8_0.gguf",
+ url: `${ALIGNER_RELEASE}/w2v-fr-large-q8_0.gguf`,
+ expectedSha256: "e3c284da3e27564db07ac226bf6402a4d7856b806455b3e0f5283f29f9495f48",
+ approximateBytes: 348_037_120,
+ // The repo's safetensors conversion PR: main only has pytorch_model.bin.
+ source: "jonatasgrosman/wav2vec2-large-xlsr-53-french@70db24a266633ffcc8edce4e72f3a5cb69d602d6",
+ },
+};
+
+const ALIGNER_DIR = "ctc-aligner";
+
+/** Where the aligner for `language` lives, or null when there is none for it. */
+export function alignerPath(baseDir: string, language: string): string | null {
+ const file = Object.keys(CTC_ALIGNERS).includes(language) ? CTC_ALIGNERS[language] : null;
+ return file ? path.join(baseDir, ALIGNER_DIR, file.name) : null;
+}
+
+/**
+ * Make sure the aligner for `language` is on disk, downloading and verifying it
+ * like the whisper model. Null when the language has none; throws when the
+ * download fails, which the caller turns into "keep whisper's times".
+ */
+export async function ensureAligner(opts: {
+ baseDir: string;
+ language: string;
+ onProgress?: (downloadedBytes: number, totalBytes: number) => void;
+ fetcher?: typeof fetch;
+}): Promise<string | null> {
+ const filePath = alignerPath(opts.baseDir, opts.language);
+ if (!filePath) return null;
+ const file = CTC_ALIGNERS[opts.language];
+ await ensureFile(filePath, file.url, file.expectedSha256, {
+ onProgress: (bytes) => opts.onProgress?.(bytes, file.approximateBytes),
+ fetcher: opts.fetcher,
+ });
+ return filePath;
+}
+
export function modelPaths(baseDir: string): Record<SttModelId, string> {
return {
whisper: path.join(baseDir, STT_MODELS.whisper.cacheDir, MODEL_FILE),
diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts
index 5433698ac..4815da53b 100644
--- a/electron/stt/transcriptionContract.ts
+++ b/electron/stt/transcriptionContract.ts
@@ -71,7 +71,7 @@ export interface SttStatusEvent {
/** Total bytes for the in-flight download. */
totalBytes?: number;
/** Which model is downloading. */
- model?: "whisper" | "silero-vad";
+ model?: "whisper" | "silero-vad" | "ctc-aligner";
/**
* Seconds of audio transcribed so far, and the total for this request. Only
* when `phase === "transcribe"`. Progress is reported per CHUNK (see
diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts
index 62dd7442d..3f495a807 100644
--- a/electron/stt/whisperServer.test.ts
+++ b/electron/stt/whisperServer.test.ts
@@ -382,6 +382,102 @@ describe("WhisperServerManager", () => {
}
});
+ describe("CTC word aligner", () => {
+ const inference = {
+ segments: [
+ {
+ text: " ab ba",
+ start: 0.2,
+ end: 1,
+ words: [
+ { word: " ab", start: 0.3, end: 0.5, anchor: 0.4 },
+ { word: " ba", start: 0.5, end: 0.8, anchor: 0.6 },
+ ],
+ },
+ ],
+ speech: [{ start: 0.2, end: 1 }],
+ detected_language: "fr",
+ backend: "whispercpp-vulkan",
+ timing: { elapsed_s: 0.5, audio_s: 2, rtf: 0.25 },
+ };
+ // "ab" spoken on frames 20-21, the word delimiter on 22, "ba" on 24-25.
+ const vocab = ["<pad>", "|", "a", "b"];
+ const frames = Array<number>(60).fill(0);
+ frames[20] = 2;
+ frames[21] = 3;
+ frames[22] = 1;
+ frames[24] = 3;
+ frames[25] = 2;
+ const logprobs = new Float32Array(60 * vocab.length).fill(-12);
+ frames.forEach((tok, f) => {
+ logprobs[f * vocab.length + tok] = -0.01;
+ });
+ const emissions = {
+ vocab,
+ blank: 0,
+ stride_s: 0.02,
+ elapsed_s: 0.1,
+ regions: [
+ { start: 0, frames: 60, logprobs: Buffer.from(logprobs.buffer).toString("base64") },
+ ],
+ };
+
+ async function run(emissionsReply: Response) {
+ const fetchMock = vi.fn(async (url: string, _init?: RequestInit) =>
+ url.endsWith("/emissions")
+ ? emissionsReply
+ : new Response(JSON.stringify(inference), { status: 200 }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+ try {
+ const mgr = new WhisperServerManager();
+ (mgr as unknown as { process: unknown; port: number }).process = {};
+ (mgr as unknown as { process: unknown; port: number }).port = 9999;
+ const alignerFor = vi.fn(async (language: string) => `/models/${language}.gguf`);
+ const result = await mgr.transcribe({
+ samples: new Float32Array(16_000 * 2),
+ alignerFor,
+ });
+ return { result, alignerFor, fetchMock };
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ }
+
+ it("re-times the words on the aligner of the detected language, and counts its cost", async () => {
+ const { result, alignerFor, fetchMock } = await run(
+ new Response(JSON.stringify(emissions), { status: 200 }),
+ );
+ expect(alignerFor).toHaveBeenCalledWith("fr");
+ const body = fetchMock.mock.calls[1][1]?.body as FormData;
+ expect(body.get("model")).toBe("/models/fr.gguf");
+ expect(JSON.parse(String(body.get("regions")))).toEqual([[0, 1.3]]);
+ // The phrase edges stay on the speech; the inner boundary moves to the letters.
+ const ms = (x: number) => Math.round(x * 1000);
+ expect(result.wordSegments.map((w) => [w.word, ms(w.startSec), ms(w.endSec)])).toEqual([
+ ["ab", 200, 450],
+ ["ba", 450, 1000],
+ ]);
+ expect(result.timing?.elapsedSec).toBeCloseTo(0.6);
+ expect(result.timing?.rtf).toBeCloseTo(0.3);
+ });
+
+ it("keeps whisper's times when the aligner fails", async () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ try {
+ const { result } = await run(new Response("not found", { status: 404 }));
+ expect(result.wordSegments.map((w) => [w.word, w.startSec, w.endSec])).toEqual([
+ ["ab", 0.2, 0.5],
+ ["ba", 0.5, 1],
+ ]);
+ expect(result.timing?.elapsedSec).toBe(0.5);
+ expect(warn).toHaveBeenCalledWith(expect.stringMatching(/word aligner failed/));
+ } finally {
+ warn.mockRestore();
+ }
+ });
+ });
+
it("spawns whisper-stt-server with --model", async () => {
const fs = await import("node:fs/promises");
const { spawn } = await import("node:child_process");
diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts
index a03c05449..bea7c25b5 100644
--- a/electron/stt/whisperServer.ts
+++ b/electron/stt/whisperServer.ts
@@ -6,6 +6,12 @@ import os from "node:os";
import path from "node:path";
import type { Readable } from "node:stream";
+import {
+ alignWordsOnEmissions,
+ type EmissionsJson,
+ emissionRegions,
+ parseEmissions,
+} from "./ctcAlign";
import { resolveBinaryPath } from "./gpuDetector";
import { anchorWordsOnSpeech } from "./snapWordBoundaries";
import type {
@@ -76,6 +82,13 @@ export interface WhisperServerStatus {
lastError: string | null;
}
+export interface TranscribeOptions {
+ samples: Float32Array;
+ language?: string;
+ /** The aligner model for a language, or null for none (see `transcribe`). */
+ alignerFor?: (language: string) => Promise<string | null>;
+}
+
/** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */
interface WhisperJsonWord {
word?: string;
@@ -441,6 +454,33 @@ export class WhisperServerManager {
}
}
+ /**
+ * The CTC aligner's letter scores for `regions` of the same upload
+ * (`/emissions`, see ctcAlign.ts). Throws on any failure, including a helper
+ * built before the endpoint existed (404): the caller keeps whisper's times.
+ */
+ private async runEmissions(opts: {
+ wavPath: string;
+ modelPath: string;
+ regions: Array<[number, number]>;
+ }): Promise<EmissionsJson> {
+ const form = new FormData();
+ const fileBuffer = await readFile(opts.wavPath);
+ form.set("file", new Blob([fileBuffer], { type: "audio/wav" }), path.basename(opts.wavPath));
+ form.set("model", opts.modelPath);
+ form.set("regions", JSON.stringify(opts.regions));
+ const res = await fetch(`${this.baseUrl()}/emissions`, {
+ method: "POST",
+ body: form,
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ if (!res.ok) {
+ const text = await res.text().catch(() => "");
+ throw new Error(`whisper-stt-server /emissions HTTP ${res.status}: ${text.slice(0, 512)}`);
+ }
+ return (await res.json()) as EmissionsJson;
+ }
+
private async runMultipartInfer(opts: {
wavPath: string;
language?: string;
@@ -541,8 +581,15 @@ export class WhisperServerManager {
return { elapsedSec, audioSec, rtf };
}
- /** Run one transcription; serializes concurrent callers. */
- async transcribe(opts: { samples: Float32Array; language?: string }): Promise<{
+ /**
+ * Run one transcription; serializes concurrent callers.
+ *
+ * `alignerFor` resolves the CTC aligner model for the language whisper
+ * detected (downloading it if need be), or null when there is none. With one,
+ * the words are re-timed on it before their phrase edges go on the speech
+ * (ctcAlign.ts); any failure there keeps whisper's own times.
+ */
+ async transcribe(opts: TranscribeOptions): Promise<{
segments: SttPhraseSegment[];
wordSegments: SttWordSegment[];
detectedLanguage: string;
@@ -555,7 +602,7 @@ export class WhisperServerManager {
return task;
}
- private async transcribeImpl(opts: { samples: Float32Array; language?: string }): Promise<{
+ private async transcribeImpl(opts: TranscribeOptions): Promise<{
segments: SttPhraseSegment[];
wordSegments: SttWordSegment[];
detectedLanguage: string;
@@ -578,33 +625,57 @@ export class WhisperServerManager {
startSec: this.toSec(s.start, 0),
endSec: this.toSec(s.end, 0),
}));
- // The helper's word times are right inside a phrase but not on its edges,
- // which the transcript editor turns into imprecise trims: put those on the
- // helper's speech intervals (see snapWordBoundaries.ts).
- const wordSegments: SttWordSegment[] = anchorWordsOnSpeech(
- raw
- .flatMap((seg) =>
- (seg.words ?? []).map((w) => {
- const word = (w.word ?? "").trim();
- const startSec = this.toSec(w.start, 0);
- const endSec = this.toSec(w.end, startSec + 0.05);
- const anchorSec = this.toSec(w.anchor, startSec);
- const confidence = typeof w.probability === "number" ? w.probability : undefined;
- return {
- word,
- startSec,
- endSec: Math.max(startSec + 0.02, endSec),
- anchorSec,
- confidence,
- };
- }),
- )
- .filter((w) => w.word.length > 0),
- speech,
- );
const detectedLanguage = json.detected_language ?? json.language ?? "auto";
+ let words = raw
+ .flatMap((seg) =>
+ (seg.words ?? []).map((w) => {
+ const word = (w.word ?? "").trim();
+ const startSec = this.toSec(w.start, 0);
+ const endSec = this.toSec(w.end, startSec + 0.05);
+ const anchorSec = this.toSec(w.anchor, startSec);
+ const confidence = typeof w.probability === "number" ? w.probability : undefined;
+ return {
+ word,
+ startSec,
+ endSec: Math.max(startSec + 0.02, endSec),
+ anchorSec,
+ confidence,
+ };
+ }),
+ )
+ .filter((w) => w.word.length > 0);
+ let alignSec = 0;
+ if (speech?.length && words.length && opts.alignerFor) {
+ try {
+ const modelPath = await opts.alignerFor(detectedLanguage);
+ if (modelPath) {
+ const reply = await this.runEmissions({
+ wavPath,
+ modelPath,
+ regions: emissionRegions(speech, opts.samples.length / 16_000),
+ });
+ const emissions = parseEmissions(reply);
+ if (!emissions) throw new Error("unreadable /emissions reply");
+ words = alignWordsOnEmissions(words, speech, emissions);
+ alignSec = this.toSec(reply.elapsed_s, 0);
+ }
+ } catch (error) {
+ console.warn(
+ `[stt] word aligner failed, keeping whisper's word times: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+ }
+ // Phrase edges go on the helper's speech intervals, aligner or not: the
+ // VAD is the better judge of where speech starts after a pause (see
+ // snapWordBoundaries.ts).
+ const wordSegments: SttWordSegment[] = anchorWordsOnSpeech(words, speech);
const backend = this.toBackend(json.backend);
const timing = this.toTiming(json.timing);
+ // The aligner is part of what this chunk cost.
+ if (timing && alignSec > 0) {
+ timing.elapsedSec += alignSec;
+ timing.rtf = timing.elapsedSec / timing.audioSec;
+ }
return { segments, wordSegments, detectedLanguage, backend, timing };
} finally {
await cleanupWav(wavPath);
diff --git a/scripts/convert-wav2vec2-gguf.mjs b/scripts/convert-wav2vec2-gguf.mjs
new file mode 100644
index 000000000..55e102d75
--- /dev/null
+++ b/scripts/convert-wav2vec2-gguf.mjs
@@ -0,0 +1,225 @@
+// Converts a HuggingFace Wav2Vec2ForCTC checkpoint (model.safetensors +
+// config.json + vocab.json) into the GGUF file whisper-stt-server's CTC aligner
+// loads (electron/native/whisper-stt/src/ctc_aligner.cpp).
+//
+// Usage:
+// node scripts/convert-wav2vec2-gguf.mjs <model.safetensors> <config.json> <vocab.json> \
+// <out.gguf> --languages en[,xx] [--type q8_0|f16]
+//
+// Node stdlib only. What it writes, and why:
+// - Linear weights (attention, feed-forward, projection, lm_head) as Q8_0, the
+// rest of the matrices as F16, norms and biases as F32. Q8_0 is what keeps the
+// French large model under the 350 MB download budget (issue #948). The
+// convolutions must stay F16: the helper multiplies them with F16 columns.
+// - Sources, as pinned in electron/stt/modelManager.ts (CTC_ALIGNERS[].source):
+// en: https://huggingface.co/facebook/wav2vec2-base-960h (model.safetensors)
+// fr: https://huggingface.co/jonatasgrosman/wav2vec2-large-xlsr-53-french,
+// model.safetensors from its refs/pr/2 (main only has pytorch_model.bin)
+// with the config.json and vocab.json of the same revision.
+// - The positional convolution's weight norm is folded (weight = g * v / |v|),
+// so the helper never sees `weight_g` / `weight_v`.
+// - Tensor names lose the `wav2vec2.` prefix; `masked_spec_embed` is dropped.
+// - The config and the vocabulary go in as `w2v.*` metadata.
+// The output is byte-for-byte deterministic for a given input, so its SHA-256 can
+// be pinned in electron/stt/modelManager.ts.
+import fs from "node:fs";
+
+const [stPath, configPath, vocabPath, outPath, ...rest] = process.argv.slice(2);
+if (!outPath) {
+ console.error(
+ "usage: node convert-wav2vec2-gguf.mjs <model.safetensors> <config.json> <vocab.json> <out.gguf> --languages en [--type q8_0|f16]",
+ );
+ process.exit(2);
+}
+const flag = (name, dflt) => {
+ const i = rest.indexOf(name);
+ return i >= 0 ? rest[i + 1] : dflt;
+};
+const linearType = flag("--type", "q8_0");
+const languages = flag("--languages", "").split(",").filter(Boolean);
+if (!languages.length) throw new Error("--languages is required (e.g. --languages en)");
+
+const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
+const vocabMap = JSON.parse(fs.readFileSync(vocabPath, "utf8"));
+const vocab = [];
+for (const [tok, id] of Object.entries(vocabMap)) vocab[id] = tok;
+if (vocab.length !== config.vocab_size || vocab.includes(undefined))
+ throw new Error(`vocab.json has ${vocab.length} ids, config says ${config.vocab_size}`);
+
+// ---- safetensors ----
+const fd = fs.openSync(stPath);
+const lenBuf = Buffer.alloc(8);
+fs.readSync(fd, lenBuf, 0, 8, 0);
+const headerLen = Number(lenBuf.readBigUInt64LE());
+const headerBuf = Buffer.alloc(headerLen);
+fs.readSync(fd, headerBuf, 0, headerLen, 8);
+const header = JSON.parse(headerBuf.toString("utf8"));
+delete header.__metadata__;
+const readF32 = (name) => {
+ const t = header[name];
+ if (!t) throw new Error(`missing tensor ${name}`);
+ if (t.dtype !== "F32") throw new Error(`${name}: expected F32, got ${t.dtype}`);
+ const [b, e] = t.data_offsets;
+ const buf = Buffer.alloc(e - b);
+ fs.readSync(fd, buf, 0, e - b, 8 + headerLen + b);
+ return { shape: t.shape, data: new Float32Array(buf.buffer, buf.byteOffset, (e - b) / 4) };
+};
+
+// ---- f16 / q8_0 ----
+const f32 = new Float32Array(1);
+const u32 = new Uint32Array(f32.buffer);
+function toHalf(v) {
+ f32[0] = v;
+ const x = u32[0];
+ const sign = (x >>> 16) & 0x8000;
+ let exp = ((x >>> 23) & 0xff) - 127 + 15;
+ let mant = x & 0x7fffff;
+ if (exp >= 31) return sign | 0x7c00; // overflow -> inf (never for these weights)
+ if (exp <= 0) {
+ if (exp < -10) return sign;
+ mant |= 0x800000;
+ const shift = 14 - exp;
+ let h = mant >> shift;
+ if ((mant >> (shift - 1)) & 1 && (mant & ((1 << (shift - 1)) - 1) || h & 1)) h++;
+ return sign | h;
+ }
+ let h = sign | (exp << 10) | (mant >> 13);
+ // round to nearest even
+ if (mant & 0x1000 && (mant & 0xfff || h & 1)) h++;
+ return h;
+}
+function encodeF16(data) {
+ const out = Buffer.alloc(data.length * 2);
+ for (let i = 0; i < data.length; i++) out.writeUInt16LE(toHalf(data[i]), i * 2);
+ return out;
+}
+// ggml block_q8_0: f16 scale + 32 int8, scale = amax / 127 (ggml's quantize_row_q8_0_ref).
+function encodeQ8_0(data) {
+ if (data.length % 32) throw new Error("q8_0 needs a multiple of 32 elements");
+ const nb = data.length / 32;
+ const out = Buffer.alloc(nb * 34);
+ for (let b = 0; b < nb; b++) {
+ let amax = 0;
+ for (let i = 0; i < 32; i++) amax = Math.max(amax, Math.abs(data[b * 32 + i]));
+ const d = amax / 127;
+ const id = d ? 1 / d : 0;
+ out.writeUInt16LE(toHalf(d), b * 34);
+ for (let i = 0; i < 32; i++) out.writeInt8(Math.round(data[b * 32 + i] * id), b * 34 + 2 + i);
+ }
+ return out;
+}
+
+const GGML = { f32: 0, f16: 1, q8_0: 8 };
+const tensors = []; // { name, ne (ggml order), type, bytes }
+function add(name, shape, data, type) {
+ const ne = [...shape].reverse();
+ if (type === "q8_0" && ne[0] % 32) type = "f16";
+ const bytes =
+ type === "f32"
+ ? Buffer.from(data.buffer, data.byteOffset, data.byteLength)
+ : type === "f16"
+ ? encodeF16(data)
+ : encodeQ8_0(data);
+ tensors.push({ name, ne, type: GGML[type], bytes });
+}
+
+const isLinear = (n) =>
+ /(attention\.(q|k|v|out)_proj|feed_forward\.(intermediate|output)_dense|feature_projection\.projection|lm_head)\.weight$/.test(
+ n,
+ );
+for (const name of Object.keys(header).sort()) {
+ if (name.endsWith("masked_spec_embed") || name.includes("pos_conv_embed.conv.weight_")) continue;
+ const { shape, data } = readF32(name);
+ const short = name.replace(/^wav2vec2\./, "");
+ if (shape.length === 1) add(short, shape, data, "f32");
+ else add(short, shape, data, isLinear(name) ? linearType : "f16");
+}
+
+// Weight norm with dim=2: one norm per kernel tap k, over (out, in/groups).
+{
+ const g = readF32("wav2vec2.encoder.pos_conv_embed.conv.weight_g").data;
+ const v = readF32("wav2vec2.encoder.pos_conv_embed.conv.weight_v");
+ const [O, I, K] = v.shape;
+ const w = new Float32Array(v.data.length);
+ for (let k = 0; k < K; k++) {
+ let s = 0;
+ for (let o = 0; o < O; o++) for (let i = 0; i < I; i++) s += v.data[(o * I + i) * K + k] ** 2;
+ const scale = g[k] / Math.sqrt(s);
+ for (let o = 0; o < O; o++)
+ for (let i = 0; i < I; i++) w[(o * I + i) * K + k] = v.data[(o * I + i) * K + k] * scale;
+ }
+ add("encoder.pos_conv_embed.conv.weight", v.shape, w, "f16");
+}
+
+// ---- GGUF v3 ----
+const chunks = [];
+const u32b = (v) => {
+ const b = Buffer.alloc(4);
+ b.writeUInt32LE(v);
+ return b;
+};
+const u64b = (v) => {
+ const b = Buffer.alloc(8);
+ b.writeBigUInt64LE(BigInt(v));
+ return b;
+};
+const strb = (s) => {
+ const b = Buffer.from(s, "utf8");
+ return Buffer.concat([u64b(b.length), b]);
+};
+const T = { u32: 4, i32: 5, f32: 6, bool: 7, string: 8, array: 9 };
+const kvs = [];
+const kv = (key, type, value) => kvs.push({ key, type, value });
+kv("general.architecture", "string", "wav2vec2");
+kv("general.alignment", "u32", 32);
+kv("w2v.languages", "array:string", languages);
+kv("w2v.vocab", "array:string", vocab);
+kv("w2v.blank_id", "u32", config.pad_token_id ?? 0);
+kv("w2v.word_delimiter", "string", "|");
+kv("w2v.hidden_size", "u32", config.hidden_size);
+kv("w2v.intermediate_size", "u32", config.intermediate_size);
+kv("w2v.num_hidden_layers", "u32", config.num_hidden_layers);
+kv("w2v.num_attention_heads", "u32", config.num_attention_heads);
+kv("w2v.layer_norm_eps", "f32", config.layer_norm_eps);
+kv("w2v.conv_kernel", "array:u32", config.conv_kernel);
+kv("w2v.conv_stride", "array:u32", config.conv_stride);
+kv("w2v.conv_bias", "bool", !!config.conv_bias);
+kv("w2v.feat_extract_norm", "string", config.feat_extract_norm);
+kv("w2v.stable_layer_norm", "bool", !!config.do_stable_layer_norm);
+kv("w2v.num_conv_pos_embeddings", "u32", config.num_conv_pos_embeddings);
+kv("w2v.num_conv_pos_embedding_groups", "u32", config.num_conv_pos_embedding_groups);
+
+chunks.push(Buffer.from("GGUF"), u32b(3), u64b(tensors.length), u64b(kvs.length));
+for (const { key, type, value } of kvs) {
+ chunks.push(strb(key));
+ if (type.startsWith("array:")) {
+ const et = type.slice(6);
+ chunks.push(u32b(T.array), u32b(T[et]), u64b(value.length));
+ for (const v of value) chunks.push(et === "string" ? strb(v) : u32b(v));
+ continue;
+ }
+ chunks.push(u32b(T[type]));
+ if (type === "string") chunks.push(strb(value));
+ else if (type === "u32") chunks.push(u32b(value));
+ else if (type === "bool") chunks.push(Buffer.from([value ? 1 : 0]));
+ else if (type === "f32") {
+ const b = Buffer.alloc(4);
+ b.writeFloatLE(value);
+ chunks.push(b);
+ }
+}
+const ALIGN = 32;
+const pad = (n) => (ALIGN - (n % ALIGN)) % ALIGN;
+let offset = 0;
+for (const t of tensors) {
+ chunks.push(strb(t.name), u32b(t.ne.length));
+ for (const d of t.ne) chunks.push(u64b(d));
+ chunks.push(u32b(t.type), u64b(offset));
+ offset += t.bytes.length + pad(t.bytes.length);
+}
+let headLen = chunks.reduce((s, c) => s + c.length, 0);
+chunks.push(Buffer.alloc(pad(headLen)));
+for (const t of tensors) chunks.push(t.bytes, Buffer.alloc(pad(t.bytes.length)));
+fs.writeFileSync(outPath, Buffer.concat(chunks));
+headLen = fs.statSync(outPath).size;
+console.log(`${outPath}: ${tensors.length} tensors, ${(headLen / 1e6).toFixed(1)} MB`);
diff --git a/scripts/test-whisper-stt.mjs b/scripts/test-whisper-stt.mjs
index 2a0e11223..470b40793 100644
--- a/scripts/test-whisper-stt.mjs
+++ b/scripts/test-whisper-stt.mjs
@@ -21,12 +21,20 @@
// OPENSCREEN_WHISPER_MODEL GGML model (default: the userData cache location)
// OPENSCREEN_VAD_MODEL Silero VAD model (default: next to the GGML model;
// the speech checks run only when it is there, as in the app)
+// OPENSCREEN_ALIGNER_MODEL CTC word aligner (default: the cached one for the detected
+// language; the aligner checks run only when it is there)
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
+import {
+ alignWordsOnEmissions,
+ emissionRegions,
+ parseEmissions,
+} from "../electron/stt/ctcAlign.ts";
+import { CTC_ALIGNERS } from "../electron/stt/modelManager.ts";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.join(__dirname, "..");
@@ -244,6 +252,8 @@ async function main() {
});
let json;
+ let emissions = null;
+ let alignerModel = null;
try {
await waitForReady(`http://127.0.0.1:${port}/`);
const form = new FormData();
@@ -256,6 +266,26 @@ async function main() {
});
if (!res.ok) throw new Error(`/inference returned ${res.status}: ${await res.text()}`);
json = await res.json();
+ // The word aligner's acoustic pass, on the same helper, as SttManager runs it.
+ const cached = CTC_ALIGNERS[json.detected_language];
+ alignerModel =
+ process.env.OPENSCREEN_ALIGNER_MODEL ??
+ (cached ? path.join(path.dirname(path.dirname(MODEL)), "ctc-aligner", cached.name) : null);
+ if (WITH_VAD && alignerModel && fs.existsSync(alignerModel)) {
+ const speech = (json.speech ?? []).map((x) => ({ startSec: x.start, endSec: x.end }));
+ const aform = new FormData();
+ aform.append("file", new Blob([fs.readFileSync(wavPath)]), path.basename(wavPath));
+ aform.append("model", alignerModel);
+ aform.append("regions", JSON.stringify(emissionRegions(speech, meta.durationSec)));
+ const ares = await fetch(`http://127.0.0.1:${port}/emissions`, {
+ method: "POST",
+ body: aform,
+ });
+ if (!ares.ok) throw new Error(`/emissions returned ${ares.status}: ${await ares.text()}`);
+ emissions = await ares.json();
+ } else {
+ alignerModel = null;
+ }
} finally {
child.kill();
cleanup();
@@ -361,6 +391,76 @@ async function main() {
check(rate <= 0.15, "WER within tolerance", `${rate.toFixed(4)}`);
}
+ // The aligner (electron/stt/ctcAlign.ts) only gets letter scores from the
+ // helper; a forward pass gone wrong on a backend (a missing op, a wrong
+ // layout) still answers 200 with well-formed garbage. Reading its best letter
+ // per frame back as text is what tells the two apart.
+ if (emissions) {
+ const parsed = parseEmissions(emissions);
+ check(parsed !== null, "aligner answers well-formed letter scores", alignerModel);
+ if (parsed) {
+ const V = parsed.vocab.length;
+ let worst = 0;
+ let greedy = "";
+ for (const r of parsed.regions) {
+ let prev = -1;
+ for (let t = 0; t < r.frames; t++) {
+ let sum = 0;
+ let best = 0;
+ for (let v = 0; v < V; v++) {
+ sum += Math.exp(r.logprobs[t * V + v]);
+ if (r.logprobs[t * V + v] > r.logprobs[t * V + best]) best = v;
+ }
+ worst = Math.max(worst, Math.abs(sum - 1));
+ if (best !== prev && best !== parsed.blank)
+ greedy += parsed.vocab[best] === "|" ? " " : parsed.vocab[best];
+ prev = best;
+ }
+ greedy += " ";
+ }
+ check(
+ worst < 1e-3,
+ "aligner frames are log-probabilities",
+ `max |sum - 1| = ${worst.toExponential(1)}`,
+ );
+ if (gpuExpected) {
+ check(
+ emissions.device !== "CPU",
+ "aligner runs on the GPU on a GPU-capable host",
+ emissions.device,
+ );
+ }
+ // Letters, not words: the aligner spells "OpenScreen" as it hears it.
+ // Measured: 0.31 on a real French take, 0.11 on English TTS; a forward
+ // pass gone wrong decodes to noise, near 1.
+ const letters = (x) => normalize(x).join("").split("").join(" ");
+ const heard = wer(letters(text), letters(greedy));
+ console.log(`\naligner: ${greedy.trim().slice(0, 200)}`);
+ check(
+ heard <= 0.5,
+ "aligner hears what whisper heard",
+ `letter error rate ${heard.toFixed(2)} on ${emissions.device}`,
+ );
+ const speech = (json.speech ?? []).map((x) => ({ startSec: x.start, endSec: x.end }));
+ const plain = words.map((w) => ({
+ word: w.word.trim(),
+ startSec: w.start,
+ endSec: w.end,
+ anchorSec: w.anchor ?? w.start,
+ }));
+ const aligned = alignWordsOnEmissions(plain, speech, parsed);
+ check(
+ aligned.every(
+ (w, i) => w.endSec >= w.startSec && (i === 0 || w.startSec >= aligned[i - 1].startSec),
+ ),
+ "aligned words stay ordered and non-empty",
+ );
+ console.log(`aligner: ${emissions.elapsed_s.toFixed(2)}s on ${emissions.device}`);
+ }
+ } else {
+ console.log("\n(no aligner for this language in the cache: aligner checks skipped)");
+ }
+
if (json.timing) {
console.log(
`\ntiming : ${json.timing.elapsed_s?.toFixed?.(2)}s for ` +
diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md
index 41645e23a..35b9187af 100644
--- a/technical-documentation/architecture/transcription-and-captions.md
+++ b/technical-documentation/architecture/transcription-and-captions.md
@@ -112,6 +112,11 @@ renderer imposes a timeout that a slow download could trip: the preload does a
bare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, and
whisper-server's 30 s readiness budget only starts once the download resolved.
+The word aligner (§ Word-level alignment, step 5) is the exception to "first
+run only": it is fetched the first time a transcription detects English or
+French (109 or 348 MB), in the middle of that run, which then simply takes
+longer once more.
+
Three edges make that promise hold, and each is load-bearing:
- **A failed setup is not cached.** `SttManager.init` used to memoise the
@@ -184,14 +189,14 @@ it verbatim in the response.
> `ggml_backend_dev_type()` being GPU/IGPU, which also drops ggml-blas
> (Accelerate on macOS, device type ACCEL) from consideration.
-### Word-level alignment (DTW token timestamps)
+### Word-level alignment (DTW token timestamps, then a CTC aligner)
0. **Speech only** — when the Silero VAD model is on disk (`ggml-silero-v6.2.0.bin`,
downloaded next to the whisper model), the helper cuts the silence out of
the upload before whisper sees it, the way whisper.cpp's own VAD does: each
speech stretch plus 0.1 s of tail, 0.1 s of silence between stretches. It
keeps the map, and every time below goes back through it onto the upload's
- clock. The stretches are returned as `speech` for step 5.
+ clock. The stretches are returned as `speech` for steps 5 and 6.
> **Not whisper_full's `vad` param.** whisper.cpp 1.9.1 maps its *segment*
> times back onto the original audio but not its token times, `t_dtw`
> included, so every word came back early by all the silence removed before
@@ -248,7 +253,7 @@ it verbatim in the response.
first word has none and falls back to its own first token.
- `word.end` = `t_dtw` of the word's **last** token.
- `word.anchor` = `t_dtw` of the word's first token. It always lies inside
- the word, and only decides which stretch of speech owns it (step 5).
+ the word, and only decides which stretch of speech owns it (steps 5 and 6).
The result is a monotonic, gap-free timeline of word ranges on the upload's
clock. Taking the first token's `t_dtw` as the start, as the helper did
@@ -256,7 +261,37 @@ it verbatim in the response.
below, and a median 200 ms on a real French take. The repo's DTW POC saw
the same thing as whisper.cpp's `t_dtw` matching faster-whisper's word
**end** (`tools/stt-eval/whispercpp-dtw-poc/REPORT.md`).
-5. **Anchor phrase edges on the speech** —
+5. **Re-time the words on a CTC aligner** (issue #948, phase 3) — when the
+ language whisper detected has one, a wav2vec2 model fine-tuned for CTC scores
+ every 20 ms frame of the speech against every letter, and
+ [`electron/stt/ctcAlign.ts`](../../electron/stt/ctcAlign.ts) forces
+ whisper's own words through those scores:
+ - **The helper only scores.** `POST /emissions` (same WAV, the aligner's
+ GGUF path, the regions to score: each stretch of speech ±0.3 s, merged)
+ answers the log-probabilities per frame
+ ([`ctc_aligner.cpp`](../../electron/native/whisper-stt/src/ctc_aligner.cpp),
+ a wav2vec2 forward pass on ggml, on the GPU whisper uses). It is a
+ separate request so the stage stays separate: it reads whisper's words,
+ whatever produced their times.
+ - **Spelling.** Each word is lowercased (uppercased for the English model),
+ its apostrophes normalised, its punctuation dropped, and an accent the
+ vocabulary lacks falls back to the base letter. A word that cannot be
+ spelled (digits, `€`, another script) becomes one wildcard token that
+ scores as the frame's best letter, so it holds its place and its
+ neighbours stay aligned. The vocabulary's word delimiter `|` goes between
+ words.
+ - **Viterbi** over the frames of each stretch (the one that owns the words
+ by `anchor`, as in step 6), then calibration: CTC is sure of a letter a
+ little after the sound starts and before it ends, so a word starts 45 ms
+ before its first letter's frame and ends 25 ms after its last one. Where
+ the two estimates cross inside continuous speech, the boundary is their
+ midpoint. Next to a pause (a gap of 100 ms or more between letters) the
+ edges move further out, 60 ms before and 150 ms after, both capped at
+ mid-pause: early there is silence, late is an audible attack.
+ - **Fallback.** No aligner for the language, a download that failed, a
+ helper without `/emissions`, a stretch whose letters do not fit in its
+ frames: the words keep the times of step 4. Step 6 runs either way.
+6. **Anchor phrase edges on the speech** —
[`electron/stt/snapWordBoundaries.ts`](../../electron/stt/snapWordBoundaries.ts)
leaves boundaries inside a phrase alone: they are within ~16 ms of the audio
already, and an energy snap on top of them drags correct boundaries early.
@@ -287,15 +322,29 @@ scores the helper and this post-pass against a Windows-TTS corpus with exact
word times (27 min of French and English narration, clean and noisy):
boundary error by position, and the audible residue and clipping of deleting
one word or one phrase. Issue #948 has the method, the baseline and the plan.
+The same harness scores real read speech: 46 min of LibriSpeech test-clean
+(40 speakers, English) against Montreal Forced Aligner word times, which are
+themselves about 10 to 20 ms from a human's.
-| Pipeline (clean + noisy) | Inner start: median / P90 / within 50 ms | Phrase-initial start within 50 ms | One-word delete: clean cuts | Phrase delete: clean |
+| Pipeline | Inner start: median / P90 / within 50 ms | Phrase-initial start within 50 ms | One-word delete: clean cuts | Phrase delete: clean |
|---|---|---|---|---|
-| First-token start + 150 ms RMS snap + VAD edges (before #948) | 105 / 275 ms / 32% | 83% | 5% | 84% |
-| Previous-token start + two-way VAD edges | 31 / 125 ms / 64% | 89% | 19% | 88% |
-| Same, character-level DTW | 16 / 60 ms / 86% | 89% | 39% | 93% |
-
-Per language, character-level DTW gives French 19 ms median and 83% within
-50 ms (was 26 ms, 70%) and English 15 ms and 89% (was 40 ms, 57%).
+| First-token start + 150 ms RMS snap + VAD edges (before #948), TTS | 105 / 275 ms / 32% | 83% | 5% | 84% |
+| Previous-token start + two-way VAD edges (phase 1), TTS | 31 / 125 ms / 64% | 89% | 19% | 88% |
+| + CTC aligner (phase 3), TTS clean | 14 / 35 ms / 96% | 95% | 50% | 96% |
+| + CTC aligner (phase 3), TTS noisy | 15 / 39 ms / 95% | 93% | 48% | 93% |
+| Phase 1, LibriSpeech | 40 / 145 ms / 55% | 67% | 14% | 71% |
+| + CTC aligner (phase 3), LibriSpeech | 15 / 45 ms / 92% | 73% | 44% | 82% |
+
+With the aligner, deleting one word leaves on average 17 ms of it audible and
+cuts 15 ms into its neighbours (TTS clean; 20 and 13 ms on LibriSpeech), against
+40 and 44 ms before. The misses left on phrase-initial starts are mostly the
+reference's: the French TTS voices start a word on its silent stop closure,
+which no aligner hears. On a real French take (25 s, no reference) checked on
+its spectrogram, the aligner moved six boundaries by more than 40 ms: three
+onto a visible boundary that phase 1 had missed by 65 to 250 ms (e.g.
+"montrer | ça" from inside `montrer` to the onset of the /s/), one the right
+way but still about 150 ms early (a drawn-out "Bah… voilà"), and two between
+vowels, where the spectrogram cannot tell.
A +15 ms calibration offset on every boundary gained 4 points of inner
boundaries within 50 ms but dropped noisy phrase deletes to 82%, so it is not
@@ -340,6 +389,26 @@ once instead of merely breaking new installs; pinning makes the recorded
digest an invariant. Bumping the model therefore means bumping the revision
and the digest together.
+**The word aligners** (step 5) are fetched only for a language that has one,
+the first time a transcription detects it, inside the same run (`phase:
+"model"`, `model: "ctc-aligner"`), into `stt-models/ctc-aligner/`. A failed
+download keeps whisper's times for the rest of the run and is retried on the
+next one.
+
+| Language | Model | Download |
+|---|---|---|
+| English | `facebook/wav2vec2-base-960h` (12 layers, letters) | 109 MB |
+| French | `jonatasgrosman/wav2vec2-large-xlsr-53-french` (24 layers, letters and accents) | 348 MB |
+
+Both are Apache-2.0. `scripts/convert-wav2vec2-gguf.mjs` turns the upstream
+`model.safetensors` into the GGUF the helper loads: linear weights Q8_0 (which
+is what keeps the French model under 350 MB), convolutions F16, the positional
+convolution's weight norm folded. The conversion is deterministic, so the
+SHA-256 pinned in `CTC_ALIGNERS` (`electron/stt/modelManager.ts`) can be
+reproduced from the upstream revision it names. The files are served from a
+`v0.0.0-ctc-aligners-1` release, the convention this repo uses for binaries
+that need a permanent URL but are not a product version.
+
> The HuggingFace identifier is intentionally `ggerganov/whisper.cpp`,
> **not** `ggml-org/whisper.cpp`. The latter matches the GitHub org the
> engine itself now lives under, but on HuggingFace it is a separate,
@@ -352,13 +421,15 @@ and the digest together.
| Module | Role |
|---|---|
| `electron/stt/whisperServer.ts` | Server lifecycle; `POST /inference` client; verbose_json parser. |
-| `electron/stt/snapWordBoundaries.ts` | Anchors phrase edges on the VAD's speech (see step 5 above). |
+| `electron/stt/ctcAlign.ts` | Re-times words on the CTC aligner's letter scores (step 5): spelling, Viterbi, calibration, fallback. |
+| `electron/stt/snapWordBoundaries.ts` | Anchors phrase edges on the VAD's speech (see step 6 above). |
| `electron/stt/wav.ts` | WAV write + temp-file cleanup helpers. |
| `electron/stt/gpuDetector.ts` | Per-platform binary resolver (no GPU probing). |
-| `electron/stt/modelManager.ts` | Single GGML file download, SHA-256 verify, atomic write. |
+| `electron/stt/modelManager.ts` | Model downloads (whisper, Silero VAD, the per-language aligners), SHA-256 verify, atomic write. |
| `electron/stt/transcriptionContract.ts` | Shared IPC types (`SttBackend`, `SttWordSegment`, `SttPhraseSegment`, `SttStatusEvent`). |
| `electron/stt/index.ts` | `SttManager` — IPC entry point; wires the pieces together. |
-| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; cuts the silence out with Silero VAD and maps times back (step 0); calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`. |
+| `electron/native/whisper-stt/src/main.cpp` | httplib HTTP server; cuts the silence out with Silero VAD and maps times back (step 0); calls `whisper_full()` with DTW; reports the device it bound via `ggml_backend_dev_name()`; `POST /emissions` for the aligner. |
+| `electron/native/whisper-stt/src/ctc_aligner.cpp` | wav2vec2-for-CTC forward pass on ggml (base and large layouts), in 20 s windows; loads the GGUF written by `scripts/convert-wav2vec2-gguf.mjs`. |
| `electron/native/whisper-stt/CMakeLists.txt` | Pulls whisper.cpp via FetchContent; enables Metal (macOS arm64), Vulkan (Windows/Linux x64), CPU fallback everywhere; static backend linking into `whisper.dll`/`ggml.dll`. |
The helper is one executable per platform; backends are baked in at build
@@ -818,9 +889,13 @@ it deletes data
- **No C++ unit tests.** The WAV reader and the DTW-inactive guardrail
in `electron/native/whisper-stt/src/main.cpp` are exercised only at
runtime.
-- **Word timing inside a phrase.** Phrase edges sit on the VAD (step 5), but
- a word in the middle of continuous speech is only as good as whisper-small's
- character-level DTW: about 16 ms median and 60 ms P90 on synthetic speech,
- so about two single-word deletes in five are clean. A CTC forced aligner is
- the upgrade path (issue #948 Phase 3); `tools/stt-eval/word-timing`
- measures it.
+- **Word timing without an aligner.** Phrase edges sit on the VAD (step 6),
+ and English and French words are re-timed on a CTC aligner (step 5). Every
+ other language keeps whisper-small's DTW: about 30 ms median and 125 ms P90
+ inside a phrase, so one single-word delete in five is clean. A multilingual
+ CTC model (`facebook/omniASR-CTC-300M`) would cover them, at a cost per
+ language family nobody has measured yet (issue #948).
+- **The aligner on the CPU.** On the GPU it adds about 10% to a transcription.
+ On the CPU fallback (16 threads, Ryzen 7 5800X) it adds 28% in English and
+ 42 to 61% in French: the French model is a 24-layer wav2vec2 large, and its
+ forward pass costs about half of whisper-small's.
diff --git a/tools/stt-eval/word-timing/README.md b/tools/stt-eval/word-timing/README.md
index b49938d6f..9e273aab5 100644
--- a/tools/stt-eval/word-timing/README.md
+++ b/tools/stt-eval/word-timing/README.md
@@ -33,7 +33,9 @@ node validate-ref.mjs # checks the reference against the audio's energy
```sh
node run-helper.mjs <tag> <path/to/whisper-stt-server.exe> [--cpu]
-node evaluate.mjs <tag> [--snap <post-pass.ts>] [--out <name>]
+node run-align.mjs <tag> <tag>-ctc <path/to/whisper-stt-server.exe> \
+ --model en=<w2v-en-base-q8_0.gguf> --model fr=<w2v-fr-large-q8_0.gguf> [--cpu]
+node evaluate.mjs <tag>-ctc [--snap <post-pass.ts>] [--ctc <ctcAlign.ts>] [--out <name>]
node summarize.mjs "Before=<baseline>" "After=<name>"
```
@@ -45,20 +47,43 @@ node summarize.mjs "Before=<baseline>" "After=<name>"
- `evaluate.mjs` parses the responses as `whisperServer.ts` does and runs the
post-pass: the repo's `electron/stt/snapWordBoundaries.ts` by default, or the
file given with `--snap`. It reports three stages: `raw`, `post` (no speech
- intervals) and `post+vad` (what the app ships).
+ intervals) and `post+vad` (what the app ships without an aligner).
+- `run-align.mjs` adds the CTC aligner's letter scores (`/emissions`) to saved
+ responses, without running whisper again, for the language each clip was
+ detected in. On those, `evaluate.mjs` adds `ctc` (the aligner,
+ `electron/stt/ctcAlign.ts` or `--ctc`) and `ctc+vad` (what the app ships with
+ an aligner), and the aligner's share of the runtime.
+- The aligner GGUFs come from `scripts/convert-wav2vec2-gguf.mjs` (sources in
+ its header), or from the app's cache: `stt-models/ctc-aligner/`.
- A baseline is the same two commands on an older helper and post-pass:
build the helper at that revision, and pass
`git show <rev>:electron/stt/snapWordBoundaries.ts` saved to a file as `--snap`.
## Real speech
+**LibriSpeech, against forced-aligned word times** (English read speech; the
+Montreal Forced Aligner reference is itself 10 to 20 ms from a human's):
+
+```sh
+# test-clean.tar.gz from https://www.openslr.org/12, librispeech_alignments.zip
+# from https://zenodo.org/records/2619474 (both CC-BY-4.0, dev-time only)
+export OSC_WORD_TIMING_DATA=<another data dir>
+node make-librispeech.mjs <LibriSpeech/test-clean> <alignments/test-clean> # 2 clips x 40 speakers, ~46 min
+node run-helper.mjs ls <exe> --condition clean
+node run-align.mjs ls ls-ctc <exe> --model en=<w2v-en-base-q8_0.gguf>
+node evaluate.mjs ls-ctc
+```
+
+**A take of your own**, with no ground truth:
+
```sh
-node real-check.mjs <whisper-stt-server.exe> take.wav # 16 kHz mono s16
+node real-check.mjs <whisper-stt-server.exe> take.wav [--align <aligner.gguf>] # 16 kHz mono s16
```
-No ground truth there: it prints how far each word's first-token time lies after
-its start, and where each phrase's first word lands against the VAD onset, raw
-and after the post-pass.
+It prints how far each word's first-token time lies after its start, and where
+each phrase's first word lands against the VAD onset, raw and after the
+post-pass. With `--align`, every word is listed with its phase-1 and aligner
+times, so the boundaries that moved can be checked by ear or on a spectrogram.
## Reading the numbers
@@ -68,3 +93,6 @@ and after the post-pass.
at most 20 ms of its neighbours is cut.
- Synthetic speech flatters every method. Use the harness to rank approaches,
and check a winner on real speech.
+- The aligner's calibration (`START_OFFSET_SEC` and the others in
+ `ctcAlign.ts`) was fitted on the TTS corpus and checked on LibriSpeech; refit
+ both before changing a model.
diff --git a/tools/stt-eval/word-timing/evaluate.mjs b/tools/stt-eval/word-timing/evaluate.mjs
index b923c5d6a..4c7cbe873 100644
--- a/tools/stt-eval/word-timing/evaluate.mjs
+++ b/tools/stt-eval/word-timing/evaluate.mjs
@@ -1,11 +1,13 @@
// Scores helper output against the TTS reference, for each pipeline stage.
-// Usage: node evaluate.mjs <tag> [--snap <post-pass.ts>] [--out <name>] [--quiet]
+// Usage: node evaluate.mjs <tag> [--snap <post-pass.ts>] [--ctc <ctcAlign.ts>] [--out <name>] [--quiet]
// reads <data>/results/raw/<tag>/*.json, writes <data>/results/<out>.json and
// <out>.txt, and prints the table.
// Stages:
// raw helper words, parsed as whisperServer.ts transcribeImpl does
// post the post-pass without speech intervals
// post+vad the post-pass with the helper's `speech`, as the app runs it
+// ctc the CTC aligner's times (when run-align.mjs saved emissions)
+// ctc+vad ctc, then the post-pass with `speech`, as the app runs it with an aligner
// `--snap` swaps the post-pass (default: the repo's snapWordBoundaries.ts), so a
// candidate is scored exactly like the shipped code. It takes the current
// `anchorWordsOnSpeech(words, speech)` or the pre-#948
@@ -24,6 +26,8 @@ const flag = (name, dflt) => {
const snapPath = path.resolve(
flag("--snap", path.join(REPO, "electron/stt/snapWordBoundaries.ts")),
);
+const ctcPath = path.resolve(flag("--ctc", path.join(REPO, "electron/stt/ctcAlign.ts")));
+const ctc = await import(pathToFileURL(ctcPath).href);
const outName = flag("--out", tag);
const quiet = rest.includes("--quiet");
const mod = await import(pathToFileURL(snapPath).href);
@@ -96,7 +100,7 @@ const inside = (env, r0, r1, c0, c1) => {
return b > a ? { sec: b - a, aud: audible(env, a, b) } : { sec: 0, aud: 0 };
};
-const STAGES = ["raw", "post", "post+vad"];
+let STAGES = ["raw", "post", "post+vad"];
const acc = {};
const push = (k, v) => {
acc[k] ??= [];
@@ -105,6 +109,7 @@ const push = (k, v) => {
const counts = {};
const inc = (k, v = 1) => (counts[k] = (counts[k] ?? 0) + v);
const perClip = [];
+let ctcMs = 0;
const rawDir = path.join(RESULTS, "raw", tag);
for (const file of readdirSync(rawDir)
@@ -121,6 +126,14 @@ for (const file of readdirSync(rawDir)
const full = postPass(rawWords, samples, speech);
const tsMs = performance.now() - t0;
const variants = { raw: rawWords, post, "post+vad": full };
+ if (json.emissions && speech) {
+ const t1 = performance.now();
+ const aligned = ctc.alignWordsOnEmissions(rawWords, speech, ctc.parseEmissions(json.emissions));
+ variants.ctc = aligned;
+ variants["ctc+vad"] = postPass(aligned, samples, speech);
+ ctcMs += performance.now() - t1;
+ STAGES = ["raw", "post", "post+vad", "ctc", "ctc+vad"];
+ }
const R = ref.words.map((w, i) => ({ ...w, i, n: norm(w.text) })).filter((w) => w.n);
const H = rawWords.map((w, j) => ({ j, n: norm(w.word) })).filter((w) => w.n);
@@ -143,6 +156,7 @@ for (const file of readdirSync(rawDir)
audioSec: ref.durationSec,
helperSec: json.timing?.elapsed_s,
wallMs: json.wallMs,
+ alignSec: json.emissions?.elapsed_s,
tsMs,
refWords: R.length,
matched: pairs.length,
@@ -291,11 +305,15 @@ const rt = perClip.reduce(
helper: a.helper + (c.helperSec ?? 0),
wall: a.wall + c.wallMs / MS,
ts: a.ts + c.tsMs / MS,
+ align: a.align + (c.alignSec ?? 0),
}),
- { audio: 0, helper: 0, wall: 0, ts: 0 },
+ { audio: 0, helper: 0, wall: 0, ts: 0, align: 0 },
);
lines.push(
- `\nruntime: ${perClip.length} clips, ${(rt.audio / 60).toFixed(1)} min audio; helper ${rt.helper.toFixed(1)} s (RTF ${(rt.helper / rt.audio).toFixed(3)}), per clip mean ${(rt.helper / perClip.length).toFixed(2)} s; TS post-pass ${(rt.ts * MS).toFixed(0)} ms total (${((rt.ts * MS) / perClip.length).toFixed(1)} ms/clip, both stages)`,
+ `\nruntime: ${perClip.length} clips, ${(rt.audio / 60).toFixed(1)} min audio; helper ${rt.helper.toFixed(1)} s (RTF ${(rt.helper / rt.audio).toFixed(3)}), per clip mean ${(rt.helper / perClip.length).toFixed(2)} s; TS post-pass ${(rt.ts * MS).toFixed(0)} ms total (${((rt.ts * MS) / perClip.length).toFixed(1)} ms/clip, both stages)` +
+ (rt.align
+ ? `; aligner emissions ${rt.align.toFixed(1)} s (+${((100 * rt.align) / rt.helper).toFixed(0)}% of the helper), CTC Viterbi ${ctcMs.toFixed(0)} ms`
+ : ""),
);
writeFileSync(path.join(RESULTS, `${outName}.txt`), lines.join("\n") + "\n");
if (!quiet) console.log(lines.join("\n"));
diff --git a/tools/stt-eval/word-timing/make-librispeech.mjs b/tools/stt-eval/word-timing/make-librispeech.mjs
new file mode 100644
index 000000000..b5e51ea91
--- /dev/null
+++ b/tools/stt-eval/word-timing/make-librispeech.mjs
@@ -0,0 +1,125 @@
+// Real-speech corpus for the harness: LibriSpeech test-clean read speech with
+// Montreal Forced Aligner word times as the reference (silver labels, about
+// 10-20 ms from a human's).
+// Usage: OSC_WORD_TIMING_DATA=<another data dir> node make-librispeech.mjs \
+// <LibriSpeech/test-clean> <librispeech_alignments/test-clean> [--max-min 60]
+// -> the same layout as make-corpus.mjs (clips/<id>.wav + <id>.ref.json,
+// corpus-manifest.json), clean condition only, so run-helper.mjs
+// --condition clean, run-align.mjs and evaluate.mjs work unchanged.
+// Consecutive utterances of a chapter are joined into clips of up to 40 s, so a
+// clip reads like narration: phrases separated by the speaker's own pauses.
+// Two clips per speaker (the first chapter), so the corpus spans all 40 voices.
+// Sources (CC-BY-4.0, dev-time only, never shipped): https://www.openslr.org/12
+// (test-clean.tar.gz) and https://zenodo.org/records/2619474.
+import { execFileSync } from "node:child_process";
+import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { CLIPS, DATA, FFMPEG, SR } from "./lib.mjs";
+
+const [audioRoot, alignRoot, ...rest] = process.argv.slice(2);
+if (!alignRoot)
+ throw new Error(
+ "usage: node make-librispeech.mjs <LibriSpeech/test-clean> <alignments/test-clean> [--max-min 60]",
+ );
+const i = rest.indexOf("--max-min");
+const maxSec = (i >= 0 ? Number(rest[i + 1]) : 60) * 60;
+const CLIP_SEC = 40;
+const PER_SPEAKER = 2;
+
+/** Word intervals of a TextGrid's "words" tier; empty text is silence. */
+function textGridWords(file) {
+ const src = readFileSync(file, "utf8");
+ const tier = src.slice(src.indexOf('name = "words"'));
+ const end = tier.indexOf("item [2]");
+ const body = end > 0 ? tier.slice(0, end) : tier;
+ const words = [];
+ for (const m of body.matchAll(/xmin = ([\d.]+)\s+xmax = ([\d.]+)\s+text = "([^"]*)"/g)) {
+ if (m[3].trim()) words.push({ text: m[3].trim(), start: Number(m[1]), end: Number(m[2]) });
+ }
+ return words;
+}
+
+mkdirSync(CLIPS, { recursive: true });
+const clips = [];
+let total = 0;
+outer: for (const spk of readdirSync(alignRoot).sort()) {
+ for (const chap of readdirSync(path.join(alignRoot, spk)).sort()) {
+ const utts = readdirSync(path.join(alignRoot, spk, chap))
+ .filter((f) => f.endsWith(".TextGrid"))
+ .map((f) => f.replace(/\.TextGrid$/, ""))
+ .sort();
+ let group = [];
+ let groupSec = 0;
+ const flush = () => {
+ if (!group.length) return;
+ const id = `ls-${group[0]}`;
+ const pcm = [];
+ const words = [];
+ let at = 0;
+ for (const u of group) {
+ const raw = execFileSync(
+ FFMPEG,
+ [
+ "-v",
+ "error",
+ "-i",
+ path.join(audioRoot, spk, chap, `${u}.flac`),
+ "-ac",
+ "1",
+ "-ar",
+ String(SR),
+ "-f",
+ "s16le",
+ "-",
+ ],
+ { maxBuffer: 1 << 28 },
+ );
+ for (const w of textGridWords(path.join(alignRoot, spk, chap, `${u}.TextGrid`)))
+ words.push({ text: w.text, start: at + w.start, end: at + w.end });
+ pcm.push(raw);
+ at += raw.length / 2 / SR;
+ }
+ const data = Buffer.concat(pcm);
+ const header = Buffer.alloc(44);
+ header.write("RIFF", 0);
+ header.writeUInt32LE(36 + data.length, 4);
+ header.write("WAVEfmt ", 8);
+ header.writeUInt32LE(16, 16);
+ header.writeUInt16LE(1, 20);
+ header.writeUInt16LE(1, 22);
+ header.writeUInt32LE(SR, 24);
+ header.writeUInt32LE(SR * 2, 28);
+ header.writeUInt16LE(2, 32);
+ header.writeUInt16LE(16, 34);
+ header.write("data", 36);
+ header.writeUInt32LE(data.length, 40);
+ writeFileSync(path.join(CLIPS, `${id}.wav`), Buffer.concat([header, data]));
+ writeFileSync(
+ path.join(CLIPS, `${id}.ref.json`),
+ JSON.stringify({ id, lang: "en", engine: "mfa", durationSec: at, words }, null, 1),
+ );
+ clips.push({ id, lang: "en", durationSec: at, words: words.length });
+ total += at;
+ console.log(id, at.toFixed(1), "s", words.length, "words");
+ group = [];
+ groupSec = 0;
+ };
+ const before = clips.length;
+ for (const u of utts) {
+ if (clips.length - before >= PER_SPEAKER) break;
+ const w = textGridWords(path.join(alignRoot, spk, chap, `${u}.TextGrid`));
+ const sec = w.length ? w[w.length - 1].end + 0.3 : 0;
+ if (groupSec + sec > CLIP_SEC) flush();
+ group.push(u);
+ groupSec += sec;
+ }
+ if (clips.length - before < PER_SPEAKER) flush();
+ if (total >= maxSec) break outer;
+ break;
+ }
+}
+writeFileSync(
+ path.join(DATA, "corpus-manifest.json"),
+ JSON.stringify({ totalSec: total, source: "LibriSpeech test-clean + MFA", clips }, null, 1),
+);
+console.log(`${clips.length} clips, ${(total / 60).toFixed(1)} min`);
diff --git a/tools/stt-eval/word-timing/real-check.mjs b/tools/stt-eval/word-timing/real-check.mjs
index 09a3d7865..b0fca0230 100644
--- a/tools/stt-eval/word-timing/real-check.mjs
+++ b/tools/stt-eval/word-timing/real-check.mjs
@@ -1,36 +1,58 @@
// Sanity check on real speech, which has no ground truth: the VAD onset is the
// only boundary we can trust there.
// Usage: node real-check.mjs <whisper-stt-server.exe> <take.wav> [--cpu] [--snap <post-pass.ts>]
+// [--align <aligner.gguf>]
// (16 kHz mono s16 WAV, e.g. ffmpeg -i take.webm -ar 16000 -ac 1 take.wav)
// Prints how far each word's first-token time (`anchor`) lies after its start
// (the one-token lag the helper corrects), and, per VAD stretch, where its first
// word starts relative to the onset: as the helper reports it, and after the
-// post-pass. Writes the raw response to <data>/results/real-<take>.json.
-import { mkdirSync, writeFileSync } from "node:fs";
+// post-pass. With `--align`, the CTC aligner (electron/stt/ctcAlign.ts) runs too,
+// and every word is listed with both times so the boundaries can be checked by
+// ear or on a spectrogram. Writes the raw response (and the aligner's words) to
+// <data>/results/real-<take>.json.
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
-import { REPO, RESULTS, startHelper, transcribe } from "./lib.mjs";
+import {
+ alignWordsOnEmissions,
+ emissionRegions,
+ parseEmissions,
+} from "../../../electron/stt/ctcAlign.ts";
+import { REPO, RESULTS, readWav, SR, startHelper, transcribe } from "./lib.mjs";
const [exe, wav, ...rest] = process.argv.slice(2);
if (!exe || !wav)
throw new Error(
- "usage: node real-check.mjs <whisper-stt-server.exe> <take.wav> [--cpu] [--snap <post-pass.ts>]",
+ "usage: node real-check.mjs <whisper-stt-server.exe> <take.wav> [--cpu] [--snap <post-pass.ts>] [--align <aligner.gguf>]",
);
-const snapAt = rest.indexOf("--snap");
+const flag = (name) => {
+ const i = rest.indexOf(name);
+ return i >= 0 ? rest[i + 1] : undefined;
+};
const snapPath = path.resolve(
- snapAt >= 0 ? rest[snapAt + 1] : path.join(REPO, "electron/stt/snapWordBoundaries.ts"),
+ flag("--snap") ?? path.join(REPO, "electron/stt/snapWordBoundaries.ts"),
);
+const alignModel = flag("--align") && path.resolve(flag("--align"));
const { anchorWordsOnSpeech } = await import(pathToFileURL(snapPath).href);
const { base, stop } = await startHelper(exe, { cpu: rest.includes("--cpu") });
let json;
+let emissions;
try {
({ json } = await transcribe(base, wav));
+ if (alignModel) {
+ const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));
+ const form = new FormData();
+ form.set("file", new Blob([readFileSync(wav)], { type: "audio/wav" }), path.basename(wav));
+ form.set("model", alignModel);
+ form.set("regions", JSON.stringify(emissionRegions(speech, readWav(wav).length / SR)));
+ const res = await fetch(`${base}/emissions`, { method: "POST", body: form });
+ emissions = await res.json();
+ if (!res.ok) throw new Error(`/emissions: HTTP ${res.status} ${JSON.stringify(emissions)}`);
+ }
} finally {
stop();
}
-mkdirSync(RESULTS, { recursive: true });
-writeFileSync(path.join(RESULTS, `real-${path.basename(wav, ".wav")}.json`), JSON.stringify(json));
const isP = (w) => /^[\p{P}\p{S}]+$/u.test(w.word);
const raw = json.segments
@@ -44,6 +66,15 @@ const raw = json.segments
.filter((w) => w.word);
const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));
const post = anchorWordsOnSpeech(raw, speech);
+const aligned = emissions
+ ? anchorWordsOnSpeech(alignWordsOnEmissions(raw, speech, parseEmissions(emissions)), speech)
+ : null;
+mkdirSync(RESULTS, { recursive: true });
+writeFileSync(
+ path.join(RESULTS, `real-${path.basename(wav, ".wav")}.json`),
+ JSON.stringify({ ...json, phase1: post, aligned }),
+);
+
const lag = raw
.filter((w) => !isP(w))
.map((w) => w.anchorSec - w.startSec)
@@ -63,3 +94,20 @@ for (const [i, s] of speech.entries()) {
);
while (k < raw.length && raw[k].anchorSec < tail) k++;
}
+if (aligned) {
+ const moved = [];
+ console.log("\nword phase 1 (s) aligner (s) start / end moved");
+ raw.forEach((w, j) => {
+ const a = post[j];
+ const b = aligned[j];
+ if (!isP(w)) moved.push(Math.abs(b.startSec - a.startSec), Math.abs(b.endSec - a.endSec));
+ console.log(
+ `${w.word.padEnd(20)} ${a.startSec.toFixed(3)}-${a.endSec.toFixed(3)} ${b.startSec.toFixed(3)}-${b.endSec.toFixed(3)} ${ms(b.startSec - a.startSec)} / ${ms(b.endSec - a.endSec)}`,
+ );
+ });
+ moved.sort((x, y) => x - y);
+ const m = (p) => (moved[Math.floor(moved.length * p)] * 1000).toFixed(0);
+ console.log(
+ `\naligner vs phase 1, |moved| per boundary: median ${m(0.5)} ms, p90 ${m(0.9)} ms (${emissions.elapsed_s.toFixed(2)} s on ${emissions.device})`,
+ );
+}
diff --git a/tools/stt-eval/word-timing/run-align.mjs b/tools/stt-eval/word-timing/run-align.mjs
new file mode 100644
index 000000000..9f372aa51
--- /dev/null
+++ b/tools/stt-eval/word-timing/run-align.mjs
@@ -0,0 +1,64 @@
+// Adds the CTC aligner's emissions to helper responses already saved by
+// run-helper.mjs, so evaluate.mjs can score the aligner without re-running whisper.
+// Usage: node run-align.mjs <from-tag> <to-tag> <whisper-stt-server.exe> \
+// --model en=<w2v-en.gguf> --model fr=<w2v-fr.gguf> [--cpu] [--only <substr>]
+// reads <data>/results/raw/<from-tag>/*.json (needs `speech`: run with the VAD model)
+// writes <data>/results/raw/<to-tag>/*.json with `emissions` (the /emissions answer
+// + wallMs) next to the /inference answer
+// The helper is started once and asked like the app does (electron/stt/index.ts):
+// the regions come from emissionRegions(), the model from the detected language.
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { emissionRegions } from "../../../electron/stt/ctcAlign.ts";
+import { CLIPS, RESULTS, readWav, SR, startHelper } from "./lib.mjs";
+
+const [fromTag, toTag, exe, ...rest] = process.argv.slice(2);
+if (!exe)
+ throw new Error(
+ "usage: node run-align.mjs <from-tag> <to-tag> <exe> --model en=<gguf> [--model fr=<gguf>] [--cpu] [--only <substr>]",
+ );
+const models = {};
+let only = "";
+for (let i = 0; i < rest.length; i++) {
+ if (rest[i] === "--model") {
+ const [lang, file] = rest[++i].split("=");
+ models[lang] = path.resolve(file);
+ }
+ if (rest[i] === "--only") only = rest[++i];
+}
+
+const { base, stop } = await startHelper(exe, { cpu: rest.includes("--cpu") });
+try {
+ const inDir = path.join(RESULTS, "raw", fromTag);
+ const outDir = path.join(RESULTS, "raw", toTag);
+ mkdirSync(outDir, { recursive: true });
+ for (const file of readdirSync(inDir).filter((f) => f.endsWith(".json"))) {
+ if (only && !file.includes(only)) continue;
+ const out = path.join(outDir, file);
+ if (existsSync(out)) continue;
+ const json = JSON.parse(readFileSync(path.join(inDir, file), "utf8"));
+ const model = models[json.detected_language];
+ if (!model || !json.speech) {
+ writeFileSync(out, JSON.stringify(json));
+ continue;
+ }
+ const [id, cond] = file.replace(/\.json$/, "").split(".");
+ const wav = path.join(CLIPS, cond === "clean" ? `${id}.wav` : `${id}.noisy.wav`);
+ const bytes = readFileSync(wav);
+ const durationSec = readWav(wav).length / SR;
+ const speech = json.speech.map((s) => ({ startSec: s.start, endSec: s.end }));
+ const form = new FormData();
+ form.set("file", new Blob([bytes], { type: "audio/wav" }), path.basename(wav));
+ form.set("model", model);
+ form.set("regions", JSON.stringify(emissionRegions(speech, durationSec)));
+ const t0 = performance.now();
+ const res = await fetch(`${base}/emissions`, { method: "POST", body: form });
+ const wallMs = performance.now() - t0;
+ const emissions = await res.json();
+ if (!res.ok) throw new Error(`${file}: HTTP ${res.status} ${JSON.stringify(emissions)}`);
+ writeFileSync(out, JSON.stringify({ ...json, emissions: { ...emissions, wallMs } }));
+ console.log(file, `${emissions.elapsed_s.toFixed(3)} s on ${emissions.device}`);
+ }
+} finally {
+ stop();
+}