Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
// wav2vec2-for-CTC forward pass on ggml (HuggingFace Wav2Vec2ForCTC, both the
// "group norm, post-LN" base layout and the "layer norm, pre-LN" large one).
// Weights come from scripts/convert-wav2vec2-gguf.mjs.
//
// Layout: activations are [channels, time] (ne0 = channels) for the linear
// layers, and transposed to [time, channels] only where a convolution or a
// per-channel norm needs time contiguous.
#include "ctc_aligner.h"
#include <ggml.h>
#include <ggml-alloc.h>
#include <ggml-backend.h>
#include <gguf.h>
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <filesystem>
namespace {
constexpr int kGraphSize = 8192;
// Longest input one graph sees; longer ones run in windows. Attention is
// quadratic in time and the first convolutions are wide, so this bounds memory
// (about 265 MB of compute buffers at 20 s, for either model: the first
// convolutions dominate).
constexpr int kWindowFrames = 1000; // 20 s
constexpr int kContextFrames = 50; // 1 s of context kept on each side of a window
struct Layer {
ggml_tensor *q_w, *q_b, *k_w, *k_b, *v_w, *v_b, *o_w, *o_b;
ggml_tensor *ln1_w, *ln1_b, *ff1_w, *ff1_b, *ff2_w, *ff2_b, *ln2_w, *ln2_b;
};
} // namespace
struct CtcModel {
CtcModelInfo info;
int hidden = 0, n_heads = 0, pos_k = 0, pos_groups = 0;
float eps = 1e-5f;
bool group_norm = false, stable = false;
std::vector<int> conv_k, conv_s;
std::vector<ggml_tensor*> conv_w, conv_b, conv_ln_w, conv_ln_b;
ggml_tensor *fp_ln_w = nullptr, *fp_ln_b = nullptr, *fp_w = nullptr, *fp_b = nullptr;
ggml_tensor *pos_w = nullptr, *pos_b = nullptr, *enc_ln_w = nullptr, *enc_ln_b = nullptr;
ggml_tensor *head_w = nullptr, *head_b = nullptr;
std::vector<Layer> layers;
ggml_context* wctx = nullptr;
gguf_context* gctx = nullptr;
ggml_backend_buffer_t wbuf = nullptr;
ggml_backend_t gpu = nullptr, cpu = nullptr;
ggml_backend_sched_t sched = nullptr;
std::string device = "CPU";
~CtcModel() {
if (sched) ggml_backend_sched_free(sched);
if (wbuf) ggml_backend_buffer_free(wbuf);
if (gpu) ggml_backend_free(gpu);
if (cpu) ggml_backend_free(cpu);
if (wctx) ggml_free(wctx);
if (gctx) gguf_free(gctx);
}
};
void CtcModelDeleter::operator()(CtcModel* m) const { delete m; }
const CtcModelInfo& ctc_info(const CtcModel& model) { return model.info; }
std::string ctc_device(const CtcModel& model) { return model.device; }
namespace {
int64_t key(gguf_context* g, const char* k, std::string& err) {
const int64_t id = gguf_find_key(g, k);
if (id < 0 && err.empty()) err = std::string("missing metadata ") + k;
return id;
}
std::vector<std::string> str_array(gguf_context* g, int64_t id) {
std::vector<std::string> out;
if (id < 0) return out;
const size_t n = gguf_get_arr_n(g, id);
for (size_t i = 0; i < n; ++i) out.emplace_back(gguf_get_arr_str(g, id, i));
return out;
}
std::vector<int> u32_array(gguf_context* g, int64_t id) {
std::vector<int> out;
if (id < 0) return out;
const auto* p = static_cast<const uint32_t*>(gguf_get_arr_data(g, id));
out.assign(p, p + gguf_get_arr_n(g, id));
return out;
}
ggml_tensor* layer_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, float eps) {
return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b);
}
ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b) {
return ggml_add(ctx, ggml_mul_mat(ctx, w, x), b);
}
// 1-D convolution of `x` [time, in] with `w` [k, in, out] -> [out, time']. The
// columns are F16, as in whisper.cpp's encoder: half the memory traffic of F32,
// which made the CPU path 1.7x slower, and `w` must be F16 to match.
ggml_tensor* conv1d(ggml_context* ctx, ggml_tensor* w, ggml_tensor* x, int stride, int pad) {
ggml_tensor* cols = ggml_im2col(ctx, w, x, stride, 0, pad, 0, 1, 0, false, GGML_TYPE_F16);
// cols: [k * in, time']
cols = ggml_reshape_2d(ctx, cols, cols->ne[0], cols->ne[1]);
ggml_tensor* w2 = ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]);
return ggml_mul_mat(ctx, w2, cols);
}
ggml_tensor* build(CtcModel& m, ggml_context* ctx, ggml_tensor* wave) {
// ---- feature encoder ----
ggml_tensor* x = wave; // [time, 1]
ggml_tensor* y = nullptr;
for (size_t i = 0; i < m.conv_w.size(); ++i) {
y = conv1d(ctx, m.conv_w[i], x, m.conv_s[i], 0); // [512, t]
if (m.conv_b[i]) y = ggml_add(ctx, y, m.conv_b[i]);
if (m.group_norm) {
// Time-major for the next convolution. Only the first layer is normed:
// GroupNorm with one group per channel, i.e. each channel over time.
ggml_tensor* t = ggml_cont(ctx, ggml_transpose(ctx, y)); // [t, 512]
if (i == 0) {
t = ggml_norm(ctx, t, m.eps);
t = ggml_add(ctx, ggml_mul(ctx, t, ggml_reshape_2d(ctx, m.conv_ln_w[0], 1, t->ne[1])),
ggml_reshape_2d(ctx, m.conv_ln_b[0], 1, t->ne[1]));
}
x = ggml_gelu_erf(ctx, t);
} else {
y = layer_norm(ctx, y, m.conv_ln_w[i], m.conv_ln_b[i], m.eps);
x = ggml_cont(ctx, ggml_transpose(ctx, ggml_gelu_erf(ctx, y))); // [t, 512]
}
}
// x: [t, 512] -> [512, t]
ggml_tensor* h = ggml_cont(ctx, ggml_transpose(ctx, x));
const int64_t T = h->ne[1];
// ---- feature projection ----
h = layer_norm(ctx, h, m.fp_ln_w, m.fp_ln_b, m.eps);
h = linear(ctx, h, m.fp_w, m.fp_b); // [H, T]
// ---- positional convolution (grouped, same padding, drop the last frame) ----
{
ggml_tensor* ht = ggml_cont(ctx, ggml_transpose(ctx, h)); // [T, H]
const int64_t cg = m.hidden / m.pos_groups;
ggml_tensor* pos = nullptr;
for (int g = 0; g < m.pos_groups; ++g) {
ggml_tensor* xin = ggml_view_2d(ctx, ht, T, cg, ht->nb[1], g * cg * ht->nb[1]);
ggml_tensor* wg = ggml_view_3d(ctx, m.pos_w, m.pos_k, cg, cg, m.pos_w->nb[1],
m.pos_w->nb[2], g * cg * m.pos_w->nb[2]);
ggml_tensor* part = conv1d(ctx, wg, ggml_cont(ctx, xin), 1, m.pos_k / 2); // [cg, T+1]
pos = pos ? ggml_concat(ctx, pos, part, 0) : part;
}
pos = ggml_view_2d(ctx, pos, m.hidden, T, pos->nb[1], 0);
pos = ggml_gelu_erf(ctx, ggml_add(ctx, pos, m.pos_b));
h = ggml_add(ctx, h, pos);
}
if (!m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps);
// ---- transformer ----
const int64_t dh = m.hidden / m.n_heads;
const float scale = 1.0f / std::sqrt(static_cast<float>(dh));
for (const Layer& L : m.layers) {
ggml_tensor* res = h;
ggml_tensor* a = m.stable ? layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps) : h;
ggml_tensor* q = ggml_reshape_3d(ctx, linear(ctx, a, L.q_w, L.q_b), dh, m.n_heads, T);
ggml_tensor* k = ggml_reshape_3d(ctx, linear(ctx, a, L.k_w, L.k_b), dh, m.n_heads, T);
ggml_tensor* v = ggml_reshape_3d(ctx, linear(ctx, a, L.v_w, L.v_b), dh, m.n_heads, T);
q = ggml_permute(ctx, q, 0, 2, 1, 3); // [dh, T, nh]
k = ggml_permute(ctx, k, 0, 2, 1, 3); // [dh, T, nh]
v = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, dh, nh]
ggml_tensor* kq = ggml_mul_mat(ctx, k, q); // [Tk, Tq, nh]
kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f);
ggml_tensor* o = ggml_mul_mat(ctx, v, kq); // [dh, Tq, nh]
o = ggml_cont(ctx, ggml_permute(ctx, o, 0, 2, 1, 3)); // [dh, nh, T]
o = linear(ctx, ggml_reshape_2d(ctx, o, m.hidden, T), L.o_w, L.o_b);
h = ggml_add(ctx, res, o);
if (m.stable) {
ggml_tensor* f = layer_norm(ctx, h, L.ln2_w, L.ln2_b, m.eps);
f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, f, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b);
h = ggml_add(ctx, h, f);
} else {
h = layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps);
ggml_tensor* f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, h, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b);
h = layer_norm(ctx, ggml_add(ctx, h, f), L.ln2_w, L.ln2_b, m.eps);
}
}
if (m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps);
return linear(ctx, h, m.head_w, m.head_b); // [V, T]
}
int frames_for(const CtcModel& m, int64_t n) {
for (size_t i = 0; i < m.conv_k.size(); ++i) {
if (n < m.conv_k[i]) return 0;
n = (n - m.conv_k[i]) / m.conv_s[i] + 1;
}
return static_cast<int>(n);
}
// Logits for one window of already-normalized samples.
bool run_window(CtcModel& m, const float* pcm, int64_t n, std::vector<float>& logits, int& frames,
std::string& err) {
frames = frames_for(m, n);
if (frames <= 0) return true;
ggml_init_params p = {ggml_tensor_overhead() * kGraphSize + ggml_graph_overhead_custom(kGraphSize, false),
nullptr, true};
ggml_context* ctx = ggml_init(p);
ggml_tensor* wave = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1);
ggml_set_input(wave);
ggml_tensor* out = build(m, ctx, wave);
ggml_set_output(out);
ggml_cgraph* gf = ggml_new_graph_custom(ctx, kGraphSize, false);
ggml_build_forward_expand(gf, out);
ggml_backend_sched_reset(m.sched);
bool ok = ggml_backend_sched_alloc_graph(m.sched, gf);
if (ok) {
ggml_backend_tensor_set(wave, pcm, 0, n * sizeof(float));
ok = ggml_backend_sched_graph_compute(m.sched, gf) == GGML_STATUS_SUCCESS;
}
if (ok) {
if (out->ne[1] != frames) {
err = "unexpected frame count";
ok = false;
} else {
logits.resize(static_cast<size_t>(out->ne[0]) * frames);
ggml_backend_tensor_get(out, logits.data(), 0, logits.size() * sizeof(float));
}
} else if (err.empty()) {
err = "ggml graph allocation or compute failed";
}
ggml_free(ctx);
return ok;
}
} // namespace
CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err) {
CtcModelPtr m(new CtcModel());
gguf_init_params gp = {true, &m->wctx};
m->gctx = gguf_init_from_file(path.c_str(), gp);
if (!m->gctx) {
err = "cannot read " + path;
return nullptr;
}
gguf_context* g = m->gctx;
CtcModelInfo& info = m->info;
info.vocab = str_array(g, key(g, "w2v.vocab", err));
info.languages = str_array(g, key(g, "w2v.languages", err));
int64_t id;
if ((id = key(g, "w2v.blank_id", err)) >= 0) info.blank = gguf_get_val_u32(g, id);
if ((id = key(g, "w2v.hidden_size", err)) >= 0) m->hidden = gguf_get_val_u32(g, id);
if ((id = key(g, "w2v.num_attention_heads", err)) >= 0) m->n_heads = gguf_get_val_u32(g, id);
if ((id = key(g, "w2v.layer_norm_eps", err)) >= 0) m->eps = gguf_get_val_f32(g, id);
if ((id = key(g, "w2v.num_conv_pos_embeddings", err)) >= 0) m->pos_k = gguf_get_val_u32(g, id);
if ((id = key(g, "w2v.num_conv_pos_embedding_groups", err)) >= 0) m->pos_groups = gguf_get_val_u32(g, id);
if ((id = key(g, "w2v.feat_extract_norm", err)) >= 0) m->group_norm = std::string(gguf_get_val_str(g, id)) == "group";
if ((id = key(g, "w2v.stable_layer_norm", err)) >= 0) m->stable = gguf_get_val_bool(g, id);
m->conv_k = u32_array(g, key(g, "w2v.conv_kernel", err));
m->conv_s = u32_array(g, key(g, "w2v.conv_stride", err));
int n_layers = 0;
if ((id = key(g, "w2v.num_hidden_layers", err)) >= 0) n_layers = gguf_get_val_u32(g, id);
if (!err.empty()) return nullptr;
info.stride = 1;
for (int s : m->conv_s) info.stride *= s;
info.receptive_field = 1;
for (int i = static_cast<int>(m->conv_k.size()) - 1; i >= 0; --i)
info.receptive_field = (info.receptive_field - 1) * m->conv_s[i] + m->conv_k[i];
auto T = [&](const std::string& name, bool required = true) -> ggml_tensor* {
ggml_tensor* t = ggml_get_tensor(m->wctx, name.c_str());
if (!t && required && err.empty()) err = "missing tensor " + name;
return t;
};
for (size_t i = 0; i < m->conv_k.size(); ++i) {
const std::string p = "feature_extractor.conv_layers." + std::to_string(i) + ".";
m->conv_w.push_back(T(p + "conv.weight"));
m->conv_b.push_back(T(p + "conv.bias", false));
const bool has_ln = !m->group_norm || i == 0;
m->conv_ln_w.push_back(has_ln ? T(p + "layer_norm.weight") : nullptr);
m->conv_ln_b.push_back(has_ln ? T(p + "layer_norm.bias") : nullptr);
}
m->fp_ln_w = T("feature_projection.layer_norm.weight");
m->fp_ln_b = T("feature_projection.layer_norm.bias");
m->fp_w = T("feature_projection.projection.weight");
m->fp_b = T("feature_projection.projection.bias");
m->pos_w = T("encoder.pos_conv_embed.conv.weight");
m->pos_b = T("encoder.pos_conv_embed.conv.bias");
m->enc_ln_w = T("encoder.layer_norm.weight");
m->enc_ln_b = T("encoder.layer_norm.bias");
m->head_w = T("lm_head.weight");
m->head_b = T("lm_head.bias");
for (int l = 0; l < n_layers; ++l) {
const std::string p = "encoder.layers." + std::to_string(l) + ".";
m->layers.push_back({
T(p + "attention.q_proj.weight"), T(p + "attention.q_proj.bias"),
T(p + "attention.k_proj.weight"), T(p + "attention.k_proj.bias"),
T(p + "attention.v_proj.weight"), T(p + "attention.v_proj.bias"),
T(p + "attention.out_proj.weight"), T(p + "attention.out_proj.bias"),
T(p + "layer_norm.weight"), T(p + "layer_norm.bias"),
T(p + "feed_forward.intermediate_dense.weight"), T(p + "feed_forward.intermediate_dense.bias"),
T(p + "feed_forward.output_dense.weight"), T(p + "feed_forward.output_dense.bias"),
T(p + "final_layer_norm.weight"), T(p + "final_layer_norm.bias"),
});
}
if (!err.empty()) return nullptr;
// conv1d() multiplies them with F16 columns; anything else aborts inside ggml.
for (const ggml_tensor* w : m->conv_w) {
if (w->type != GGML_TYPE_F16) err = "convolution weights must be F16";
}
if (m->pos_w->type != GGML_TYPE_F16) err = "convolution weights must be F16";
if (!err.empty()) return nullptr;
if (static_cast<int>(info.vocab.size()) != m->head_w->ne[1]) {
err = "vocabulary does not match lm_head";
return nullptr;
}
// ---- backends ----
if (use_gpu) {
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
const auto type = ggml_backend_dev_type(dev);
if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU) continue;
m->gpu = ggml_backend_dev_init(dev, nullptr);
if (m->gpu) {
m->device = ggml_backend_dev_name(dev);
break;
}
}
}
m->cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
if (!m->cpu) {
err = "no CPU backend";
return nullptr;
}
{
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(m->cpu));
using set_threads_t = void (*)(ggml_backend_t, int);
auto set_threads = reinterpret_cast<set_threads_t>(
ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
if (set_threads) set_threads(m->cpu, threads);
}
ggml_backend_t weights_on = m->gpu ? m->gpu : m->cpu;
m->wbuf = ggml_backend_alloc_ctx_tensors(m->wctx, weights_on);
if (!m->wbuf) {
err = "cannot allocate the model's weights";
return nullptr;
}
{
// The path is UTF-8, which Windows' fopen reads in the ANSI code page
// (issue #1405). Not ggml_fopen: the helper links the CRT statically, so
// a FILE* opened inside ggml.dll belongs to another CRT and crashes here.
#ifdef _WIN32
FILE* f = _wfopen(std::filesystem::path(std::u8string(path.begin(), path.end())).c_str(), L"rb");
#else
FILE* f = std::fopen(path.c_str(), "rb");
#endif
if (!f) {
err = "cannot open " + path;
return nullptr;
}
std::vector<uint8_t> buf;
const size_t data_off = gguf_get_data_offset(g);
for (int64_t i = 0; i < gguf_get_n_tensors(g); ++i) {
ggml_tensor* t = ggml_get_tensor(m->wctx, gguf_get_tensor_name(g, i));
const size_t nbytes = ggml_nbytes(t);
buf.resize(nbytes);
#ifdef _WIN32
_fseeki64(f, static_cast<int64_t>(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET);
#else
fseeko(f, static_cast<off_t>(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET);
#endif
if (std::fread(buf.data(), 1, nbytes, f) != nbytes) {
std::fclose(f);
err = "truncated model file " + path;
return nullptr;
}
ggml_backend_tensor_set(t, buf.data(), 0, nbytes);
}
std::fclose(f);
}
std::vector<ggml_backend_t> backends;
if (m->gpu) backends.push_back(m->gpu);
backends.push_back(m->cpu);
m->sched = ggml_backend_sched_new(backends.data(), nullptr, static_cast<int>(backends.size()),
kGraphSize, false, true);
if (!m->sched) {
err = "cannot create the ggml scheduler";
return nullptr;
}
return m;
}
bool ctc_emissions(CtcModel& m, const float* pcm, size_t n, std::vector<float>& out, int& frames,
std::string& err) {
const int V = static_cast<int>(m.info.vocab.size());
const int64_t stride = m.info.stride;
frames = frames_for(m, static_cast<int64_t>(n));
out.assign(static_cast<size_t>(std::max(frames, 0)) * V, 0.0f);
if (frames <= 0) return true;
// Wav2Vec2FeatureExtractor(do_normalize=True): zero mean, unit variance, per input.
double mean = 0.0, var = 0.0;
for (size_t i = 0; i < n; ++i) mean += pcm[i];
mean /= static_cast<double>(n);
for (size_t i = 0; i < n; ++i) var += (pcm[i] - mean) * (pcm[i] - mean);
var /= static_cast<double>(n);
const double inv = 1.0 / std::sqrt(var + 1e-7);
std::vector<float> x(n);
for (size_t i = 0; i < n; ++i) x[i] = static_cast<float>((pcm[i] - mean) * inv);
// Windows of kWindowFrames, stepping by kWindowFrames - 2 * kContextFrames; each
// keeps its frames away from the edges it shares with a neighbour.
const int64_t win_samples = kWindowFrames * stride + (m.info.receptive_field - stride);
const int hop = kWindowFrames - 2 * kContextFrames;
std::vector<float> logits;
for (int f0 = 0;; f0 += hop) {
const int64_t s0 = static_cast<int64_t>(f0) * stride;
const int64_t len = std::min<int64_t>(win_samples, static_cast<int64_t>(n) - s0);
int wf = 0;
if (!run_window(m, x.data() + s0, len, logits, wf, err)) return false;
const bool last = s0 + len >= static_cast<int64_t>(n);
const int keep_from = f0 == 0 ? 0 : kContextFrames;
const int keep_to = last ? wf : std::min(wf, kWindowFrames - kContextFrames);
for (int i = keep_from; i < keep_to && f0 + i < frames; ++i) {
const float* row = logits.data() + static_cast<size_t>(i) * V;
float mx = row[0];
for (int v = 1; v < V; ++v) mx = std::max(mx, row[v]);
double sum = 0.0;
for (int v = 0; v < V; ++v) sum += std::exp(row[v] - mx);
const float lse = mx + static_cast<float>(std::log(sum));
float* dst = out.data() + static_cast<size_t>(f0 + i) * V;
for (int v = 0; v < V; ++v) dst[v] = row[v] - lse;
}
if (last) break;
}
return true;
}