Skip to content

A cross platform Audiobookshelf client for Android and iOS

Dart
13.5 KB
Raw Blame
#include "whisper/whisper.h"

#define DR_WAV_IMPLEMENTATION
#include "whisper/examples/dr_wav.h"

#include <cmath>
#include <fstream>
#include <cstdio>
#include <string>
#include <thread>
#include <vector>
#include <mutex>
#include <iostream>
#include <chrono>
#include "json/json.hpp"
#include <stdio.h>

#if defined(__GNUC__)
#define FUNCTION_ATTRIBUTE __attribute__((visibility("default"))) __attribute__((used))
#elif defined(_MSC_VER)
#define FUNCTION_ATTRIBUTE __declspec(dllexport)
#else
#define FUNCTION_ATTRIBUTE
#endif

extern "C" const char* get_vad_model_path();
extern "C" bool absorb_metal_usable();
extern "C" const char * absorb_metal_reason();
extern "C" void absorb_gpu_watch_begin();
extern "C" void absorb_gpu_watch_end();

using json = nlohmann::json;

enum class whisper_vad_mode {
    auto_mode,
    disabled,
    enabled,
};

struct whisper_params
{
    int32_t seed = -1;
    int32_t n_threads = std::min(4, (int32_t)std::thread::hardware_concurrency());

    int32_t n_processors = 1;
    int32_t offset_t_ms = 0;
    int32_t offset_n = 0;
    int32_t duration_ms = 0;
    int32_t max_context = -1;
    int32_t max_len = 0;
    int32_t best_of = 5;
    int32_t beam_size = -1;

    float word_thold = 0.01f;
    float entropy_thold = 2.40f;
    float logprob_thold = -1.00f;

    bool verbose = false;
    bool print_special_tokens = false;
    bool speed_up = false;
    bool translate = false;
    bool diarize = false;
    bool no_fallback = false;
    bool output_txt = false;
    bool output_vtt = false;
    bool output_srt = false;
    bool output_wts = false;
    bool output_csv = false;
    bool print_special = false;
    bool print_colors = false;
    bool print_progress = false;
    bool no_timestamps = false;
    bool split_on_word = false;
    whisper_vad_mode vad_mode = whisper_vad_mode::auto_mode;

    std::string language = "id";
    std::string prompt;
    std::string model = "";
    std::string audio = "";
    std::string vad_model_path = "";
    std::vector<std::string> fname_inp = {};
    std::vector<std::string> fname_outp = {};
};

static whisper_vad_mode parse_vad_mode(const json & json_body) {
    const std::string vad_mode = json_body.value("vad_mode", std::string("auto"));
    if (vad_mode == "disabled") {
        return whisper_vad_mode::disabled;
    }
    if (vad_mode == "enabled") {
        return whisper_vad_mode::enabled;
    }
    return whisper_vad_mode::auto_mode;
}

static std::string default_vad_model_path() {
    const char * vad_path = get_vad_model_path();
    return vad_path == nullptr ? std::string() : std::string(vad_path);
}

static struct whisper_context * g_ctx = nullptr;
static std::string g_model_path = "";
static bool g_use_gpu = true;
static bool g_gpu_proven = false;
static std::mutex g_mutex;
static std::atomic<bool> g_should_abort(false);

// Everything ggml and whisper say, including a Metal kernel that failed to
// build. The tail rides back to Dart with each response.
static std::mutex g_log_mutex;
static std::string g_log;
static bool g_log_hooked = false;

static void capture_whisper_log(enum ggml_log_level level, const char * text, void * user_data) {
    (void) level;
    (void) user_data;
    fputs(text, stderr);
    std::lock_guard<std::mutex> lock(g_log_mutex);
    g_log += text;
    if (g_log.size() > 8192) {
        g_log.erase(0, g_log.size() - 8192);
    }
}

static std::string take_whisper_log() {
    std::lock_guard<std::mutex> lock(g_log_mutex);
    std::string out;
    out.swap(g_log);
    return out;
}

static void dispose_context_locked() {
    if (g_ctx != nullptr) {
        whisper_free(g_ctx);
        g_ctx = nullptr;
    }
    g_model_path.clear();
}

// Abort callback for whisper transcription
static bool abort_callback(void* user_data) {
    return g_should_abort.load();
}

char *jsonToChar(json jsonData)
{
    try {
        // Ensure ASCII encoding to avoid UTF-8 issues across FFI boundary
        // Non-ASCII characters (Korean, etc.) will be escaped as \uXXXX
        // Use 'replace' instead of 'strict' to handle malformed UTF-8 from Whisper output
        // (e.g., truncated multibyte sequences like 0xEC without following bytes)
        std::string result = jsonData.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace);
        char *ch = new char[result.size() + 1];
        if (ch) {
            strcpy(ch, result.c_str());
        }
        return ch;
    } catch (const std::exception& e) {
        // Fallback for absolute safety
        std::string errorJson = "{\"@type\":\"error\",\"message\":\"JSON serialization failed\"}";
        char *ch = new char[errorJson.size() + 1];
        strcpy(ch, errorJson.c_str());
        return ch;
    }
}

json transcribe(json jsonBody)
{
    std::lock_guard<std::mutex> lock(g_mutex);
    
    g_should_abort.store(false);

    whisper_params params;
    params.n_threads = jsonBody["threads"];
    params.verbose = jsonBody["is_verbose"];
    params.translate = jsonBody["is_translate"];
    params.language = jsonBody["language"];
    params.print_special_tokens = jsonBody["is_special_tokens"];
    params.no_timestamps = jsonBody["is_no_timestamps"];
    params.model = jsonBody["model"];
    params.audio = jsonBody["audio"];
    params.split_on_word = jsonBody["split_on_word"];
    params.diarize = jsonBody["diarize"];
    params.speed_up = jsonBody["speed_up"];
    params.vad_mode = parse_vad_mode(jsonBody);
    // Dart always sends the key and it is null whenever VAD is off, and
    // value() only falls back for a MISSING key - a present null throws.
    if (jsonBody.contains("vad_model_path") && jsonBody["vad_model_path"].is_string()) {
        params.vad_model_path = jsonBody["vad_model_path"].get<std::string>();
    }

    json jsonResult;
    jsonResult["@type"] = "transcribe";

    if (!g_log_hooked) {
        whisper_log_set(capture_whisper_log, nullptr);
        g_log_hooked = true;
    }
    const bool use_gpu = absorb_metal_usable();
    if (g_ctx == nullptr || g_model_path != params.model || g_use_gpu != use_gpu) {
        dispose_context_locked();
        if (!use_gpu) {
            const std::string why = std::string("absorb: whisper runs on the CPU: ") + absorb_metal_reason() + "\n";
            capture_whisper_log(GGML_LOG_LEVEL_INFO, why.c_str(), nullptr);
        }

        whisper_context_params cparams = whisper_context_default_params();
        cparams.use_gpu = use_gpu;
        cparams.flash_attn = true;

        if (use_gpu) absorb_gpu_watch_begin();
        g_ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);
        if (use_gpu) absorb_gpu_watch_end();
        g_gpu_proven = false;
        if (g_ctx != nullptr) {
            g_model_path = params.model;
            g_use_gpu = use_gpu;
        }
    }

    if (g_ctx == nullptr)
    {
        jsonResult["@type"] = "error";
        jsonResult["message"] = "failed to initialize whisper context (possibly OOM)";
        return jsonResult;
    }

    std::vector<float> pcmf32;
    {
        drwav wav;
        if (!drwav_init_file(&wav, params.audio.c_str(), NULL))
        {
            jsonResult["@type"] = "error";
            jsonResult["message"] = " failed to open WAV file ";
            return jsonResult;
        }

        int n = wav.totalPCMFrameCount;
        std::vector<int16_t> pcm16(n * wav.channels);
        drwav_read_pcm_frames_s16(&wav, n, pcm16.data());
        drwav_uninit(&wav);

        pcmf32.resize(n);
        if (wav.channels == 1) {
            for (int i = 0; i < n; i++) pcmf32[i] = float(pcm16[i]) / 32768.0f;
        } else {
            for (int i = 0; i < n; i++) pcmf32[i] = float(pcm16[2 * i] + pcm16[2 * i + 1]) / 65536.0f;
        }
    }

    // Debug: Print model hyperparameters to diagnose segmentation issues
    const int model_n_text_layer = whisper_model_n_text_layer(g_ctx);
    const int model_n_vocab = whisper_model_n_vocab(g_ctx);
    const bool is_turbo = (model_n_text_layer == 4 && model_n_vocab == 51866);
    
    fprintf(stderr, "[DEBUG] Model info - n_text_layer: %d, n_vocab: %d, is_turbo: %d\n", 
            model_n_text_layer, model_n_vocab, is_turbo);

    whisper_sampling_strategy strategy = is_turbo ? WHISPER_SAMPLING_BEAM_SEARCH : WHISPER_SAMPLING_GREEDY;
    whisper_full_params wparams = whisper_full_default_params(strategy);
    
    wparams.print_realtime = false;
    wparams.print_progress = false;
    wparams.print_timestamps = !params.no_timestamps;
    wparams.translate = params.translate;
    wparams.language = params.language.c_str();
    wparams.n_threads = params.n_threads;
    wparams.split_on_word = params.split_on_word;
    wparams.audio_ctx = params.speed_up ? 768 : 0; // Use smaller audio context for speedUp
    wparams.single_segment = false;

    if (params.split_on_word) {
        fprintf(stderr, "[DEBUG] Disabling VAD because split_on_word requires stable timestamps\n");
        wparams.vad = false;
    } else if (params.vad_mode == whisper_vad_mode::disabled) {
        wparams.vad = false;
    } else {
        if (params.vad_model_path.empty()) {
            params.vad_model_path = default_vad_model_path();
        }
        if (!params.vad_model_path.empty()) {
            wparams.vad = true;
            wparams.vad_model_path = params.vad_model_path.c_str();
        } else if (params.vad_mode == whisper_vad_mode::enabled) {
            jsonResult["@type"] = "error";
            jsonResult["message"] =
                "VAD was explicitly enabled but no vad_model_path was available";
            return jsonResult;
        } else {
            wparams.vad = false;
        }
    }

    if (is_turbo) {
        wparams.beam_search.beam_size = 3;
        fprintf(stderr, "[DEBUG] Turbo model detected - using beam search (beam_size=3)\n");
    }

    if (params.split_on_word) {
        wparams.max_len = 1;
        wparams.token_timestamps = true;
    }
    
    // Set abort callback
    wparams.abort_callback = abort_callback;
    wparams.abort_callback_user_data = nullptr;

    // Debug: Print critical parameters before transcription
    fprintf(stderr, "[DEBUG] Transcription params - threads: %d, speed_up: %d, no_timestamps: %d, single_segment: %d, split_on_word: %d, max_len: %d\n",
            wparams.n_threads, params.speed_up, wparams.no_timestamps, wparams.single_segment, wparams.split_on_word, wparams.max_len);
    fflush(stderr);

    auto start_time = std::chrono::high_resolution_clock::now();

    // A GPU that can't run whisper dies on its first run, so only that one is
    // watched; a long session isn't, where being swiped away would count too.
    const bool watch = g_use_gpu && !g_gpu_proven;
    if (watch) absorb_gpu_watch_begin();
    const int full_status = whisper_full(g_ctx, wparams, pcmf32.data(), pcmf32.size());
    if (watch) {
        absorb_gpu_watch_end();
        g_gpu_proven = true;
    }

    if (full_status != 0)
    {
        if (g_should_abort.load()) {
            jsonResult["@type"] = "aborted";
            jsonResult["message"] = "transcription aborted by user";
            g_should_abort.store(false);
            return jsonResult;
        }
        jsonResult["@type"] = "error";
        jsonResult["message"] = "failed to process audio";
        return jsonResult;
    }

    auto end_time = std::chrono::high_resolution_clock::now();
    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();

    fprintf(stderr, "[DEBUG] Transcription completed in %lldms\n", (int)duration);
    fflush(stderr);

    const int n_segments = whisper_full_n_segments(g_ctx);
    std::vector<json> segmentsJson = {};
    std::string text_result = "";

    for (int i = 0; i < n_segments; ++i)
    {
        const char *text = whisper_full_get_segment_text(g_ctx, i);
        text_result += std::string(text);
        
        if (!params.no_timestamps) {
            json jsonSegment;
            jsonSegment["from_ts"] = whisper_full_get_segment_t0(g_ctx, i);
            jsonSegment["to_ts"] = whisper_full_get_segment_t1(g_ctx, i);
            jsonSegment["text"] = text;
            segmentsJson.push_back(jsonSegment);
        }
    }

    if (!params.no_timestamps) {
        jsonResult["segments"] = segmentsJson;
    }
    
    jsonResult["text"] = text_result;
    jsonResult["language"] = whisper_lang_str(whisper_full_lang_id(g_ctx));
    return jsonResult;
}

extern "C"
{
    FUNCTION_ATTRIBUTE char *request(char *body)
    {
        try {
            json jsonBody = json::parse(body);
            if (jsonBody["@type"] == "abort") {
                g_should_abort.store(true);
                return jsonToChar({{"@type", "abort"}, {"message", "abort signal sent"}});
            }
            if (jsonBody["@type"] == "dispose") {
                std::lock_guard<std::mutex> lock(g_mutex);
                dispose_context_locked();
                return jsonToChar({{"@type", "dispose"}, {"message", "whisper context disposed"}});
            }
            if (jsonBody["@type"] == "getTextFromWavFile") {
                json result = transcribe(jsonBody);
                result["backend"] = g_use_gpu ? "metal" : "cpu";
                result["log"] = take_whisper_log();
                return jsonToChar(result);
            }
            if (jsonBody["@type"] == "getVersion") {
                return jsonToChar({{"@type", "version"}, {"message", "lib v1.8.3-accel"}});
            }
            return jsonToChar({{"@type", "error"}, {"message", "method not found"}});
        } catch (const std::exception &e) {
            return jsonToChar({{"@type", "error"}, {"message", e.what()}});
        }
    }

    FUNCTION_ATTRIBUTE void free_string(char *ptr)
    {
        delete[] ptr;
    }
}