Skip to content

Distribute and run LLMs with a single file.

7.1 KB
Raw Blame
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
//
// transcribefile entry point.
//
// Wraps transcribe.cpp's example CLI (transcribe_cli_main, exposed when
// examples/cli/main.cpp is built with -DTRANSCRIBEFILE) with the
// llamafile-style conveniences: a crash handler and, crucially, loading
// default arguments from the executable's own zip store at /zip/.args.
//
// That last bit is what makes a self-contained `foo.transcribefile`
// possible: zipalign a GGUF plus a .args file (e.g. containing
// `-m\n/zip/model.gguf`) into the executable and it runs with no
// command-line arguments. Models can still be loaded from disk, or from
// the zip explicitly via `-m /zip/<name>.gguf`.

#include <cosmo.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "llamafile/llamafile.h"

// Defined in transcribe.cpp/examples/cli/main.cpp, compiled with
// -DTRANSCRIBEFILE (which renames its main() to transcribe_cli_main() and
// drops the standalone main()). C++ linkage — both sides are C++.
int transcribe_cli_main(int argc, char ** argv);


#ifndef TRANSCRIBEFILE_VERSION_STRING
#define TRANSCRIBEFILE_VERSION_STRING "0.0.0-dev"
#endif

static bool has_flag(char ** argv, const char * flag) {
    for (char ** p = argv + 1; p && *p; ++p) {
        if (!strcmp(*p, flag)) {
            return true;
        }
    }
    return false;
}

// Remove `flag` from argv (first occurrence) and report whether it was
// there. For flags the wrapper owns, like --verbose: upstream's CLI
// errors out on options it doesn't know, so these must not be forwarded.
static bool consume_flag(char ** argv, const char * flag) {
    for (char ** p = argv + 1; p && *p; ++p) {
        if (!strcmp(*p, flag)) {
            do {
                p[0] = p[1];
            } while (*p++);
            return true;
        }
    }
    return false;
}

// Appended to upstream's usage text (same stream: stderr) so the flags
// and behaviors added by this wrapper are discoverable from --help.
static void print_wrapper_help(void) {
    fprintf(stderr,
            "\n"
            "transcribefile options:\n"
            "  --version             print transcribefile version and exit\n"
            "  --verbose             show GPU loader and backend logging\n"
            "                        (suppressed by default)\n"
            "transcribefile notes:\n"
            "  No GPU backend in this build: runs on CPU. Metal is disabled\n"
            "  until transcribe.cpp's ggml matches llamafile's again;\n"
            "  vulkan and cuda are not available yet.\n"
            "  Default arguments are read from .args in the executable's zip\n"
            "  store (one per line), so models can be bundled with zipalign.\n");
}

// Register GPU backends with ggml before the CLI enumerates devices.
// Only Metal is wired up for now (Vulkan/CUDA are follow-up work), via
// llamafile's runtime loader: on Apple Silicon it compiles the bundled
// ggml Metal sources into ggml-metal.dylib (cached under ~/.llamafile),
// loads it with cosmo_dlopen, and registers it with transcribe.cpp's
// ggml. The CLI owns --backend semantics; this only controls which
// backends get a chance to register:
//   - cpu / cpu_accel: don't touch the GPU machinery at all
//   - auto / metal:    try Metal; on failure or non-mac hardware nothing
//                      registers and selection proceeds CPU-only
// FLAG_gpu stays AUTO (not APPLE) even for --backend metal so a failed
// load degrades silently here; transcribe.cpp reports the missing
// backend itself, with wording that matches its own CLI.
// Metal is disabled for now. The dylib is built from llamafile's bundled
// llama.cpp ggml-metal sources and registered with transcribe.cpp's own
// ggml (0.15.2), which only works while both trees agree on the backend
// interface structs. llama.cpp v0.8.0 (ggml 0.28) added alloc_buffer_n
// and get_alloc_size_n to ggml_backend_buffer_type_i, so transcribe.cpp
// called a NULL slot in ggml_backend_buft_get_alignment and crashed on
// model load. Re-enable together with a transcribe.cpp update whose ggml
// matches llama.cpp's.
static constexpr bool kMetalEnabled = false;

static void load_gpu_backends(char ** argv) {
    if (!kMetalEnabled) {
        FLAG_gpu = LLAMAFILE_GPU_DISABLE;
        return;
    }
    const char * backend = "auto";
    for (char ** p = argv + 1; p && *p; ++p) {
        if (!strcmp(*p, "--backend") && p[1]) {
            backend = p[1];
        }
    }
    if (strcmp(backend, "auto") != 0 && strcmp(backend, "metal") != 0) {
        FLAG_gpu = LLAMAFILE_GPU_DISABLE;
        return;
    }
    // The Metal dylib logs through ggml's default stderr callback: device
    // init banners plus one "compiling pipeline" DEBUG line per kernel on
    // EVERY run (pipeline-state objects live in an in-memory cache only,
    // so each process recreates them; the on-disk dylib cache is a
    // different layer). Route the dylib's logging to the null sink unless
    // --verbose was given, exactly like llamafile: FLAG_verbose also
    // unlocks the loader's own llamafile_info diagnostics.
    if (!FLAG_verbose) {
        llamafile_metal_log_set(llamafile_log_callback_null, nullptr);
    }
    FLAG_gpu = LLAMAFILE_GPU_AUTO;
    llamafile_has_metal();
}

int main(int argc, char ** argv) {
    // Symbolized backtraces on crash (cosmopolitan).
    ShowCrashReports();

    // Cached artifacts (the runtime-compiled Metal dylib and the ggml
    // sources it is built from) go under ~/.transcribefile, not
    // ~/.llamafile: the two products' ggml trees are aligned today but
    // may diverge, and neither should ever overwrite the other's cache.
    llamafile_set_app_name("transcribefile");

    // Answer --version before touching args or the zip store. The
    // version string is transcribe.cpp's `git describe` output, which
    // already carries its own "v" prefix (e.g. v0.0.11-7-gdf1a4ad).
    if (has_flag(argv, "--version")) {
        puts("transcribefile " TRANSCRIBEFILE_VERSION_STRING);
        return 0;
    }

    // Merge default arguments embedded at /zip/.args (if present) with the
    // user's argv. No-op for a bare executable with no bundled .args.
    argc = cosmo_args("/zip/.args", &argv);

    // Wrapper-owned flag, same meaning as llamafile's --verbose: GPU
    // loader diagnostics and dylib logging. Consumed here because the
    // upstream CLI rejects options it doesn't know.
    if (consume_flag(argv, "--verbose")) {
        FLAG_verbose = 1;
        --argc;
    }

    // --help never needs a compute device: printing usage should not pay
    // the Metal dylib build/load latency or emit device-init logs, so the
    // GPU load below is skipped. The CLI's help path ends in std::exit(0)
    // (and its print_usage has internal linkage), so the wrapper's help
    // section is appended via atexit: it runs right after upstream's
    // usage text, whether the CLI exits or returns.
    if (has_flag(argv, "-h") || has_flag(argv, "--help")) {
        atexit(print_wrapper_help);
        return transcribe_cli_main(argc, argv);
    }

    load_gpu_backends(argv);

    return transcribe_cli_main(argc, argv);
}