Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

47.3 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * evals.c - OpenAI's Evals API: an eval is a data schema and graders; a
 * run of it takes rows of data (given in the request, or a JSONL file),
 * has the model write a sample for each where the run asks (a template of
 * messages filled from the row), and grades each row: string_check,
 * text_similarity (evals_metrics.c), label_model and score_model, the
 * model itself answering through a JSON schema that holds it to a label or
 * a score. A run goes on a thread of its own, behind every client's
 * request, each row through the same handlers as /v1/chat/completions and
 * /v1/embeddings; its rows' results are written as they come, beside the
 * files (.evals), and the run saved after each, so a stopped server goes
 * on from there.
 */
#include "server.h"

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <yyjson.h>

#include "evals_metrics.h"
#include "files_store.h"
#include "llm/json.h"
#include "store_disk.h"

#define EVAL "eval"
#define RUN "eval.run"
#define MAX_ROWS 10000

static pthread_mutex_t mu = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cv = PTHREAD_COND_INITIALIZER;
static pthread_t worker;
static int running, stopping;
static struct srv *server;
static char work[4200];

/* ---- helpers ---- */

static int bad(struct srv_req *r, const char *msg)
{
    return srv_reply_error(r, 400, "invalid_request_error", "invalid_value",
                           "%s", msg);
}

static int not_found(struct srv_req *r, const char *what, const char *id)
{
    return srv_reply_error(r, 404, "invalid_request_error", "not_found",
                           "No %s found with id '%s'.", what, id);
}

static int reply_text(struct srv_req *r, char *json)
{
    return srv_reply_json(r, 200, json, json ? strlen(json) : 0);
}

static char *store_get(const char *kind, const char *id, char **aux)
{
    char *j = NULL;
    if (srv_store_get(server->store, kind, id, &j, aux) != 0)
        return NULL;
    return j;
}

/* A document of the store as a mutable JSON object (NULL: none). */
static yyjson_mut_val *load(yyjson_mut_doc *d, const char *kind, const char *id)
{
    char *t = store_get(kind, id, NULL);
    yyjson_doc *x = t ? yyjson_read(t, strlen(t), 0) : NULL;
    free(t);
    yyjson_mut_val *o =
        x ? yyjson_val_mut_copy(d, yyjson_doc_get_root(x)) : NULL;
    yyjson_doc_free(x);
    return o;
}

static int save(const char *kind, const char *id, yyjson_mut_val *o,
                const char *aux, int fresh)
{
    char *t = yyjson_mut_val_write(o, 0, NULL);
    int rc = -1;
    if (t)
        rc = fresh ? srv_store_put(server->store, kind, id, (int64_t)time(NULL),
                                   t, aux)
                   : srv_store_update(server->store, kind, id, t, aux);
    free(t);
    return rc;
}

static int metadata_ok(yyjson_val *m)
{
    if (!m || yyjson_is_null(m))
        return 1;
    if (!yyjson_is_obj(m) || yyjson_obj_size(m) > 16)
        return 0;
    size_t i, n;
    yyjson_val *k, *v;
    yyjson_obj_foreach(m, i, n, k, v) if (!yyjson_is_str(v) ||
                                          yyjson_get_len(v) > 512) return 0;
    return 1;
}

/* ---- templates: {{item.a.b}}, {{sample.output_text}} ---- */

static yyjson_val *path(yyjson_val *root, const char *p, size_t n)
{
    yyjson_val *v = root;
    while (v && n) {
        size_t k = 0;
        while (k < n && p[k] != '.')
            k++;
        char key[128];
        if (k >= sizeof key)
            return NULL;
        memcpy(key, p, k);
        key[k] = 0;
        if (yyjson_is_arr(v) && isdigit((unsigned char)key[0]))
            v = yyjson_arr_get(v, (size_t)atol(key));
        else
            v = yyjson_obj_get(v, key);
        p += k;
        n -= k;
        if (n && *p == '.')
            p++, n--;
    }
    return v;
}

/* A template filled from the namespaces item and sample (malloc'd). */
static char *render(const char *t, yyjson_val *item, yyjson_val *sample)
{
    struct janas_buf b = {0};
    for (const char *p = t; *p;) {
        const char *open = strstr(p, "{{");
        if (!open) {
            janas_buf_put(&b, p, strlen(p));
            break;
        }
        janas_buf_put(&b, p, (size_t)(open - p));
        const char *close = strstr(open + 2, "}}");
        if (!close) {
            janas_buf_put(&b, open, strlen(open));
            break;
        }
        const char *a = open + 2, *e = close;
        while (a < e && *a == ' ')
            a++;
        while (e > a && e[-1] == ' ')
            e--;
        yyjson_val *v = NULL;
        if ((size_t)(e - a) >= 5 && !strncmp(a, "item.", 5))
            v = path(item, a + 5, (size_t)(e - a) - 5);
        else if ((size_t)(e - a) == 4 && !strncmp(a, "item", 4))
            v = item;
        else if ((size_t)(e - a) >= 7 && !strncmp(a, "sample.", 7))
            v = path(sample, a + 7, (size_t)(e - a) - 7);
        if (yyjson_is_str(v))
            janas_buf_put(&b, yyjson_get_str(v), yyjson_get_len(v));
        else if (v) {
            size_t n;
            char *j = yyjson_val_write(v, 0, &n);
            if (j)
                janas_buf_put(&b, j, n);
            free(j);
        }
        p = close + 2;
    }
    janas_buf_put(&b, "", 1);
    if (b.oom) {
        janas_buf_free(&b);
        return NULL;
    }
    return b.p;
}

/* A message's text: content a string, or {type: input_text, text}, or a
   list of those. */
static const char *message_text(yyjson_val *m)
{
    yyjson_val *c = yyjson_obj_get(m, "content");
    if (yyjson_is_str(c))
        return yyjson_get_str(c);
    if (yyjson_is_obj(c))
        return yyjson_get_str(yyjson_obj_get(c, "text"));
    if (yyjson_is_arr(c))
        return yyjson_get_str(yyjson_obj_get(yyjson_arr_get_first(c), "text"));
    return NULL;
}

/* ---- the model, asked from inside ---- */

/* A chat completion of messages (a JSON array text) with the run's
   sampling, and an optional response_format (JSON text): the status, the
   answer's body in *out (malloc'd). */
static unsigned chat(const char *messages, yyjson_val *sampling,
                     const char *format, char **out)
{
    *out = NULL;
    srv_handler h = srv_route_find("POST", "/chat/completions");
    if (!h)
        return 500;
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *o = yyjson_mut_obj(d);
    yyjson_mut_doc_set_root(d, o);
    yyjson_doc *md = yyjson_read(messages, strlen(messages), 0);
    yyjson_mut_obj_add_val(d, o, "messages",
                           yyjson_val_mut_copy(d, yyjson_doc_get_root(md)));
    static const char *const keys[] = {"temperature", "top_p", "seed"};
    for (int i = 0; i < 3; i++) {
        yyjson_val *v = yyjson_obj_get(sampling, keys[i]);
        if (yyjson_is_num(v))
            yyjson_mut_obj_add_val(d, o, keys[i], yyjson_val_mut_copy(d, v));
    }
    yyjson_val *mx = yyjson_obj_get(sampling, "max_completion_tokens");
    if (!mx)
        mx = yyjson_obj_get(sampling, "max_completions_tokens");
    yyjson_mut_obj_add_int(d, o, "max_completion_tokens",
                           yyjson_is_int(mx) ? yyjson_get_sint(mx) : 1024);
    /* graders and samples answer, they do not muse at length */
    yyjson_val *re = yyjson_obj_get(sampling, "reasoning_effort");
    yyjson_mut_obj_add_strcpy(d, o, "reasoning_effort",
                              yyjson_is_str(re) ? yyjson_get_str(re) : "none");
    yyjson_doc *fd = format ? yyjson_read(format, strlen(format), 0) : NULL;
    if (fd)
        yyjson_mut_obj_add_val(d, o, "response_format",
                               yyjson_val_mut_copy(d, yyjson_doc_get_root(fd)));
    size_t bn;
    char *body = yyjson_mut_write(d, 0, &bn);
    yyjson_mut_doc_free(d);
    yyjson_doc_free(md);
    yyjson_doc_free(fd);
    struct srv_capture cap = {0};
    struct srv_req r = {.srv = server,
                        .cap = &cap,
                        .method = "POST",
                        .path = "/chat/completions",
                        .body = body ? body : "{}",
                        .n_body = body ? bn : 2};
    h(&r);
    free(body);
    *out = cap.body;
    return cap.status;
}

/* The messages of a template filled from a row, as a JSON array text. */
static char *fill_messages(yyjson_val *tmpl, yyjson_val *item,
                           yyjson_val *sample)
{
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *arr = yyjson_mut_arr(d);
    size_t i, n;
    yyjson_val *m;
    yyjson_arr_foreach(tmpl, i, n, m)
    {
        const char *role = yyjson_get_str(yyjson_obj_get(m, "role"));
        const char *text = message_text(m);
        char *t = render(text ? text : "", item, sample);
        yyjson_mut_val *o = yyjson_mut_arr_add_obj(d, arr);
        yyjson_mut_obj_add_strcpy(d, o, "role",
                                  role && !strcmp(role, "developer") ? "system"
                                  : role                             ? role
                                                                     : "user");
        yyjson_mut_obj_add_strcpy(d, o, "content", t ? t : "");
        free(t);
    }
    char *out = yyjson_mut_val_write(arr, 0, NULL);
    yyjson_mut_doc_free(d);
    return out;
}

/* ---- graders ---- */

/* One grader on a row: its result object into results; 1 passed, 0
   failed, -1 an error. */
static int grade(yyjson_mut_doc *d, yyjson_mut_val *results, yyjson_val *g,
                 yyjson_val *item, yyjson_val *sample)
{
    const char *type = yyjson_get_str(yyjson_obj_get(g, "type"));
    const char *name = yyjson_get_str(yyjson_obj_get(g, "name"));
    yyjson_mut_val *o = yyjson_mut_arr_add_obj(d, results);
    yyjson_mut_obj_add_strcpy(d, o, "name", name ? name : "");
    yyjson_mut_obj_add_strcpy(d, o, "type", type ? type : "");
    double score = 0;
    int passed = 0, err = 0;
    if (type && !strcmp(type, "string_check")) {
        char *in =
            render(yyjson_get_str(yyjson_obj_get(g, "input")), item, sample);
        char *ref = render(yyjson_get_str(yyjson_obj_get(g, "reference")), item,
                           sample);
        const char *op = yyjson_get_str(yyjson_obj_get(g, "operation"));
        if (in && ref && op) {
            if (!strcmp(op, "eq"))
                passed = !strcmp(in, ref);
            else if (!strcmp(op, "ne"))
                passed = strcmp(in, ref) != 0;
            else if (!strcmp(op, "like"))
                passed = strstr(in, ref) != NULL;
            else if (!strcmp(op, "ilike"))
                passed = strcasestr(in, ref) != NULL;
        } else {
            err = 1;
        }
        score = passed;
        free(in);
        free(ref);
    } else if (type && !strcmp(type, "text_similarity")) {
        char *in =
            render(yyjson_get_str(yyjson_obj_get(g, "input")), item, sample);
        char *ref = render(yyjson_get_str(yyjson_obj_get(g, "reference")), item,
                           sample);
        const char *m = yyjson_get_str(yyjson_obj_get(g, "evaluation_metric"));
        if (in && ref && m)
            score = srv_eval_similarity(m, in, ref);
        else
            err = 1;
        yyjson_val *th = yyjson_obj_get(g, "pass_threshold");
        passed = !err && score >= (th ? yyjson_get_num(th) : 1);
        free(in);
        free(ref);
    } else if (type &&
               (!strcmp(type, "label_model") || !strcmp(type, "score_model"))) {
        int label = type[0] == 'l';
        char *msgs = fill_messages(yyjson_obj_get(g, "input"), item, sample);
        /* the answer held to a label of the list, or a number */
        yyjson_mut_doc *fd = yyjson_mut_doc_new(NULL);
        yyjson_mut_val *f = yyjson_mut_obj(fd);
        yyjson_mut_doc_set_root(fd, f);
        yyjson_mut_obj_add_str(fd, f, "type", "json_schema");
        yyjson_mut_val *js = yyjson_mut_obj_add_obj(fd, f, "json_schema");
        yyjson_mut_obj_add_str(fd, js, "name", "grade");
        yyjson_mut_val *sc = yyjson_mut_obj_add_obj(fd, js, "schema");
        yyjson_mut_obj_add_str(fd, sc, "type", "object");
        yyjson_mut_val *pr = yyjson_mut_obj_add_obj(fd, sc, "properties");
        yyjson_mut_val *res = yyjson_mut_obj_add_obj(fd, pr, "result");
        yyjson_val *range = yyjson_obj_get(g, "range");
        double lo = 0, hi = 1;
        if (yyjson_arr_size(range) == 2) {
            lo = yyjson_get_num(yyjson_arr_get(range, 0));
            hi = yyjson_get_num(yyjson_arr_get(range, 1));
        }
        if (label) {
            yyjson_mut_obj_add_str(fd, res, "type", "string");
            yyjson_mut_obj_add_val(
                fd, res, "enum",
                yyjson_val_mut_copy(fd, yyjson_obj_get(g, "labels")));
        } else {
            yyjson_mut_obj_add_str(fd, res, "type", "number");
            yyjson_mut_obj_add_real(fd, res, "minimum", lo);
            yyjson_mut_obj_add_real(fd, res, "maximum", hi);
        }
        yyjson_mut_val *rq = yyjson_mut_obj_add_arr(fd, sc, "required");
        yyjson_mut_arr_add_str(fd, rq, "result");
        yyjson_mut_obj_add_bool(fd, sc, "additionalProperties", false);
        char *fmt = yyjson_mut_write(fd, 0, NULL);
        yyjson_mut_doc_free(fd);
        char *body = NULL;
        unsigned st =
            msgs && fmt
                ? chat(msgs, yyjson_obj_get(g, "sampling_params"), fmt, &body)
                : 500;
        free(msgs);
        free(fmt);
        yyjson_doc *bd = body ? yyjson_read(body, strlen(body), 0) : NULL;
        const char *content = yyjson_get_str(yyjson_obj_get(
            yyjson_obj_get(yyjson_arr_get_first(yyjson_obj_get(
                               yyjson_doc_get_root(bd), "choices")),
                           "message"),
            "content"));
        yyjson_doc *cd =
            content ? yyjson_read(content, strlen(content), 0) : NULL;
        yyjson_val *rv = yyjson_obj_get(yyjson_doc_get_root(cd), "result");
        if (st != 200 || !rv) {
            err = 1;
        } else if (label) {
            const char *l = yyjson_get_str(rv);
            size_t i, n;
            yyjson_val *x;
            yyjson_arr_foreach(yyjson_obj_get(g, "passing_labels"), i, n, x)
                passed |= l && yyjson_equals_str(x, l);
            score = passed;
            yyjson_mut_val *smp = yyjson_mut_obj_add_obj(d, o, "sample");
            yyjson_mut_obj_add_strcpy(d, smp, "label", l ? l : "");
        } else {
            score = yyjson_get_num(rv);
            yyjson_val *th = yyjson_obj_get(g, "pass_threshold");
            passed = score >= (th ? yyjson_get_num(th) : (lo + hi) / 2);
        }
        yyjson_doc_free(cd);
        yyjson_doc_free(bd);
        free(body);
    } else {
        err = 1;
    }
    yyjson_mut_obj_add_real(d, o, "score", score);
    yyjson_mut_obj_add_bool(d, o, "passed", passed && !err);
    if (err)
        yyjson_mut_obj_add_str(d, o, "error", "the grader could not run");
    return err ? -1 : passed;
}

/* ---- a run, row by row ---- */

static void out_path(char *p, size_t cap, const char *run)
{
    snprintf(p, cap, "%s/%s.jsonl", work, run);
}

/* The rows of a run's data source: a JSON array text of {item, sample?}
   (malloc'd), or NULL with the reason. */
static char *rows_of(yyjson_val *ds, char *why, size_t wl)
{
    yyjson_val *src = yyjson_obj_get(ds, "source");
    const char *st = yyjson_get_str(yyjson_obj_get(src, "type"));
    if (st && !strcmp(st, "file_content")) {
        yyjson_val *c = yyjson_obj_get(src, "content");
        if (!yyjson_is_arr(c)) {
            snprintf(why, wl, "the source's content is a list of rows");
            return NULL;
        }
        return yyjson_val_write(c, 0, NULL);
    }
    if (st && !strcmp(st, "file_id")) {
        const char *id = yyjson_get_str(yyjson_obj_get(src, "id"));
        struct srv_file f;
        if (!id || srv_files_get(server->files, id, &f) != 0 ||
            f.bytes > ((size_t)64 << 20)) {
            snprintf(why, wl, "the source's file is not here");
            return NULL;
        }
        int fd = srv_files_open_content(server->files, id);
        char *t = fd >= 0 ? malloc(f.bytes + 1) : NULL;
        size_t got = 0;
        while (t && got < f.bytes) {
            ssize_t k = read(fd, t + got, f.bytes - got);
            if (k <= 0)
                break;
            got += (size_t)k;
        }
        if (fd >= 0)
            close(fd);
        if (!t) {
            snprintf(why, wl, "the source's file cannot be read");
            return NULL;
        }
        t[got] = 0;
        /* its JSON lines into an array */
        struct janas_buf b = {0};
        janas_buf_put(&b, "[", 1);
        int first = 1;
        for (char *l = t; *l;) {
            char *e = strchr(l, '\n');
            size_t n = e ? (size_t)(e - l) : strlen(l);
            size_t k = 0;
            while (k < n && isspace((unsigned char)l[k]))
                k++;
            if (k < n) {
                if (!first)
                    janas_buf_put(&b, ",", 1);
                janas_buf_put(&b, l, n);
                first = 0;
            }
            l = e ? e + 1 : l + n;
        }
        janas_buf_put(&b, "]", 2);
        free(t);
        if (b.oom || !yyjson_read(b.p, b.n - 1, 0)) {
            janas_buf_free(&b);
            snprintf(why, wl, "the source's file is not JSON lines");
            return NULL;
        }
        return b.p;
    }
    snprintf(why, wl, "the source is file_content or file_id");
    return NULL;
}

static void run_one(const char *run_id)
{
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    char *aux = NULL;
    char *rt = store_get(RUN, run_id, &aux);
    yyjson_doc *rd = rt ? yyjson_read(rt, strlen(rt), 0) : NULL;
    free(rt);
    yyjson_val *run = yyjson_doc_get_root(rd);
    const char *eval_id = yyjson_get_str(yyjson_obj_get(run, "eval_id"));
    char *et = eval_id ? store_get(EVAL, eval_id, NULL) : NULL;
    yyjson_doc *ed = et ? yyjson_read(et, strlen(et), 0) : NULL;
    free(et);
    yyjson_val *criteria =
        yyjson_obj_get(yyjson_doc_get_root(ed), "testing_criteria");
    yyjson_val *ds = yyjson_obj_get(run, "data_source");
    char why[200] = "";
    char *rows = run && criteria ? rows_of(ds, why, sizeof why) : NULL;
    yyjson_doc *rowsd = rows ? yyjson_read(rows, strlen(rows), 0) : NULL;
    free(rows);
    yyjson_val *all = yyjson_doc_get_root(rowsd);
    yyjson_mut_val *o = run ? yyjson_val_mut_copy(d, run) : NULL;
    long next = aux ? atol(aux) : 0;
    free(aux);
    if (!o) {
        yyjson_doc_free(rd);
        yyjson_doc_free(ed);
        yyjson_doc_free(rowsd);
        yyjson_mut_doc_free(d);
        return;
    }
    if (!all || yyjson_arr_size(all) > MAX_ROWS) {
        yyjson_mut_obj_remove_key(o, "status");
        yyjson_mut_obj_add_str(d, o, "status", "failed");
        yyjson_mut_obj_remove_key(o, "error");
        yyjson_mut_val *e = yyjson_mut_obj_add_obj(d, o, "error");
        yyjson_mut_obj_add_str(d, e, "code", "invalid_data_source");
        yyjson_mut_obj_add_strcpy(d, e, "message",
                                  all ? "more than 10,000 rows" : why);
        save(RUN, run_id, o, "0", 0);
        yyjson_doc_free(rd);
        yyjson_doc_free(ed);
        yyjson_doc_free(rowsd);
        yyjson_mut_doc_free(d);
        return;
    }
    yyjson_mut_obj_remove_key(o, "status");
    yyjson_mut_obj_add_str(d, o, "status", "in_progress");
    const char *dtype = yyjson_get_str(yyjson_obj_get(ds, "type"));
    int sampled = dtype && strcmp(dtype, "jsonl") != 0;
    yyjson_val *tmpl =
        yyjson_obj_get(yyjson_obj_get(ds, "input_messages"), "template");
    const char *iref = yyjson_get_str(
        yyjson_obj_get(yyjson_obj_get(ds, "input_messages"), "item_reference"));
    yyjson_val *sampling = yyjson_obj_get(ds, "sampling_params");
    size_t total = yyjson_arr_size(all), nc = yyjson_arr_size(criteria);
    char op[4600];
    out_path(op, sizeof op, run_id);
    for (size_t i = (size_t)next; i < total; i++) {
        pthread_mutex_lock(&mu);
        int stop = stopping;
        pthread_mutex_unlock(&mu);
        /* a cancel saved meanwhile */
        char *now = store_get(RUN, run_id, NULL);
        int cancelled = now && strstr(now, "\"status\":\"canceled\"");
        free(now);
        if (stop || cancelled)
            break;
        yyjson_val *row = yyjson_arr_get(all, i);
        yyjson_val *item = yyjson_obj_get(row, "item");
        if (!item)
            item = row;
        yyjson_mut_doc *od = yyjson_mut_doc_new(NULL);
        yyjson_mut_val *oi = yyjson_mut_obj(od);
        char oid[48];
        srv_new_id(oid, sizeof oid, "outputitem_");
        yyjson_mut_obj_add_str(od, oi, "object", "eval.run.output_item");
        yyjson_mut_obj_add_strcpy(od, oi, "id", oid);
        yyjson_mut_obj_add_strcpy(od, oi, "run_id", run_id);
        yyjson_mut_obj_add_strcpy(od, oi, "eval_id", eval_id);
        yyjson_mut_obj_add_int(od, oi, "created_at", (int64_t)time(NULL));
        yyjson_mut_obj_add_int(od, oi, "datasource_item_id", (int64_t)i);
        yyjson_mut_obj_add_val(od, oi, "datasource_item",
                               yyjson_val_mut_copy(od, item));
        /* the sample: the row's, or the model's answer to the template */
        yyjson_doc *sd = NULL;
        int errored = 0;
        yyjson_mut_val *smp = yyjson_mut_obj(od);
        if (sampled) {
            char *msgs = NULL;
            if (tmpl)
                msgs = fill_messages(tmpl, item, NULL);
            else if (iref) {
                const char *p = !strncmp(iref, "item.", 5) ? iref + 5 : iref;
                yyjson_val *ms = path(item, p, strlen(p));
                msgs = ms ? fill_messages(ms, item, NULL) : NULL;
            }
            char *body = NULL;
            unsigned st = msgs ? chat(msgs, sampling, NULL, &body) : 400;
            yyjson_doc *bd = body ? yyjson_read(body, strlen(body), 0) : NULL;
            yyjson_val *ch0 = yyjson_arr_get_first(
                yyjson_obj_get(yyjson_doc_get_root(bd), "choices"));
            const char *text = yyjson_get_str(
                yyjson_obj_get(yyjson_obj_get(ch0, "message"), "content"));
            yyjson_doc *md = msgs ? yyjson_read(msgs, strlen(msgs), 0) : NULL;
            yyjson_mut_obj_add_val(
                od, smp, "input",
                md ? yyjson_val_mut_copy(od, yyjson_doc_get_root(md))
                   : yyjson_mut_arr(od));
            yyjson_mut_val *outs = yyjson_mut_obj_add_arr(od, smp, "output");
            yyjson_mut_val *om = yyjson_mut_arr_add_obj(od, outs);
            yyjson_mut_obj_add_str(od, om, "role", "assistant");
            yyjson_mut_obj_add_strcpy(od, om, "content", text ? text : "");
            const char *fr =
                yyjson_get_str(yyjson_obj_get(ch0, "finish_reason"));
            yyjson_mut_obj_add_strcpy(od, smp, "finish_reason",
                                      fr ? fr : "error");
            yyjson_mut_obj_add_strcpy(od, smp, "model", server->cfg.model_id);
            yyjson_val *u = yyjson_obj_get(yyjson_doc_get_root(bd), "usage");
            if (u)
                yyjson_mut_obj_add_val(od, smp, "usage",
                                       yyjson_val_mut_copy(od, u));
            yyjson_mut_obj_add_strcpy(od, smp, "output_text", text ? text : "");
            if (st != 200) {
                errored = 1;
                yyjson_mut_val *e = yyjson_mut_obj_add_obj(od, smp, "error");
                yyjson_mut_obj_add_str(od, e, "code", "sample_failed");
                yyjson_mut_obj_add_strcpy(od, e, "message",
                                          body ? body : "no answer");
            }
            yyjson_doc_free(md);
            yyjson_doc_free(bd);
            free(body);
            free(msgs);
        } else if (yyjson_obj_get(row, "sample")) {
            yyjson_val *rs = yyjson_obj_get(row, "sample");
            smp = yyjson_val_mut_copy(od, rs);
        }
        /* the graders see the sample as written */
        char *st = yyjson_mut_val_write(smp, 0, NULL);
        sd = st ? yyjson_read(st, strlen(st), 0) : NULL;
        free(st);
        yyjson_mut_val *results = yyjson_mut_arr(od);
        int all_pass = !errored;
        yyjson_val *g;
        size_t gi, gm;
        yyjson_arr_foreach(criteria, gi, gm, g)
        {
            int p = errored
                        ? -1
                        : grade(od, results, g, item, yyjson_doc_get_root(sd));
            if (p < 0)
                errored = 1;
            if (p != 1)
                all_pass = 0;
        }
        yyjson_doc_free(sd);
        yyjson_mut_obj_add_val(od, oi, "results", results);
        yyjson_mut_obj_add_val(od, oi, "sample", smp);
        yyjson_mut_obj_add_str(od, oi, "status",
                               errored    ? "error"
                               : all_pass ? "pass"
                                          : "fail");
        size_t ln;
        char *line = yyjson_mut_val_write(oi, 0, &ln);
        int fd = open(op, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0600);
        if (fd >= 0 && line) {
            ssize_t w1 = write(fd, line, ln), w2 = write(fd, "\n", 1);
            (void)w1;
            (void)w2;
        }
        if (fd >= 0)
            close(fd);
        free(line);
        /* the run's counts */
        yyjson_mut_val *rc = yyjson_mut_obj_get(o, "result_counts");
        static const char *const K[] = {"total", "errored", "failed", "passed"};
        int64_t c[4];
        for (int k = 0; k < 4; k++)
            c[k] = yyjson_mut_get_sint(yyjson_mut_obj_get(rc, K[k]));
        c[0]++;
        c[errored ? 1 : all_pass ? 3 : 2]++;
        yyjson_mut_obj_remove_key(o, "result_counts");
        rc = yyjson_mut_obj_add_obj(d, o, "result_counts");
        for (int k = 0; k < 4; k++)
            yyjson_mut_obj_add_int(d, rc, K[k], c[k]);
        /* per grader: its passes and fails */
        yyjson_mut_val *pt =
            yyjson_mut_obj_get(o, "per_testing_criteria_results");
        size_t ri;
        yyjson_mut_val *rv;
        size_t rmax;
        yyjson_mut_arr_foreach(results, ri, rmax, rv)
        {
            yyjson_mut_val *slot = yyjson_mut_arr_get(pt, ri);
            if (!slot || ri >= nc)
                continue;
            const char *key =
                yyjson_mut_get_bool(yyjson_mut_obj_get(rv, "passed"))
                    ? "passed"
                    : "failed";
            int64_t v = yyjson_mut_get_sint(yyjson_mut_obj_get(slot, key));
            yyjson_mut_obj_remove_key(slot, key);
            yyjson_mut_obj_add_int(d, slot, key, v + 1);
        }
        yyjson_mut_doc_free(od);
        char nx[32];
        snprintf(nx, sizeof nx, "%zu", i + 1);
        pthread_mutex_lock(&mu);
        char *cur = store_get(RUN, run_id, NULL);
        int was_cancelled = cur && strstr(cur, "\"status\":\"canceled\"");
        free(cur);
        if (was_cancelled) {
            yyjson_mut_obj_remove_key(o, "status");
            yyjson_mut_obj_add_str(d, o, "status", "canceled");
        }
        save(RUN, run_id, o, nx, 0);
        pthread_mutex_unlock(&mu);
        if (was_cancelled)
            break;
    }
    pthread_mutex_lock(&mu);
    char *cur = store_get(RUN, run_id, NULL);
    int was_cancelled = cur && strstr(cur, "\"status\":\"canceled\"");
    free(cur);
    if (!stopping && !was_cancelled) {
        yyjson_mut_obj_remove_key(o, "status");
        yyjson_mut_obj_add_str(d, o, "status", "completed");
        char nx[32];
        snprintf(nx, sizeof nx, "%zu", total);
        save(RUN, run_id, o, nx, 0);
    }
    pthread_mutex_unlock(&mu);
    yyjson_doc_free(rd);
    yyjson_doc_free(ed);
    yyjson_doc_free(rowsd);
    yyjson_mut_doc_free(d);
}

/* The oldest run with work left: 1 with its id. */
static int next_run(char *id, size_t cap)
{
    char **ids;
    size_t n = srv_store_list(server->store, RUN, &ids);
    int found = 0;
    for (size_t i = 0; i < n && !found; i++) {
        char *t = store_get(RUN, ids[i], NULL);
        if (t && (strstr(t, "\"status\":\"queued\"") ||
                  strstr(t, "\"status\":\"in_progress\""))) {
            snprintf(id, cap, "%s", ids[i]);
            found = 1;
        }
        free(t);
    }
    srv_store_list_free(ids, n);
    return found;
}

static void *worker_main(void *arg)
{
    (void)arg;
    srv_engine_low = 1;
    char id[64];
    for (;;) {
        pthread_mutex_lock(&mu);
        while (!stopping && !next_run(id, sizeof id))
            pthread_cond_wait(&cv, &mu);
        int stop = stopping;
        pthread_mutex_unlock(&mu);
        if (stop)
            break;
        run_one(id);
    }
    return NULL;
}

int srv_evals_start(struct srv *s)
{
    server = s;
    snprintf(work, sizeof work, "%s/.evals", srv_files_dir(s->files));
    if (mkdir(work, 0700) != 0 && errno != EEXIST)
        return -1;
    stopping = 0;
    if (pthread_create(&worker, NULL, worker_main, NULL) != 0)
        return -1;
    running = 1;
    return 0;
}

void srv_evals_stop(void)
{
    if (!running)
        return;
    pthread_mutex_lock(&mu);
    stopping = 1;
    pthread_cond_broadcast(&cv);
    pthread_mutex_unlock(&mu);
    pthread_join(worker, NULL);
    running = 0;
}

/* ---- the API: evals ---- */

static const char *grader_error(yyjson_val *g)
{
    const char *type = yyjson_get_str(yyjson_obj_get(g, "type"));
    if (!type || !yyjson_get_str(yyjson_obj_get(g, "name")))
        return "Every grader has a type and a name.";
    if (!strcmp(type, "string_check")) {
        const char *op = yyjson_get_str(yyjson_obj_get(g, "operation"));
        if (!yyjson_get_str(yyjson_obj_get(g, "input")) ||
            !yyjson_get_str(yyjson_obj_get(g, "reference")) || !op ||
            (strcmp(op, "eq") && strcmp(op, "ne") && strcmp(op, "like") &&
             strcmp(op, "ilike")))
            return "A string_check grader has input, reference and an "
                   "operation of eq, ne, like or ilike.";
        return NULL;
    }
    if (!strcmp(type, "text_similarity"))
        return yyjson_get_str(yyjson_obj_get(g, "input")) &&
                       yyjson_get_str(yyjson_obj_get(g, "reference")) &&
                       srv_eval_metric_ok(yyjson_get_str(
                           yyjson_obj_get(g, "evaluation_metric")))
                   ? NULL
                   : "A text_similarity grader has input, reference and an "
                     "evaluation_metric.";
    if (!strcmp(type, "label_model"))
        return yyjson_is_arr(yyjson_obj_get(g, "input")) &&
                       yyjson_arr_size(yyjson_obj_get(g, "labels")) &&
                       yyjson_is_arr(yyjson_obj_get(g, "passing_labels"))
                   ? NULL
                   : "A label_model grader has input messages, labels and "
                     "passing_labels.";
    if (!strcmp(type, "score_model"))
        return yyjson_is_arr(yyjson_obj_get(g, "input"))
                   ? NULL
                   : "A score_model grader has input messages.";
    if (!strcmp(type, "python"))
        return "python graders are not run here.";
    return "A grader is string_check, text_similarity, label_model or "
           "score_model.";
}

int srv_evals_create(struct srv_req *r)
{
    yyjson_doc *doc = yyjson_read(r->body, r->n_body, 0);
    yyjson_val *root = yyjson_doc_get_root(doc);
    yyjson_val *dsc = yyjson_obj_get(root, "data_source_config");
    yyjson_val *tc = yyjson_obj_get(root, "testing_criteria");
    yyjson_val *md = yyjson_obj_get(root, "metadata");
    const char *dtype = yyjson_get_str(yyjson_obj_get(dsc, "type"));
    const char *why = NULL;
    if (!yyjson_is_obj(root))
        why = "The body is a JSON object.";
    else if (!dtype || (strcmp(dtype, "custom") && strcmp(dtype, "logs") &&
                        strcmp(dtype, "stored_completions")))
        why = "'data_source_config' is custom, logs or stored_completions.";
    else if (!strcmp(dtype, "custom") &&
             !yyjson_is_obj(yyjson_obj_get(dsc, "item_schema")))
        why = "A custom data source config has an item_schema.";
    else if (!yyjson_is_arr(tc) || !yyjson_arr_size(tc))
        why = "'testing_criteria' is a list of graders.";
    else if (!metadata_ok(md))
        why = "'metadata' holds at most 16 strings.";
    size_t i, n;
    yyjson_val *g;
    yyjson_arr_foreach(tc, i, n, g) if (!why) why = grader_error(g);
    if (why) {
        yyjson_doc_free(doc);
        return bad(r, why);
    }
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *o = yyjson_mut_obj(d);
    char id[48];
    srv_new_id(id, sizeof id, "eval_");
    yyjson_mut_obj_add_str(d, o, "object", "eval");
    yyjson_mut_obj_add_strcpy(d, o, "id", id);
    const char *name = yyjson_get_str(yyjson_obj_get(root, "name"));
    yyjson_mut_obj_add_strcpy(d, o, "name", name ? name : "");
    /* the config as OpenAI gives it back: its schema of item and sample */
    yyjson_mut_val *c = yyjson_mut_obj_add_obj(d, o, "data_source_config");
    yyjson_mut_obj_add_strcpy(d, c, "type", dtype);
    yyjson_mut_val *sch = yyjson_mut_obj_add_obj(d, c, "schema");
    yyjson_mut_obj_add_str(d, sch, "type", "object");
    yyjson_mut_val *props = yyjson_mut_obj_add_obj(d, sch, "properties");
    yyjson_val *is = yyjson_obj_get(dsc, "item_schema");
    yyjson_mut_obj_add_val(d, props, "item",
                           is ? yyjson_val_mut_copy(d, is) : yyjson_mut_obj(d));
    yyjson_mut_val *req = yyjson_mut_obj_add_arr(d, sch, "required");
    yyjson_mut_arr_add_str(d, req, "item");
    if (yyjson_get_bool(yyjson_obj_get(dsc, "include_sample_schema"))) {
        yyjson_mut_val *s = yyjson_mut_obj_add_obj(d, props, "sample");
        yyjson_mut_obj_add_str(d, s, "type", "object");
        yyjson_mut_arr_add_str(d, req, "sample");
    }
    yyjson_mut_obj_add_val(d, o, "testing_criteria",
                           yyjson_val_mut_copy(d, tc));
    yyjson_mut_obj_add_int(d, o, "created_at", (int64_t)time(NULL));
    yyjson_mut_obj_add_val(d, o, "metadata",
                           yyjson_is_obj(md) ? yyjson_val_mut_copy(d, md)
                                             : yyjson_mut_obj(d));
    yyjson_doc_free(doc);
    int rc = save(EVAL, id, o, NULL, 1);
    char *out = rc == 0 ? yyjson_mut_val_write(o, 0, NULL) : NULL;
    yyjson_mut_doc_free(d);
    if (!out)
        return srv_reply_error(r, 500, "server_error", NULL,
                               "cannot keep the eval");
    return reply_text(r, out);
}

int srv_evals_get(struct srv_req *r)
{
    char *t = store_get(EVAL, r->param[0], NULL);
    return t ? reply_text(r, t) : not_found(r, "eval", r->param[0]);
}

int srv_evals_update(struct srv_req *r)
{
    yyjson_doc *doc = yyjson_read(r->body, r->n_body, 0);
    yyjson_val *root = yyjson_doc_get_root(doc);
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *o = load(d, EVAL, r->param[0]);
    yyjson_val *name = yyjson_obj_get(root, "name");
    yyjson_val *md = yyjson_obj_get(root, "metadata");
    int ret;
    if (!o) {
        ret = not_found(r, "eval", r->param[0]);
    } else if (!yyjson_is_obj(root) || !metadata_ok(md)) {
        ret = bad(r, "The body is {name, metadata}, metadata at most 16 "
                     "strings.");
    } else {
        if (yyjson_is_str(name)) {
            yyjson_mut_obj_remove_key(o, "name");
            yyjson_mut_obj_add_strcpy(d, o, "name", yyjson_get_str(name));
        }
        if (md) {
            yyjson_mut_obj_remove_key(o, "metadata");
            yyjson_mut_obj_add_val(d, o, "metadata",
                                   yyjson_is_obj(md)
                                       ? yyjson_val_mut_copy(d, md)
                                       : yyjson_mut_obj(d));
        }
        ret = save(EVAL, r->param[0], o, NULL, 0) == 0
                  ? reply_text(r, yyjson_mut_val_write(o, 0, NULL))
                  : srv_reply_error(r, 500, "server_error", NULL,
                                    "cannot keep the eval");
    }
    yyjson_mut_doc_free(d);
    yyjson_doc_free(doc);
    return ret;
}

int srv_evals_delete(struct srv_req *r)
{
    if (srv_store_del(r->srv->store, EVAL, r->param[0]) != 0)
        return not_found(r, "eval", r->param[0]);
    char out[512];
    snprintf(
        out, sizeof out,
        "{\"object\":\"eval.deleted\",\"deleted\":true,\"eval_id\":\"%s\"}",
        r->param[0]);
    return reply_text(r, strdup(out));
}

/* A page of documents of kind (those of eval_id for runs), filtered by a
   key's value; asc by default as OpenAI lists evals. */
static int list_page(struct srv_req *r, const char *kind, const char *eval_id,
                     const char *status)
{
    const char *l = srv_query(r, "limit"), *o = srv_query(r, "order");
    const char *after = srv_query(r, "after");
    long limit = 20;
    if (l) {
        char *end;
        limit = strtol(l, &end, 10);
        if (*end || limit < 1 || limit > 100)
            return bad(r, "'limit' is a number from 1 to 100.");
    }
    int desc = o && !strcmp(o, "desc");
    char **ids;
    size_t n = srv_store_list(r->srv->store, kind, &ids);
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *root = yyjson_mut_obj(d), *data = yyjson_mut_arr(d);
    long shown = 0;
    int more = 0, past = after == NULL;
    char first[64] = "", last[64] = "";
    for (size_t k = 0; k < n; k++) {
        const char *id = ids[desc ? n - 1 - k : k];
        char *t = store_get(kind, id, NULL);
        yyjson_doc *x = t ? yyjson_read(t, strlen(t), 0) : NULL;
        free(t);
        yyjson_val *v = yyjson_doc_get_root(x);
        int keep =
            v &&
            (!eval_id ||
             yyjson_equals_str(yyjson_obj_get(v, "eval_id"), eval_id)) &&
            (!status || yyjson_equals_str(yyjson_obj_get(v, "status"), status));
        if (keep && !past) {
            past = !strcmp(id, after);
            keep = 0;
        }
        if (keep && shown == limit) {
            more = 1;
            yyjson_doc_free(x);
            break;
        }
        if (keep) {
            yyjson_mut_arr_append(data, yyjson_val_mut_copy(d, v));
            if (!shown)
                snprintf(first, sizeof first, "%s", id);
            snprintf(last, sizeof last, "%s", id);
            shown++;
        }
        yyjson_doc_free(x);
    }
    srv_store_list_free(ids, n);
    yyjson_mut_obj_add_str(d, root, "object", "list");
    yyjson_mut_obj_add_val(d, root, "data", data);
    if (shown) {
        yyjson_mut_obj_add_strcpy(d, root, "first_id", first);
        yyjson_mut_obj_add_strcpy(d, root, "last_id", last);
    } else {
        yyjson_mut_obj_add_null(d, root, "first_id");
        yyjson_mut_obj_add_null(d, root, "last_id");
    }
    yyjson_mut_obj_add_bool(d, root, "has_more", more);
    char *out = yyjson_mut_val_write(root, 0, NULL);
    yyjson_mut_doc_free(d);
    return reply_text(r, out);
}

int srv_evals_list(struct srv_req *r)
{
    return list_page(r, EVAL, NULL, NULL);
}

/* ---- the API: runs ---- */

int srv_eval_runs_create(struct srv_req *r)
{
    char *et = store_get(EVAL, r->param[0], NULL);
    yyjson_doc *ed = et ? yyjson_read(et, strlen(et), 0) : NULL;
    free(et);
    if (!ed)
        return not_found(r, "eval", r->param[0]);
    yyjson_val *ev = yyjson_doc_get_root(ed);
    yyjson_doc *doc = yyjson_read(r->body, r->n_body, 0);
    yyjson_val *root = yyjson_doc_get_root(doc);
    yyjson_val *ds = yyjson_obj_get(root, "data_source");
    yyjson_val *md = yyjson_obj_get(root, "metadata");
    const char *dtype = yyjson_get_str(yyjson_obj_get(ds, "type"));
    const char *why = NULL;
    char tmp[200];
    if (!yyjson_is_obj(root) || !dtype)
        why = "'data_source' is jsonl, completions or responses.";
    else if (strcmp(dtype, "jsonl") && strcmp(dtype, "completions") &&
             strcmp(dtype, "responses"))
        why = "'data_source' is jsonl, completions or responses.";
    else if (strcmp(dtype, "jsonl") && !yyjson_obj_get(ds, "input_messages"))
        why = "A completions data source has input_messages (a template or "
              "an item_reference).";
    else if (!metadata_ok(md))
        why = "'metadata' holds at most 16 strings.";
    else {
        char *rows = rows_of(ds, tmp, sizeof tmp);
        if (!rows)
            why = tmp;
        free(rows);
    }
    if (why) {
        yyjson_doc_free(doc);
        yyjson_doc_free(ed);
        return bad(r, why);
    }
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *o = yyjson_mut_obj(d);
    char id[48];
    srv_new_id(id, sizeof id, "evalrun_");
    yyjson_mut_obj_add_str(d, o, "object", "eval.run");
    yyjson_mut_obj_add_strcpy(d, o, "id", id);
    yyjson_mut_obj_add_strcpy(d, o, "eval_id", r->param[0]);
    yyjson_mut_obj_add_str(d, o, "status", "queued");
    yyjson_mut_obj_add_strcpy(d, o, "model", r->srv->cfg.model_id);
    const char *name = yyjson_get_str(yyjson_obj_get(root, "name"));
    yyjson_mut_obj_add_strcpy(d, o, "name", name ? name : id);
    yyjson_mut_obj_add_int(d, o, "created_at", (int64_t)time(NULL));
    yyjson_mut_obj_add_str(d, o, "report_url", "");
    yyjson_mut_val *rc = yyjson_mut_obj_add_obj(d, o, "result_counts");
    yyjson_mut_obj_add_int(d, rc, "total", 0);
    yyjson_mut_obj_add_int(d, rc, "errored", 0);
    yyjson_mut_obj_add_int(d, rc, "failed", 0);
    yyjson_mut_obj_add_int(d, rc, "passed", 0);
    yyjson_mut_obj_add_arr(d, o, "per_model_usage");
    yyjson_mut_val *pt =
        yyjson_mut_obj_add_arr(d, o, "per_testing_criteria_results");
    size_t i, n;
    yyjson_val *g;
    yyjson_arr_foreach(yyjson_obj_get(ev, "testing_criteria"), i, n, g)
    {
        yyjson_mut_val *s = yyjson_mut_arr_add_obj(d, pt);
        const char *gn = yyjson_get_str(yyjson_obj_get(g, "name"));
        yyjson_mut_obj_add_strcpy(d, s, "testing_criteria", gn ? gn : "");
        yyjson_mut_obj_add_int(d, s, "passed", 0);
        yyjson_mut_obj_add_int(d, s, "failed", 0);
    }
    yyjson_mut_obj_add_val(d, o, "data_source", yyjson_val_mut_copy(d, ds));
    yyjson_mut_obj_add_val(d, o, "metadata",
                           yyjson_is_obj(md) ? yyjson_val_mut_copy(d, md)
                                             : yyjson_mut_obj(d));
    yyjson_mut_obj_add_null(d, o, "error");
    yyjson_doc_free(doc);
    yyjson_doc_free(ed);
    int ok = save(RUN, id, o, "0", 1) == 0;
    char *out = ok ? yyjson_mut_val_write(o, 0, NULL) : NULL;
    yyjson_mut_doc_free(d);
    if (!out)
        return srv_reply_error(r, 500, "server_error", NULL,
                               "cannot keep the run");
    pthread_mutex_lock(&mu);
    pthread_cond_broadcast(&cv);
    pthread_mutex_unlock(&mu);
    return reply_text(r, out);
}

static yyjson_doc *run_of(struct srv_req *r)
{
    char *t = store_get(RUN, r->param[1], NULL);
    yyjson_doc *x = t ? yyjson_read(t, strlen(t), 0) : NULL;
    free(t);
    if (x &&
        !yyjson_equals_str(yyjson_obj_get(yyjson_doc_get_root(x), "eval_id"),
                           r->param[0])) {
        yyjson_doc_free(x);
        x = NULL;
    }
    return x;
}

int srv_eval_runs_get(struct srv_req *r)
{
    yyjson_doc *x = run_of(r);
    if (!x)
        return not_found(r, "run", r->param[1]);
    char *out = yyjson_val_write(yyjson_doc_get_root(x), 0, NULL);
    yyjson_doc_free(x);
    return reply_text(r, out);
}

int srv_eval_runs_list(struct srv_req *r)
{
    if (srv_store_get(r->srv->store, EVAL, r->param[0], NULL, NULL) != 0)
        return not_found(r, "eval", r->param[0]);
    return list_page(r, RUN, r->param[0], srv_query(r, "status"));
}

int srv_eval_runs_cancel(struct srv_req *r)
{
    yyjson_doc *x = run_of(r);
    if (!x)
        return not_found(r, "run", r->param[1]);
    yyjson_mut_doc *d = yyjson_doc_mut_copy(x, NULL);
    yyjson_mut_val *o = yyjson_mut_doc_get_root(d);
    const char *st = yyjson_mut_get_str(yyjson_mut_obj_get(o, "status"));
    if (st && (!strcmp(st, "queued") || !strcmp(st, "in_progress"))) {
        yyjson_mut_obj_remove_key(o, "status");
        yyjson_mut_obj_add_str(d, o, "status", "canceled");
        pthread_mutex_lock(&mu);
        save(RUN, r->param[1], o, NULL, 0);
        pthread_mutex_unlock(&mu);
    }
    char *out = yyjson_mut_val_write(o, 0, NULL);
    yyjson_mut_doc_free(d);
    yyjson_doc_free(x);
    return reply_text(r, out);
}

int srv_eval_runs_delete(struct srv_req *r)
{
    yyjson_doc *x = run_of(r);
    if (!x)
        return not_found(r, "run", r->param[1]);
    yyjson_doc_free(x);
    pthread_mutex_lock(&mu);
    srv_store_del(r->srv->store, RUN, r->param[1]);
    pthread_mutex_unlock(&mu);
    char p[4600];
    if (srv_disk_name_ok(r->param[1])) {
        out_path(p, sizeof p, r->param[1]);
        unlink(p);
    }
    char out[512];
    snprintf(out, sizeof out,
             "{\"object\":\"eval.run.deleted\",\"deleted\":true,"
             "\"run_id\":\"%s\"}",
             r->param[1]);
    return reply_text(r, strdup(out));
}

/* The output items of a run, from its file: those with id (NULL: all,
   paged), filtered by status. */
static int items(struct srv_req *r, const char *id)
{
    yyjson_doc *x = run_of(r);
    if (!x)
        return not_found(r, "run", r->param[1]);
    yyjson_doc_free(x);
    char p[4600];
    if (!srv_disk_name_ok(r->param[1]))
        return not_found(r, "run", r->param[1]);
    out_path(p, sizeof p, r->param[1]);
    FILE *f = fopen(p, "rb");
    const char *status = srv_query(r, "status"), *after = srv_query(r, "after");
    const char *l = srv_query(r, "limit"), *o = srv_query(r, "order");
    long limit = l ? strtol(l, NULL, 10) : 20;
    if (limit < 1 || limit > 100)
        limit = 20;
    yyjson_mut_doc *d = yyjson_mut_doc_new(NULL);
    yyjson_mut_val *data = yyjson_mut_arr(d);
    char *line = NULL;
    size_t cap = 0;
    ssize_t n;
    yyjson_mut_val *found = NULL;
    while (f && (n = getline(&line, &cap, f)) > 0) {
        yyjson_doc *ld = yyjson_read(line, (size_t)n, 0);
        yyjson_val *v = yyjson_doc_get_root(ld);
        if (v && (!id || yyjson_equals_str(yyjson_obj_get(v, "id"), id)) &&
            (!status || yyjson_equals_str(yyjson_obj_get(v, "status"), status)))
            yyjson_mut_arr_append(data, yyjson_val_mut_copy(d, v));
        if (v && id && yyjson_equals_str(yyjson_obj_get(v, "id"), id))
            found = yyjson_mut_arr_get_last(data);
        yyjson_doc_free(ld);
    }
    free(line);
    if (f)
        fclose(f);
    if (id) {
        int ret = found ? reply_text(r, yyjson_mut_val_write(found, 0, NULL))
                        : not_found(r, "output item", id);
        yyjson_mut_doc_free(d);
        return ret;
    }
    /* a page of them */
    size_t total = yyjson_mut_arr_size(data);
    int desc = o && !strcmp(o, "desc");
    yyjson_mut_val *root = yyjson_mut_obj(d), *page = yyjson_mut_arr(d);
    long shown = 0;
    int more = 0, past = after == NULL;
    char first[64] = "", last[64] = "";
    for (size_t k = 0; k < total; k++) {
        yyjson_mut_val *v = yyjson_mut_arr_get(data, desc ? total - 1 - k : k);
        const char *vid = yyjson_mut_get_str(yyjson_mut_obj_get(v, "id"));
        if (!past) {
            past = vid && !strcmp(vid, after);
            continue;
        }
        if (shown == limit) {
            more = 1;
            break;
        }
        yyjson_mut_arr_append(page, yyjson_mut_val_mut_copy(d, v));
        if (!shown)
            snprintf(first, sizeof first, "%s", vid ? vid : "");
        snprintf(last, sizeof last, "%s", vid ? vid : "");
        shown++;
    }
    yyjson_mut_obj_add_str(d, root, "object", "list");
    yyjson_mut_obj_add_val(d, root, "data", page);
    if (shown) {
        yyjson_mut_obj_add_strcpy(d, root, "first_id", first);
        yyjson_mut_obj_add_strcpy(d, root, "last_id", last);
    } else {
        yyjson_mut_obj_add_null(d, root, "first_id");
        yyjson_mut_obj_add_null(d, root, "last_id");
    }
    yyjson_mut_obj_add_bool(d, root, "has_more", more);
    char *out = yyjson_mut_val_write(root, 0, NULL);
    yyjson_mut_doc_free(d);
    return reply_text(r, out);
}

int srv_eval_items_list(struct srv_req *r)
{
    return items(r, NULL);
}

int srv_eval_items_get(struct srv_req *r)
{
    return items(r, r->param[2]);
}