Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llama: update to b11521 and use native decision support

Jeffrey Morganadc4aae9d68c
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 156916bbf..f7ea8bf56 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -46,7 +46,6 @@ jobs:
'cmake/**/*' \
'llama/server/**/*' \
'llama/compat/**/*' \
- 'llama/clef/**/*' \
'LLAMA_CPP_VERSION' \
'MLX_VERSION' \
'MLX_C_VERSION' \
diff --git a/Dockerfile b/Dockerfile
index 118082ba8..e68861205 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -84,7 +84,6 @@ FROM cpu-deps AS llama-server-cpu
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset cpu \
@@ -105,7 +104,6 @@ FROM cuda-12-deps AS llama-server-cuda_v12
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset llama_cuda_v12_linux \
@@ -119,7 +117,6 @@ FROM cuda-13-deps AS llama-server-cuda_v13
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset llama_cuda_v13_linux \
@@ -134,7 +131,6 @@ ENV CC=clang CXX=clang++ CXXFLAGS=--gcc-toolchain=/opt/rh/gcc-toolset-13/root/us
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset rocm_v7_2_linux \
@@ -149,7 +145,6 @@ FROM vulkan-deps AS llama-server-vulkan
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset vulkan \
@@ -175,7 +170,6 @@ ENV CMAKE_GENERATOR=Ninja
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset llama_cuda_jetpack5 \
@@ -197,7 +191,6 @@ ENV CMAKE_GENERATOR=Ninja
COPY LLAMA_CPP_VERSION .
COPY llama/server llama/server
COPY llama/compat llama/compat
-COPY llama/clef llama/clef
COPY cmake cmake
RUN --mount=type=cache,target=/root/.ccache \
cmake -S llama/server --preset llama_cuda_jetpack6 \
diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSION
index a00ca832a..cf976c99f 100644
--- a/LLAMA_CPP_VERSION
+++ b/LLAMA_CPP_VERSION
@@ -1 +1 @@
-b11351
+b11521
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index b6bf8b777..6f2c2fe26 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -899,12 +899,12 @@ components:
model:
type: string
pattern: '\S'
- description: Local model trained for System One, such as nimble. Requires compatible GGUF weights and a scoring-capable runner; cloud and MLX/Safetensors models are not supported.
+ description: Local model trained for System One, such as nimble.
state:
$ref: "#/components/schemas/SystemOneContent"
images:
type: array
- description: Base64-encoded images shared by all questions, in request order. Requires Clef or Clef Flash with vision weights. URLs and data URLs are not supported.
+ description: Base64-encoded images shared by all questions, in request order. Requires a model with vision, such as Clef. URLs and data URLs are not supported.
items:
type: string
format: byte
diff --git a/llama/clef/clef.cpp b/llama/clef/clef.cpp
deleted file mode 100644
index ef3b251c9..000000000
--- a/llama/clef/clef.cpp
+++ /dev/null
@@ -1,388 +0,0 @@
-// Clef joint schema head, ported from Cloudflare's Apache-2.0 reference.
-// https://huggingface.co/Cloudflare/clef/blob/main/joint_schema_model.py
-#include "clef.h"
-#include "ggml-backend.h"
-#include "ggml.h"
-#include "gguf.h"
-#include <algorithm>
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <map>
-#include <stdexcept>
-
-#if !defined(_WIN32)
-#include <sys/types.h>
-#endif
-
-namespace {
-struct matrix {
- int rows = 0, cols = 0;
- std::vector<float> data;
- matrix() = default;
- matrix(int r, int c) : rows(r), cols(c), data(size_t(r) * c) {}
- float &at(int r, int c) { return data[size_t(r) * cols + c]; }
- float at(int r, int c) const { return data[size_t(r) * cols + c]; }
-};
-void require(bool condition, const char *message) {
- if (!condition)
- throw std::runtime_error(message);
-}
-// Use the backend registry so builds with dynamically selected CPU variants
-// work without linking to a particular ggml-cpu library.
-struct cpu_backend {
- ggml_backend_t handle = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
- cpu_backend() {
- require(handle != nullptr, "Clef: CPU backend unavailable");
- auto reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(handle));
- auto set_threads = reinterpret_cast<ggml_backend_set_n_threads_t>(
- ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));
- if (set_threads)
- set_threads(handle, 8);
- }
- ~cpu_backend() { ggml_backend_free(handle); }
-};
-struct graph {
- ggml_context *ctx;
- explicit graph(size_t bytes) {
- ctx = ggml_init({bytes + (2 << 20), nullptr, true});
- require(ctx != nullptr, "Clef: cannot allocate graph");
- }
- ~graph() { ggml_free(ctx); }
- ggml_tensor *input(const matrix &m) {
- auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, m.cols, m.rows);
- t->data = const_cast<float *>(m.data.data());
- return t;
- }
- matrix run(ggml_tensor *output, int rows, int cols) {
- auto gf = ggml_new_graph(ctx);
- ggml_build_forward_expand(gf, output);
- static thread_local cpu_backend cpu;
- require(ggml_backend_graph_compute(cpu.handle, gf) == GGML_STATUS_SUCCESS,
- "Clef: graph evaluation failed");
- matrix result(rows, cols);
- memcpy(result.data.data(), output->data, result.data.size() * sizeof(float));
- return result;
- }
-};
-matrix linear(const matrix &x, const matrix &w) {
- require(x.cols == w.cols, "Clef: linear shape mismatch");
- graph g(size_t(x.rows) * w.rows * sizeof(float));
- auto tx = g.input(x), tw = g.input(w);
- ggml_set_no_alloc(g.ctx, false);
- return g.run(ggml_mul_mat(g.ctx, tw, tx), x.rows, w.rows);
-}
-matrix add(matrix a, const matrix &b) {
- require(a.cols == b.cols && (b.rows == 1 || a.rows == b.rows), "Clef: addition shape mismatch");
- for (int r = 0; r < a.rows; ++r)
- for (int c = 0; c < a.cols; ++c)
- a.at(r, c) += b.at(b.rows == 1 ? 0 : r, c);
- return a;
-}
-matrix slice(const matrix &m, int start, int end) {
- require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span");
- matrix out(end - start, m.cols);
- std::copy(m.data.begin() + size_t(start) * m.cols, m.data.begin() + size_t(end) * m.cols,
- out.data.begin());
- return out;
-}
-matrix mean(const matrix &m, int start, int end) {
- require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span");
- matrix out(1, m.cols);
- for (int r = start; r < end; ++r)
- for (int c = 0; c < m.cols; ++c)
- out.at(0, c) += m.at(r, c) / (end - start);
- return out;
-}
-void append(matrix &a, const matrix &b) {
- require(a.rows == 0 || a.cols == b.cols, "Clef: append shape mismatch");
- a.cols = b.cols;
- a.rows += b.rows;
- a.data.insert(a.data.end(), b.data.begin(), b.data.end());
-}
-float cosine(const matrix &a, int ar, const matrix &b, int br, double eps = 1e-8) {
- require(a.cols == b.cols, "Clef: cosine shape mismatch");
- double dot = 0, aa = 0, bb = 0;
- for (int c = 0; c < a.cols; ++c) {
- dot += double(a.at(ar, c)) * b.at(br, c);
- aa += double(a.at(ar, c)) * a.at(ar, c);
- bb += double(b.at(br, c)) * b.at(br, c);
- }
- return dot / (std::max(std::sqrt(aa), eps) * std::max(std::sqrt(bb), eps));
-}
-} // namespace
-
-struct clef_head::impl {
- FILE *file = nullptr;
- gguf_context *meta = nullptr;
- ggml_context *tensors = nullptr;
- std::map<std::string, matrix> weights;
- int hidden, width, heads, routing_layers, layers;
- int output_index;
- ggml_tensor *output;
-
- explicit impl(const std::string &path) : file(ggml_fopen(path.c_str(), "rb")) {
- try {
- meta = gguf_init_from_file(path.c_str(), {true, &tensors});
- require(meta && tensors && file, "Clef: cannot open model");
- const int arch_key = gguf_find_key(meta, "general.architecture");
- require(arch_key >= 0 && gguf_get_kv_type(meta, arch_key) == GGUF_TYPE_STRING,
- "Clef: missing architecture");
- const std::string prefix = std::string(gguf_get_val_str(meta, arch_key)) + ".decision.";
- auto integer = [&](const std::string &key) {
- int i = gguf_find_key(meta, (prefix + key).c_str());
- require(i >= 0 && gguf_get_kv_type(meta, i) == GGUF_TYPE_UINT32,
- "Clef: missing head configuration");
- return int(gguf_get_val_u32(meta, i));
- };
- hidden = integer("hidden_size");
- width = integer("width");
- heads = integer("heads");
- routing_layers = integer("routing_layers");
- layers = integer("layers");
- require(hidden > 0 && hidden <= 16384 && width > 0 && width <= 4096 && heads > 0 &&
- width % heads == 0 && routing_layers >= 0 && routing_layers <= 32 && layers >= 0 &&
- layers <= 32,
- "Clef: invalid head configuration");
- for (int i = 0; i < gguf_get_n_tensors(meta); ++i) {
- std::string name = gguf_get_tensor_name(meta, i);
- if (name.rfind("clef.", 0) != 0)
- continue;
- auto t = ggml_get_tensor(tensors, name.c_str());
- require(t->ne[2] == 1 && t->ne[3] == 1 && t->type == GGML_TYPE_F32,
- "Clef: head tensors must be float32 matrices");
- matrix m(t->ne[1], t->ne[0]);
- read(i, 0, m.data.data(), m.data.size() * sizeof(float));
- weights.emplace(name.substr(5), std::move(m));
- }
- output_index = gguf_find_tensor(meta, "output.weight");
- require(output_index >= 0, "Clef: output embeddings are missing");
- output = ggml_get_tensor(tensors, "output.weight");
- require(output->ne[0] == hidden, "Clef: output embedding shape mismatch");
- } catch (...) {
- if (tensors)
- ggml_free(tensors);
- if (meta)
- gguf_free(meta);
- if (file)
- std::fclose(file);
- throw;
- }
- }
- ~impl() {
- ggml_free(tensors);
- gguf_free(meta);
- std::fclose(file);
- }
- void read(int index, size_t offset, void *data, size_t bytes) {
- // The head can sit past 2 GiB, where iostream seeks truncate on some
- // Windows C++ runtimes, so seek with an explicit 64-bit offset.
- const size_t position = gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, index) + offset;
-#if defined(_WIN32)
- const int seek_rc = _fseeki64(file, static_cast<__int64>(position), SEEK_SET);
-#else
- const int seek_rc = fseeko(file, static_cast<off_t>(position), SEEK_SET);
-#endif
- require(seek_rc == 0 && std::fread(data, 1, bytes, file) == bytes, "Clef: truncated tensor data");
- }
- const matrix &w(const std::string &key) { return weights.at(key); }
- matrix norm(const matrix &x, const std::string &key) {
- const auto &weight = w(key + ".weight"), &bias = w(key + ".bias");
- require(weight.data.size() == size_t(x.cols) && bias.data.size() == size_t(x.cols),
- "Clef: layer norm shape mismatch");
- matrix out = x;
- for (int r = 0; r < x.rows; ++r) {
- double m = 0, v = 0;
- for (int c = 0; c < x.cols; ++c)
- m += x.at(r, c) / x.cols;
- for (int c = 0; c < x.cols; ++c)
- v += (x.at(r, c) - m) * (x.at(r, c) - m) / x.cols;
- for (int c = 0; c < x.cols; ++c)
- out.at(r, c) = (x.at(r, c) - m) / std::sqrt(v + 1e-5) * weight.data[c] + bias.data[c];
- }
- return out;
- }
- matrix project(const matrix &x, const std::string &key, bool bias = false) {
- matrix out = linear(x, w(key + ".weight"));
- return bias ? add(std::move(out), w(key + ".bias")) : out;
- }
- matrix feedforward(matrix x, const std::string &first, const std::string &last) {
- x = project(x, first, true);
- for (auto &v : x.data)
- v = 0.5f * v * (1.0f + std::erf(v / std::sqrt(2.0f)));
- return project(x, last, true);
- }
- matrix attention(const matrix &query, const matrix &memory, const std::string &key) {
- const auto &weight = w(key + ".in_proj_weight"), &bias = w(key + ".in_proj_bias");
- require(weight.rows == 3 * width && weight.cols == width && bias.data.size() == size_t(3 * width),
- "Clef: attention shape mismatch");
- auto projection = [&](const matrix &x, int n) {
- auto y = linear(x, slice(weight, n * width, (n + 1) * width));
- for (int r = 0; r < y.rows; ++r)
- for (int c = 0; c < width; ++c)
- y.at(r, c) += bias.data[n * width + c];
- return y;
- };
- auto q = projection(query, 0), k = projection(memory, 1), v = projection(memory, 2);
- const int d = width / heads, nq = q.rows, nk = k.rows;
- graph g((size_t(nq) * nk * heads * 3 + size_t(nq + nk) * width * 8) * sizeof(float));
- auto tq = g.input(q), tk = g.input(k), tv = g.input(v);
- ggml_set_no_alloc(g.ctx, false);
- auto split = [&](ggml_tensor *t, int n) {
- return ggml_cont(g.ctx, ggml_permute(g.ctx, ggml_reshape_3d(g.ctx, t, d, heads, n), 0, 2, 1, 3));
- };
- tq = split(tq, nq);
- tk = split(tk, nk);
- tv = split(tv, nk);
- auto scores =
- ggml_soft_max(g.ctx, ggml_scale(g.ctx, ggml_mul_mat(g.ctx, tk, tq), 1.0f / std::sqrt(float(d))));
- tv = ggml_cont(g.ctx, ggml_transpose(g.ctx, tv));
- auto out = ggml_mul_mat(g.ctx, tv, scores);
- out = ggml_cont(g.ctx, ggml_permute(g.ctx, out, 0, 2, 1, 3));
- auto result = g.run(ggml_reshape_2d(g.ctx, out, width, nq), nq, width);
- return project(result, key + ".out_proj", true);
- }
- matrix lexical(const std::vector<int32_t> &tokens, int start, int end) {
- require(start >= 0 && start < end && end <= int(tokens.size()), "Clef: invalid lexical span");
- matrix result(1, hidden);
- const auto traits = ggml_get_type_traits(output->type);
- require(traits->to_float != nullptr || output->type == GGML_TYPE_F32,
- "Clef: unsupported output embedding type");
- size_t bytes = ggml_row_size(output->type, hidden);
- std::vector<uint8_t> raw(bytes);
- std::vector<float> row(hidden);
- for (int i = start; i < end; ++i) {
- require(tokens[i] >= 0 && tokens[i] < output->ne[1], "Clef: invalid token");
- read(output_index, size_t(tokens[i]) * bytes, raw.data(), bytes);
- if (output->type == GGML_TYPE_F32)
- memcpy(row.data(), raw.data(), bytes);
- else
- traits->to_float(raw.data(), row.data(), hidden);
- for (int c = 0; c < hidden; ++c)
- result.at(0, c) += row[c] / (end - start);
- }
- return result;
- }
- std::vector<std::vector<float>> score(const std::vector<std::vector<float>> &states,
- const std::vector<int32_t> &tokens, const common_json &fields) {
- require(states.size() == tokens.size() && !states.empty(), "Clef: missing backbone hidden states");
- require(fields.is_array() && !fields.empty() && fields.size() <= 64, "Clef: expected 1-64 fields");
- matrix x;
- for (const auto &row : states) {
- require(row.size() == size_t(hidden), "Clef: hidden size mismatch");
- matrix m(1, hidden);
- m.data = row;
- append(x, m);
- }
- x = norm(x, "hidden_norm");
- auto memory = project(x, "memory_projection"), global = slice(x, x.rows - 1, x.rows);
- matrix questions, lexical_options, queries;
- std::vector<int> counts, types;
- auto span = [](const common_json &value) {
- require(value.is_array() && value.size() == 2 && value[0].is_number_integer() &&
- value[1].is_number_integer(),
- "Clef: malformed span");
- return std::pair<int, int>{value[0].get<int>(), value[1].get<int>()};
- };
- for (const auto &field : fields) {
- int type = field.at("type").get<int>();
- require(type >= 0 && type < 3, "Clef: invalid question type");
- types.push_back(type);
- auto [qs, qe] = span(field.at("question"));
- auto question = mean(x, qs, qe);
- append(questions, question);
- const auto &options = field.at("options");
- require(options.is_array() && options.size() >= 2 && options.size() <= 26,
- "Clef: expected 2-26 options");
- counts.push_back(options.size());
- matrix contexts, lex;
- for (const auto &option : options) {
- auto [s, e] = span(option);
- append(contexts, mean(x, s, e));
- append(lex, lexical(tokens, s, e));
- }
- append(lexical_options, lex);
- append(queries, add(add(project(contexts, "option_context_projection"),
- project(lex, "option_lexical_projection")),
- project(question, "option_question_projection")));
- }
- for (int i = 0; i < routing_layers; ++i) {
- auto key = "evidence_layers." + std::to_string(i);
- queries = add(queries, attention(norm(queries, key + ".query_norm"),
- norm(memory, key + ".memory_norm"), key + ".attention"));
- queries = add(queries, feedforward(norm(queries, key + ".feedforward_norm"),
- key + ".feedforward.0", key + ".feedforward.3"));
- }
- auto base = project(questions, "question_projection");
- matrix summaries;
- int offset = 0;
- for (int f = 0; f < base.rows; ++f) {
- std::vector<float> scores(counts[f]);
- for (int j = 0; j < counts[f]; ++j)
- for (int c = 0; c < width; ++c)
- scores[j] += queries.at(offset + j, c) * base.at(f, c) / std::sqrt(float(width));
- float peak = *std::max_element(scores.begin(), scores.end()), sum = 0;
- for (auto &v : scores) {
- v = std::exp(v - peak);
- sum += v;
- }
- matrix summary(1, width);
- for (int j = 0; j < counts[f]; ++j)
- for (int c = 0; c < width; ++c)
- summary.at(0, c) += scores[j] / sum * queries.at(offset + j, c);
- append(summaries, summary);
- offset += counts[f];
- }
- auto fs =
- add(add(base, norm(summaries, "option_summary_norm")), project(global, "global_projection"));
- const auto &te = w("type_embedding.weight");
- require(te.rows == 3 && te.cols == width, "Clef: type embedding shape mismatch");
- for (int f = 0; f < fs.rows; ++f)
- for (int c = 0; c < width; ++c)
- fs.at(f, c) += te.at(types[f], c);
- for (int i = 0; i < layers; ++i) {
- auto key = "layers." + std::to_string(i);
- auto normalized = norm(fs, key + ".norm1");
- fs = add(fs, attention(normalized, normalized, key + ".self_attn"));
- fs = add(fs, attention(norm(fs, key + ".norm2"), memory, key + ".multihead_attn"));
- fs = add(fs, feedforward(norm(fs, key + ".norm3"), key + ".linear1", key + ".linear2"));
- }
- fs = norm(fs, "field_norm");
- auto options = norm(queries, "option_norm");
- float prior_scale = std::exp(std::min(w("prior_logit_scale").data.at(0), std::log(100.0f)));
- float joint_scale = std::exp(std::min(w("joint_logit_scale").data.at(0), std::log(100.0f)));
- float gate = 1.0f / (1.0f + std::exp(-w("residual_gate").data.at(0)));
- std::vector<std::vector<float>> result;
- offset = 0;
- for (int f = 0; f < fs.rows; ++f) {
- matrix features(counts[f], 4 * width);
- for (int j = 0; j < counts[f]; ++j)
- for (int c = 0; c < width; ++c) {
- float a = fs.at(f, c), b = options.at(offset + j, c);
- features.at(j, c) = a;
- features.at(j, width + c) = b;
- features.at(j, 2 * width + c) = a * b;
- features.at(j, 3 * width + c) = std::abs(a - b);
- }
- auto residual = feedforward(features, "residual_scorer.0", "residual_scorer.3");
- auto anchor = add(slice(questions, f, f + 1), global);
- std::vector<float> logits;
- for (int j = 0; j < counts[f]; ++j) {
- float v = prior_scale * cosine(lexical_options, offset + j, anchor, 0, 1e-12) +
- gate * (joint_scale * cosine(fs, f, options, offset + j) + residual.at(j, 0));
- require(std::isfinite(v), "Clef: non-finite logit");
- logits.push_back(v);
- }
- result.push_back(std::move(logits));
- offset += counts[f];
- }
- return result;
- }
-};
-clef_head::clef_head(const std::string &path) : p(new impl(path)) {}
-clef_head::~clef_head() = default;
-std::vector<std::vector<float>> clef_head::score(const std::vector<std::vector<float>> &hidden,
- const std::vector<int32_t> &tokens,
- const common_json &fields) {
- return p->score(hidden, tokens, fields);
-}
diff --git a/llama/clef/clef.h b/llama/clef/clef.h
deleted file mode 100644
index ea2607d32..000000000
--- a/llama/clef/clef.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-#include "json.h"
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-// The backbone is evaluated by llama-server. This head consumes every final
-// hidden state and the token spans produced by Cloudflare's record format.
-class clef_head {
- public:
- explicit clef_head(const std::string &path);
- ~clef_head();
- std::vector<std::vector<float>> score(const std::vector<std::vector<float>> &hidden,
- const std::vector<int32_t> &tokens, const common_json &fields);
-
- private:
- struct impl;
- std::unique_ptr<impl> p;
-};
diff --git a/llama/clef/convert.py b/llama/clef/convert.py
deleted file mode 100644
index ac18c32b2..000000000
--- a/llama/clef/convert.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-"""Convert Cloudflare Clef with its trained joint head, using pinned llama.cpp.
-
-Usage: python convert.py --llama-cpp build/llama-src MODEL --outtype q8_0 --outfile MODEL.gguf
-For the vision projector, add --mmproj --outtype f16 and use a separate output file.
-All remaining arguments are passed to convert_hf_to_gguf.py.
-"""
-import argparse
-import json
-from pathlib import Path
-import sys
-
-parser = argparse.ArgumentParser(add_help=False)
-parser.add_argument('--llama-cpp', type=Path, required=True)
-args, remaining = parser.parse_known_args()
-sys.path[:0] = [str(args.llama_cpp.resolve()), str((args.llama_cpp / 'gguf-py').resolve())]
-
-import numpy as np
-from safetensors import safe_open
-from conversion.qwen import Qwen3_5TextModel
-import convert_hf_to_gguf
-
-original_tensors = Qwen3_5TextModel.prepare_tensors
-original_metadata = Qwen3_5TextModel.set_gguf_parameters
-
-
-def prepare_tensors(self):
- original_tensors(self)
- with safe_open(self.dir_model / 'joint_head.safetensors', framework='pt', device='cpu') as head:
- for name in head.keys():
- # Keep the small head in float32, including its scalar gates.
- data = head.get_tensor(name).float().numpy()
- self.gguf_writer.add_tensor('clef.' + name, np.atleast_1d(data))
-
-
-def set_gguf_parameters(self):
- original_metadata(self)
- config = json.loads((self.dir_model / 'joint_head_config.json').read_text())
- self.gguf_writer.add_string(f'{self.gguf_writer.arch}.decision.type', 'clef')
- for name, value in config.items():
- self.gguf_writer.add_uint32(f'{self.gguf_writer.arch}.decision.{name}', value)
-
-
-Qwen3_5TextModel.prepare_tensors = prepare_tensors
-Qwen3_5TextModel.set_gguf_parameters = set_gguf_parameters
-sys.argv = [sys.argv[0], '--no-mtp', *remaining]
-convert_hf_to_gguf.main()
diff --git a/llama/compat/002-clef.patch b/llama/compat/002-clef.patch
deleted file mode 100644
index 9614e0fb5..000000000
--- a/llama/compat/002-clef.patch
+++ /dev/null
@@ -1,321 +0,0 @@
-diff --git a/common/common.cpp b/common/common.cpp
-index 7ebfa2610..695a1bfff 100644
---- a/common/common.cpp
-+++ b/common/common.cpp
-@@ -1192,6 +1192,18 @@ struct common_init_result::impl {
- std::vector<llama_sampler_seq_config> samplers_seq_config;
- };
-
-+common_decision_type common_get_decision_type(const llama_model * model) {
-+ char value[128];
-+ if (llama_model_meta_val_str(model, "general.architecture", value, sizeof(value)) < 0) {
-+ return COMMON_DECISION_TYPE_NONE;
-+ }
-+ const std::string key = std::string(value) + ".decision.type";
-+ if (llama_model_meta_val_str(model, key.c_str(), value, sizeof(value)) < 0) {
-+ return COMMON_DECISION_TYPE_NONE;
-+ }
-+ return std::strcmp(value, "clef") == 0 ? COMMON_DECISION_TYPE_CLEF : COMMON_DECISION_TYPE_UNKNOWN;
-+}
-+
- common_init_result::common_init_result(common_params & params, bool model_only) :
- pimpl(new impl{}) {
- auto mparams = common_model_params_to_llama(params);
-@@ -1244,6 +1256,18 @@ common_init_result::common_init_result(common_params & params, bool model_only)
-
- const llama_vocab * vocab = llama_model_get_vocab(model);
-
-+ if (common_get_decision_type(model) == COMMON_DECISION_TYPE_CLEF) {
-+ params.embedding = cparams.embeddings = true;
-+ cparams.ctx_type = LLAMA_CONTEXT_TYPE_DECISION;
-+ params.pooling_type = cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
-+ // The joint head consumes every hidden state; retain the whole input.
-+ const auto n_ctx = cparams.n_ctx ? cparams.n_ctx : llama_model_n_ctx_train(model);
-+ params.n_batch = cparams.n_batch = n_ctx;
-+ params.n_ubatch = cparams.n_ubatch = n_ctx;
-+ params.n_outputs_max = cparams.n_outputs_max = n_ctx;
-+ params.n_outputs_max_per_seq = cparams.n_outputs_max_per_seq = 1;
-+ }
-+
- // load and optionally apply lora adapters
- for (auto & la : params.lora_adapters) {
- llama_adapter_lora_ptr lora;
-diff --git a/common/common.h b/common/common.h
-index fd90c3b95..641f2dc6f 100644
---- a/common/common.h
-+++ b/common/common.h
-@@ -953,6 +953,14 @@ bool tty_can_use_colors();
-
- struct common_sampler;
-
-+enum common_decision_type {
-+ COMMON_DECISION_TYPE_NONE,
-+ COMMON_DECISION_TYPE_CLEF,
-+ COMMON_DECISION_TYPE_UNKNOWN,
-+};
-+
-+common_decision_type common_get_decision_type(const struct llama_model * model);
-+
- // note: defines the model, context, samplers, ets. lifetimes
- struct common_init_result {
- common_init_result(common_params & params, bool model_only = false);
-diff --git a/include/llama.h b/include/llama.h
-index dea4d4606..720b45140 100644
---- a/include/llama.h
-+++ b/include/llama.h
-@@ -224,6 +224,7 @@ extern "C" {
- enum llama_context_type {
- LLAMA_CONTEXT_TYPE_DEFAULT = 0,
- LLAMA_CONTEXT_TYPE_MTP = 1,
-+ LLAMA_CONTEXT_TYPE_DECISION = 2, // hidden states without vocabulary logits
- };
-
- // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)
-diff --git a/src/llama-context.cpp b/src/llama-context.cpp
-index 4f04db0ee..fa115ffe4 100644
---- a/src/llama-context.cpp
-+++ b/src/llama-context.cpp
-@@ -27,6 +27,7 @@
-
- static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
- switch (ctx_type) {
-+ case LLAMA_CONTEXT_TYPE_DECISION:
- case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;
- case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP;
- }
-@@ -2124,7 +2125,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {
- const auto n_embd = hparams.n_embd;
- const auto n_embd_out = hparams.n_embd_out();
-
-- bool has_logits = true;
-+ bool has_logits = cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION;
- bool has_embd = cparams.embeddings;
- bool has_embd_nextn = cparams.embeddings_nextn;
-
-diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
-index a1e263500..751c8a72e 100644
---- a/src/models/qwen35.cpp
-+++ b/src/models/qwen35.cpp
-@@ -215,11 +215,12 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para
- cb(cur, "result_norm", -1);
- res->t_embd = cur;
-
-- // LM head
-- cur = build_lora_mm(model.output, cur, model.output_s);
--
-- cb(cur, "result_output", -1);
-- res->t_logits = cur;
-+ // Decision heads consume the hidden states, not vocabulary logits.
-+ if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION) {
-+ cur = build_lora_mm(model.output, cur, model.output_s);
-+ cb(cur, "result_output", -1);
-+ res->t_logits = cur;
-+ }
-
- ggml_build_forward_expand(gf, cur);
- }
-diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
-index 470fbd977..e2ac37eb7 100644
---- a/tools/server/server-context.cpp
-+++ b/tools/server/server-context.cpp
-@@ -1,4 +1,5 @@
- #include "server-context.h"
-+#include "clef.h"
- #include "server-chat.h"
- #include "server-common.h"
- #include "server-http.h"
-@@ -206,6 +207,7 @@ struct server_slot {
- // multimodal
- mtmd_context * mctx = nullptr;
- mtmd::batch_ptr mbatch = nullptr;
-+ std::vector<std::vector<float>> score_embeddings;
-
- // speculative decoding
- common_speculative * spec;
-@@ -329,6 +331,7 @@ struct server_slot {
- SLT_DBG(*this, "%s", "\n");
-
- spec_is_replay = false;
-+ score_embeddings.clear();
-
- last_nl_pos = 0;
- generated_text = "";
-@@ -389,6 +392,18 @@ struct server_slot {
- (ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size());
- }
-
-+ void collect_score_embeddings(const common_batch & batch) {
-+ if (!task || task->score_fields.is_null()) return;
-+ // Embedding mode returns every token, including image tokens with logits disabled.
-+ const int width = llama_model_n_embd_out(llama_get_model(ctx_tgt));
-+ for (int32_t i = 0; i < batch.size(); ++i) {
-+ if (batch.tokens[i].seq_id != id) continue;
-+ const float * row = llama_get_embeddings_ith(ctx_tgt, i);
-+ if (!row) throw std::runtime_error("decision model did not return hidden states");
-+ score_embeddings.emplace_back(row, row + width);
-+ }
-+ }
-+
- bool need_embd() const {
- GGML_ASSERT(task);
- return task->need_embd();
-@@ -707,9 +722,7 @@ struct server_slot {
- // returns 0 on success
- // caller need to update prompt.tokens after a successful call to keep track of the processing progress
- // note: this is not a member of server_slot because we want to run it inside yield_to_queue
--// slot is passed as const to avoid accidental modification of the slot state
--// some pointers are allowed to be used, they are not used by to_json()
--static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) {
-+static int process_mtmd_chunk(server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) {
- GGML_ASSERT(slot.mctx);
- const auto & mctx = slot.mctx;
- const auto & input_tokens = slot.task->tokens;
-@@ -721,9 +734,9 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch
- float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get());
- if (embd) {
- struct cb_data_t {
-- common_speculative * spec;
-+ server_slot * slot;
- llama_context * ctx;
-- } cb_data = { slot.spec, slot.ctx_tgt };
-+ } cb_data = { &slot, slot.ctx_tgt };
-
- static auto cb = [](const mtmd_helper_embd_batch * b, void * user_data) {
- const auto * data = static_cast<cb_data_t *>(user_data);
-@@ -737,7 +750,9 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch
- batch.add_embd({ b->embd + (size_t) i * b->n_embd, 1, (size_t) b->n_embd }, pos, b->seq_id, false);
- }
-
-- return common_speculative_process(data->spec, batch) ? 0 : 1;
-+ if (!common_speculative_process(data->slot->spec, batch)) return 1;
-+ data->slot->collect_score_embeddings(batch);
-+ return 0;
- };
-
- llama_pos new_n_past; // unused for now
-@@ -819,6 +834,7 @@ public:
- // - when not in sleeping state
- // - and, with thread-safe APIs (e.g., tokenizer calls)
- llama_model * model_tgt = nullptr;
-+ std::unique_ptr<clef_head> clef;
-
- mtmd_context * mctx = nullptr;
- // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct
-@@ -2156,6 +2172,20 @@ private:
- res->n_tokens = slot.task->n_tokens();
- res->res_type = slot.task->params.res_type;
-
-+ if (!slot.task->score_fields.is_null()) {
-+ try {
-+ if (!clef) clef.reset(new clef_head(params_base.model.path));
-+ llama_tokens tokens;
-+ for (size_t i = 0; i < slot.task->tokens.size(); ++i) tokens.push_back(slot.task->tokens[i]);
-+ res->score_logits = clef->score(slot.score_embeddings, tokens, slot.task->score_fields);
-+ } catch (const std::exception & e) {
-+ send_error(slot, e.what(), ERROR_TYPE_SERVER);
-+ return;
-+ }
-+ queue_results.send(std::move(res));
-+ return;
-+ }
-+
- const int n_embd_out = llama_model_n_embd_out(model_tgt);
-
- std::vector<float> embd_res(n_embd_out, 0.0f);
-@@ -3803,6 +3833,7 @@ private:
- };
-
- iterate(slots, [&](server_slot & slot) {
-+ slot.collect_score_embeddings(batch.view);
- // optionally send prompt processing progress
- if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) {
- if (slot.task->params.stream && slot.task->params.return_progress) {
-@@ -4263,6 +4294,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
- GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL);
-
- auto res = create_response();
-+ if (common_get_decision_type(ctx_server.model_tgt) == COMMON_DECISION_TYPE_CLEF) {
-+ res->error(format_error_response("This model requires a decision request", ERROR_TYPE_INVALID_REQUEST));
-+ return res;
-+ }
- auto completion_id = gen_chatcmplid();
- auto & rd = res->rd;
- auto & params = this->params;
-@@ -5475,6 +5510,43 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
- // OAI-compat
- task.params.res_type = res_type;
- task.params.embd_normalize = embd_normalize;
-+ if (body.contains("score_fields")) {
-+ task.score_fields = body.at("score_fields");
-+ task.params.cache_prompt = false; // the head needs every hidden state
-+ if (body.contains("images") && !body.at("images").empty()) {
-+ if (!ctx_server.mctx) throw std::invalid_argument("model does not support images");
-+ const auto & text = task.tokens.get_tokens();
-+ const int position = body.at("image_position").get<int>();
-+ if (position < 0 || size_t(position) > text.size()) throw std::invalid_argument("invalid image position");
-+ std::string markers;
-+ for (size_t j = 0; j < body.at("images").size(); ++j) markers += mtmd_get_marker(ctx_server.mctx);
-+ auto media_inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx,
-+ json{{"prompt_string", markers + "\n"}, {"multimodal_data", body.at("images")}}, true, true, ctx_server.init_opt);
-+ auto & media = media_inputs[0];
-+ if (text.size() + media.size() > size_t(meta->slot_n_ctx)) throw std::invalid_argument("decision prompt exceeds the model context (input is never truncated)");
-+ server_tokens input(llama_tokens(text.begin(), text.begin() + position), true);
-+ for (size_t j = 0; j < media.size();) {
-+ if (media[j] == LLAMA_TOKEN_NULL) {
-+ const auto & chunk = media.find_chunk(j);
-+ input.push_back(chunk.get());
-+ j += mtmd_input_chunk_get_n_tokens(chunk.get());
-+ } else {
-+ input.push_back(media[j++]);
-+ }
-+ }
-+ input.insert(llama_tokens(text.begin() + position, text.end()));
-+ for (auto & field : task.score_fields) {
-+ auto shift = [&](json & span) {
-+ const int start = span.at(0).get<int>(), end = span.at(1).get<int>();
-+ if (start < position && end > position) throw std::invalid_argument("image splits a scoring span");
-+ if (start >= position) span = json::array({start + media.size(), end + media.size()});
-+ };
-+ shift(field.at("question"));
-+ for (auto & option : field.at("options")) shift(option);
-+ }
-+ task.tokens = std::move(input);
-+ }
-+ }
-
- tasks.push_back(std::move(task));
- }
-diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
-index 0d3beb313..f5f3f2e18 100644
---- a/tools/server/server-task.cpp
-+++ b/tools/server/server-task.cpp
-@@ -1464,6 +1464,9 @@ json server_task_result_cmpl_partial::to_json_anthropic() {
- // server_task_result_embd
- //
- json server_task_result_embd::to_json() {
-+ if (!score_logits.is_null()) {
-+ return json {{"logits", score_logits}, {"tokens_evaluated", n_tokens}};
-+ }
- return res_type == TASK_RESPONSE_TYPE_OAI_EMBD
- ? to_json_oaicompat()
- : to_json_non_oaicompat();
-diff --git a/tools/server/server-task.h b/tools/server/server-task.h
-index 9c99143f8..94c724ed1 100644
---- a/tools/server/server-task.h
-+++ b/tools/server/server-task.h
-@@ -152,6 +152,7 @@ struct server_task {
- // used by SERVER_TASK_TYPE_INFERENCE
- task_params params;
- server_tokens tokens;
-+ common_json score_fields; // optional decision question and option spans
-
- // only used by CLI, this allow tokenizing CLI inputs on server side
- // we need this because mtmd_context and vocab are not accessible outside of server_context
-@@ -452,6 +453,7 @@ struct server_task_result_cmpl_partial : server_task_result {
- };
-
- struct server_task_result_embd : server_task_result {
-+ common_json score_logits;
- std::vector<std::vector<float>> embedding;
-
- int32_t n_tokens;
diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp
index 848b84fa0..3e1cbddd4 100644
--- a/llama/compat/llama-ollama-compat.cpp
+++ b/llama/compat/llama-ollama-compat.cpp
@@ -3242,12 +3242,6 @@ bool translate_metadata(const llama_model_loader * ml,
std::lock_guard<std::mutex> lk(g_loader_path_mutex);
g_loader_paths[ml] = fname ? fname : "";
}
- // Clef's joint head is consumed by the server after backbone evaluation.
- const int decision_key = gguf_find_key(meta, (arch_name + ".decision.type").c_str());
- if (decision_key >= 0 && gguf_get_kv_type(meta, decision_key) == GGUF_TYPE_STRING &&
- std::strcmp(gguf_get_val_str(meta, decision_key), "clef") == 0) {
- add_skip_prefix(ml, "clef.");
- }
// embeddinggemma must run before gemma3: it switches arch_name to
// "gemma-embedding", which is what later checks (and the loader's KV
// prefix) need to see.
diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt
index aabc1a28d..9153c9bf0 100644
--- a/llama/server/CMakeLists.txt
+++ b/llama/server/CMakeLists.txt
@@ -186,8 +186,6 @@ FetchContent_Declare(
${_ollama_compat_patch_cmd}
)
FetchContent_MakeAvailable(llama_cpp)
-target_sources(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef/clef.cpp)
-target_include_directories(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef)
if(NOT COMMAND license_add_file)
message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update")
diff --git a/llm/llama_server.go b/llm/llama_server.go
index c7cd91518..746d389f7 100644
--- a/llm/llama_server.go
+++ b/llm/llama_server.go
@@ -105,6 +105,38 @@ func WithDefaultEmbeddingNumBatch(opts api.Options) api.Options {
return opts
}
+// LlamaServerBatch resolves the batch size and extra memory allowance used
+// when deciding whether to load a model alongside other models.
+func LlamaServerBatch(f *gguf.Model, opts api.Options, completion bool) (int, uint64) {
+ batch := opts.NumBatch
+ // These decision heads read the whole prompt without a KV cache, so the
+ // physical batch must hold the entire effective context.
+ wholePrompt := f.KV().Uint("decision.block_count") > 0
+ if wholePrompt {
+ batch = opts.NumCtx
+ if trainCtx := int(f.KV().ContextLength()); trainCtx > 0 {
+ batch = min(batch, trainCtx)
+ }
+ }
+ if !completion && !wholePrompt {
+ return batch, 0
+ }
+ return batch, LlamaServerBatchMemory(batch)
+}
+
+// LlamaServerBatchMemory is the conservative extra memory allowance for a
+// generation or whole-prompt decision batch above the default batch size.
+func LlamaServerBatchMemory(batch int) uint64 {
+ switch {
+ case batch >= 2048:
+ return 2 << 30
+ case batch >= 1024:
+ return 768 << 20
+ default:
+ return 0
+ }
+}
+
func boundedNumPredict(numPredict, numCtx int) int {
if numCtx <= 0 {
return numPredict
@@ -183,6 +215,7 @@ type llamaServerLaunchConfig struct {
numParallel int
kvCacheType string
embedding bool
+ decision bool
config LlamaServerConfig
gpus []ml.DeviceInfo
gpuLibs []string
@@ -285,7 +318,7 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req
return prompt, nil
}
- tokens, err := s.tokenize(ctx, prompt, true, nil)
+ tokens, err := s.tokenize(ctx, prompt, true)
if err != nil {
return nil, err
}
@@ -296,10 +329,7 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req
}
if !s.launch.config.ContextShift {
- return nil, api.StatusError{
- StatusCode: http.StatusBadRequest,
- ErrorMessage: "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length",
- }
+ return nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: promptTooLongMessage}
}
nKeep := req.Options.NumKeep
@@ -376,7 +406,14 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec.
"--no-webui",
"--offline",
"-c", strconv.Itoa(launch.opts.NumCtx * launch.numParallel),
- "-np", strconv.Itoa(launch.numParallel),
+ }
+ // The questions of a decision request share their prompt only while they
+ // run in parallel slots. Their prompts rarely repeat across requests, and
+ // at trace verbosity llama-server logs its whole prompt cache after each one.
+ if launch.decision {
+ params = append(params, "-np", "16", "--kv-unified", "--cache-ram", "0")
+ } else {
+ params = append(params, "-np", strconv.Itoa(launch.numParallel))
}
params = appendLlamaServerLogArgs(params)
params = appendJinjaArgs(params, launch.config)
@@ -863,6 +900,7 @@ func NewLlamaServerRunner(
// Check if this is an embedding model
arch := f.KV().Architecture()
isEmbedding := f.KV().Has("pooling_type")
+ decisionType := f.KV().String("decision.type")
// Older Ollama-format GGUFs store vision tensors (v.*, mm.*) inline in
// the main model file rather than in a separate projector layer. When
@@ -943,6 +981,7 @@ func NewLlamaServerRunner(
numParallel: numParallel,
kvCacheType: kvCacheType,
embedding: isEmbedding,
+ decision: decisionType != "",
config: config,
gpus: slices.Clone(gpus),
gpuLibs: slices.Clone(gpuLibs),
@@ -1886,7 +1925,10 @@ func llamaServerStreamLimitError(label string, err error) error {
}
func (s *llamaServerRunner) statusErrorMessage(body []byte) string {
- errMsg := strings.TrimSpace(string(body))
+ errMsg := extractLlamaServerErrorMessage(body)
+ if errMsg == "" {
+ errMsg = strings.TrimSpace(string(body))
+ }
statusMsg := s.lastErrMsg()
if statusMsg == "" {
return errMsg
@@ -2579,6 +2621,8 @@ func extractLlamaServerErrorMessage(body []byte) string {
return ""
}
+const promptTooLongMessage = "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length"
+
func isEmbeddingInputLimitError(errMsg string) bool {
msg := strings.ToLower(errMsg)
return strings.Contains(msg, "too large") ||
@@ -2588,15 +2632,13 @@ func isEmbeddingInputLimitError(errMsg string) bool {
strings.Contains(msg, "exceeds the available context")
}
-func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecial bool, parseSpecial *bool) ([]int, error) {
+func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecial bool) ([]int, error) {
req := struct {
- Content any `json:"content"`
- AddSpecial bool `json:"add_special,omitempty"`
- ParseSpecial *bool `json:"parse_special,omitempty"`
+ Content any `json:"content"`
+ AddSpecial bool `json:"add_special,omitempty"`
}{
- Content: content,
- AddSpecial: addSpecial,
- ParseSpecial: parseSpecial,
+ Content: content,
+ AddSpecial: addSpecial,
}
data, err := json.Marshal(req)
@@ -2637,7 +2679,7 @@ func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecia
// Tokenize calls llama-server's /tokenize endpoint.
func (s *llamaServerRunner) Tokenize(ctx context.Context, content string) ([]int, error) {
- return s.tokenize(ctx, content, false, nil)
+ return s.tokenize(ctx, content, false)
}
// Detokenize calls llama-server's /detokenize endpoint.
diff --git a/llm/llama_server_score.go b/llm/llama_server_score.go
deleted file mode 100644
index fc3474d06..000000000
--- a/llm/llama_server_score.go
+++ /dev/null
@@ -1,370 +0,0 @@
-package llm
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "math"
- "net/http"
- "slices"
- "strings"
-
- "github.com/ollama/ollama/api"
-)
-
-var _ Scorer = (*llamaServerRunner)(nil)
-
-const (
- // Candidates are lifted above every other token so that top_k keeps exactly
- // them; a shared bias does not change their softmax. A larger bias is
- // retried if another token still outranks a candidate.
- scoreBias = 100
- scoreRetryBias = 1000
-)
-
-// A candidate whose probability underflows float32 is omitted by llama-server.
-var scoreFloorLogit = float32(math.Log(math.SmallestNonzeroFloat32))
-
-type scoreRowTokens struct {
- tokens, candidates []int
-}
-
-func (s *llamaServerRunner) Score(ctx context.Context, input ScoreRequest) (ScoreResponse, error) {
- var result ScoreResponse
- badRequest := func(format string, args ...any) (ScoreResponse, error) {
- return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf(format, args...)}
- }
- if len(input.Segments) == 0 && len(input.Fields) == 0 && (len(input.Rows) < 1 || len(input.Rows) > 64) {
- return badRequest("scoring requires 1–64 prompts")
- }
- if input.MaxTokens < 1 || input.MaxTokens > s.ContextLength() {
- return badRequest("max_tokens must be between 1 and %d", s.ContextLength())
- }
- if err := s.sem.Acquire(ctx, 1); err != nil {
- return result, err
- }
- defer s.sem.Release(1)
- status, err := s.getServerStatusRetry(ctx)
- if err != nil {
- return result, err
- } else if status != ServerStatusReady {
- return result, fmt.Errorf("unexpected server status: %s", status)
- }
-
- if len(input.Segments) > 0 || len(input.Fields) > 0 {
- return s.scoreFields(ctx, input)
- }
-
- rows := make([]scoreRowTokens, len(input.Rows))
- parseSpecial, ordinary := true, false
- for i, row := range input.Rows {
- if len(row.Candidates) < 1 || len(row.Candidates) > 26 {
- return badRequest("prompt %d requires 1–26 candidates", i)
- }
- tokens, err := s.tokenize(ctx, row.Prompt, false, &parseSpecial)
- if err != nil {
- return result, err
- }
- if len(tokens) == 0 || len(tokens) > input.MaxTokens {
- return badRequest("prompt %d has %d tokens; expected 1–%d (input is never truncated)", i, len(tokens), input.MaxTokens)
- }
- // llama-server marks the result truncated when prompt+1 reaches the
- // context limit, so scoring needs two positions of headroom.
- if len(tokens)+2 > s.ContextLength() {
- return badRequest("prompt %d requires %d context tokens for scoring; model has %d", i, len(tokens)+2, s.ContextLength())
- }
- rows[i].tokens = tokens
- result.InputTokens += len(tokens)
- for _, candidate := range row.Candidates {
- joined, err := s.tokenize(ctx, row.Prompt+candidate, false, &parseSpecial)
- if err != nil {
- return result, err
- }
- literal, err := s.tokenize(ctx, candidate, false, &ordinary)
- if err != nil {
- return result, err
- }
- if len(joined) != len(tokens)+1 || !slices.Equal(joined[:len(tokens)], tokens) || len(literal) != 1 || literal[0] != joined[len(tokens)] {
- return badRequest("candidate %q must append exactly one ordinary token to prompt %d", candidate, i)
- }
- id := joined[len(tokens)]
- if id < 0 || slices.Contains(rows[i].candidates, id) {
- return badRequest("prompt %d has invalid or duplicate candidate tokens", i)
- }
- rows[i].candidates = append(rows[i].candidates, id)
- }
- }
-
- result.OutputTokens, err = s.primeSharedPrefix(ctx, rows)
- if err != nil {
- return ScoreResponse{}, err
- }
-
- // Rows are scored in order: the first row continues from the primer.
- result.Logits = make([][]float32, len(rows))
- for i, row := range rows {
- logits, outputTokens, err := s.scoreRow(ctx, row)
- if err != nil {
- return ScoreResponse{}, err
- }
- result.Logits[i] = logits
- result.OutputTokens += outputTokens
- }
- return result, nil
-}
-
-// llama-server saves a checkpoint of state it cannot roll back, from recurrent
-// and sliding-window layers, this many tokens before the end of each prompt it
-// evaluates. See https://github.com/ggml-org/llama.cpp/pull/20288.
-const scoreCheckpointOffset = 4
-
-// primeSharedPrefix makes llama-server save a checkpoint at the end of the
-// tokens every row shares, so each row evaluates only its own last few tokens
-// instead of its whole prompt.
-//
-// Rows share the rendered context and schema and differ only near the end.
-// llama-server usually reuses a cached prefix by trimming its cache, but it
-// cannot trim the state of recurrent layers, as in hybrid models like Qwen 3.5,
-// or of sliding-window layers. Such models can only resume from a checkpoint,
-// and the checkpoints llama-server saves on its own sit near the end of each
-// prompt, past the point where the next row differs. Without a primer, every row would be
-// evaluated from the start.
-//
-// The primer evaluates the shared tokens plus the first row's next
-// scoreCheckpointOffset tokens into the cache, so llama-server saves its
-// checkpoint exactly at the end of the shared tokens. The pinned server still
-// generates one token with n_predict 0; it is discarded but counted in usage.
-// The first row continues from the primer; later rows restore the checkpoint.
-// With 8 questions, scoring is about 3x faster.
-//
-// Priming only changes speed: if llama-server stops saving that checkpoint,
-// rows are evaluated from the start and scores stay the same. It can be
-// removed once llama-server can save a checkpoint at a requested position or
-// score rows as one batch.
-func (s *llamaServerRunner) primeSharedPrefix(ctx context.Context, rows []scoreRowTokens) (int, error) {
- if len(rows) < 2 {
- return 0, nil
- }
- first := rows[0].tokens
- shared := len(first)
- for _, row := range rows[1:] {
- shared = min(shared, len(row.tokens))
- for i := range shared {
- if row.tokens[i] != first[i] {
- shared = i
- break
- }
- }
- }
- // Nothing is shared, or the first row is too short to continue from the
- // primer.
- if shared == 0 || len(first) <= shared+scoreCheckpointOffset {
- return 0, nil
- }
- tokens := first[:shared+scoreCheckpointOffset]
- var output scoreCompletionResponse
- if err := s.scoreRequest(ctx, "/completion", scoreCompletionRequest{Prompt: tokens, CachePrompt: true, NPredict: 0}, &output); err != nil {
- return 0, err
- }
- if output.Truncated || output.TokensEvaluated != len(tokens) {
- return 0, fmt.Errorf("scoring did not evaluate the complete shared prefix")
- }
- return output.TokensPredicted, nil
-}
-
-// scoreCompletionRequest is the part of llama-server's /completion request
-// that scoring uses.
-type scoreCompletionRequest struct {
- Prompt []int `json:"prompt"`
- Stream bool `json:"stream"`
- CachePrompt bool `json:"cache_prompt"`
- NPredict int `json:"n_predict"`
- NProbs int `json:"n_probs,omitempty"`
- PostSamplingProbs bool `json:"post_sampling_probs"`
- Samplers []string `json:"samplers,omitempty"`
- TopK int `json:"top_k"`
- Temperature float32 `json:"temperature"`
- LogitBias [][2]float64 `json:"logit_bias,omitempty"`
-}
-
-// scoreCompletionResponse is the part of llama-server's /completion response
-// that scoring reads.
-type scoreCompletionResponse struct {
- TokensPredicted int `json:"tokens_predicted"`
- TokensEvaluated int `json:"tokens_evaluated"`
- Truncated bool `json:"truncated"`
- Probabilities []struct {
- TopProbs []struct {
- ID *int `json:"id"`
- Prob *float64 `json:"prob"`
- } `json:"top_probs"`
- } `json:"completion_probabilities"`
-}
-
-// scoreRow reads every candidate from one next-token distribution. Biasing the
-// candidates by the same amount keeps their relative logits, so the returned
-// probabilities are the softmax over the candidates' original logits.
-func (s *llamaServerRunner) scoreRow(ctx context.Context, row scoreRowTokens) ([]float32, int, error) {
- var outputTokens int
- for _, bias := range []float64{scoreBias, scoreRetryBias} {
- req := scoreCompletionRequest{
- Prompt: row.tokens, CachePrompt: true, NPredict: 1,
- NProbs: len(row.candidates), PostSamplingProbs: true,
- Samplers: []string{"top_k", "temperature"}, TopK: len(row.candidates), Temperature: 1,
- }
- for _, id := range row.candidates {
- req.LogitBias = append(req.LogitBias, [2]float64{float64(id), bias})
- }
- var output scoreCompletionResponse
- if err := s.scoreRequest(ctx, "/completion", req, &output); err != nil {
- return nil, 0, err
- }
- if output.Truncated || output.TokensEvaluated != len(row.tokens) || output.TokensPredicted != 1 || len(output.Probabilities) != 1 || len(output.Probabilities[0].TopProbs) == 0 {
- return nil, 0, fmt.Errorf("scoring did not evaluate the complete prompt and return one distribution")
- }
- outputTokens += output.TokensPredicted
- logits := slices.Repeat([]float32{scoreFloorLogit}, len(row.candidates))
- seen := make([]bool, len(row.candidates))
- outranked := false
- for _, p := range output.Probabilities[0].TopProbs {
- if p.ID == nil || p.Prob == nil || math.IsNaN(*p.Prob) || *p.Prob <= 0 || *p.Prob > 1 {
- return nil, 0, fmt.Errorf("scoring returned an invalid candidate probability")
- }
- j := slices.Index(row.candidates, *p.ID)
- if j < 0 {
- outranked = true
- break
- }
- if seen[j] {
- return nil, 0, fmt.Errorf("scoring returned a candidate twice")
- }
- seen[j] = true
- logits[j] = max(float32(math.Log(*p.Prob)), scoreFloorLogit)
- }
- if !outranked {
- return logits, outputTokens, nil
- }
- }
- return nil, 0, fmt.Errorf("scoring candidates were outranked by other tokens")
-}
-
-func (s *llamaServerRunner) scoreRequest(ctx context.Context, path string, input, output any) error {
- data, err := json.Marshal(input)
- if err != nil {
- return err
- }
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d%s", s.port, path), bytes.NewReader(data))
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", "application/json")
- res, err := s.httpClient().Do(req)
- if err != nil {
- if ctx.Err() != nil {
- return ctx.Err()
- }
- if msg := s.lastErrMsg(); msg != "" {
- return fmt.Errorf("scoring failed: %s: %w", msg, err)
- }
- return err
- }
- defer res.Body.Close()
- body, err := io.ReadAll(res.Body)
- if err != nil {
- return err
- }
- if res.StatusCode != http.StatusOK {
- return api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(body)}
- }
- if err := json.Unmarshal(body, output); err != nil {
- return fmt.Errorf("invalid scoring response: %w", err)
- }
- return nil
-}
-
-func (s *llamaServerRunner) scoreFields(ctx context.Context, input ScoreRequest) (ScoreResponse, error) {
- bad := func(message string) (ScoreResponse, error) {
- return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: message}
- }
- if len(input.Fields) < 1 || len(input.Fields) > 64 || len(input.Segments) == 0 {
- return bad("invalid decision scoring request")
- }
- var err error
- // Do not concatenate before tokenizing: BPE may merge across the reference
- // encoder's segment boundaries, changing both embeddings and span offsets.
- var tokens []int
- offsets := []int{0}
- cache := map[string][]int{}
- special := true
- for _, segment := range input.Segments {
- ids, ok := cache[segment]
- if !ok {
- ids, err = s.tokenize(ctx, segment, false, &special)
- if err != nil {
- return ScoreResponse{}, err
- }
- cache[segment] = ids
- }
- tokens = append(tokens, ids...)
- offsets = append(offsets, len(tokens))
- if len(tokens) > input.MaxTokens {
- return bad("decision prompt exceeds the model context (input is never truncated)")
- }
- }
- if len(tokens) == 0 {
- return bad("decision prompt must not be empty")
- }
- fields := make([]ScoreField, len(input.Fields))
- for i, f := range input.Fields {
- if f.Type < 0 || f.Type > 2 || len(f.Options) < 2 || len(f.Options) > 26 {
- return bad("invalid decision field")
- }
- // Map the question and its options from segment indexes to token offsets.
- spans := append([][2]int{f.Question}, f.Options...)
- for j, span := range spans {
- start, end := span[0], span[1]
- if start < 0 || start >= end || end >= len(offsets) || offsets[start] == offsets[end] {
- return bad("invalid decision token span")
- }
- spans[j] = [2]int{offsets[start], offsets[end]}
- }
- fields[i] = ScoreField{Type: f.Type, Question: spans[0], Options: spans[1:]}
- }
- var images []api.ImageData
- for _, image := range input.Images {
- data, err := llamaServerMediaBytes(image)
- if err != nil {
- return bad(err.Error())
- }
- if !strings.HasPrefix(http.DetectContentType(data), "image/") {
- return bad("invalid decision image")
- }
- images = append(images, data)
- }
- imagePosition := 0
- if len(images) > 0 {
- if input.ImagePosition < 0 || input.ImagePosition >= len(offsets) {
- return bad("invalid image position")
- }
- imagePosition = offsets[input.ImagePosition]
- }
- request := struct {
- Images []api.ImageData `json:"images,omitempty"`
- ImagePosition int `json:"image_position"`
- Input []int `json:"input"`
- Fields []ScoreField `json:"score_fields"`
- }{images, imagePosition, tokens, fields}
- var result []struct {
- Logits [][]float32 `json:"logits"`
- Tokens int `json:"tokens_evaluated"`
- }
- if err := s.scoreRequest(ctx, "/embedding", request, &result); err != nil {
- return ScoreResponse{}, err
- }
- if len(result) != 1 || result[0].Tokens > input.MaxTokens || (len(images) == 0 && result[0].Tokens != len(tokens)) || (len(images) > 0 && result[0].Tokens <= len(tokens)) || len(result[0].Logits) != len(fields) {
- return ScoreResponse{}, fmt.Errorf("decision runner did not score the complete request")
- }
- return ScoreResponse{Logits: result[0].Logits, InputTokens: result[0].Tokens}, nil
-}
diff --git a/llm/llama_server_score_test.go b/llm/llama_server_score_test.go
deleted file mode 100644
index 9995ee289..000000000
--- a/llm/llama_server_score_test.go
+++ /dev/null
@@ -1,545 +0,0 @@
-package llm
-
-import (
- "cmp"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "maps"
- "math"
- "net"
- "net/http"
- "net/http/httptest"
- "reflect"
- "slices"
- "strings"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/ollama/ollama/api"
- "golang.org/x/sync/semaphore"
-)
-
-func newLlamaScoreTestRunner(t *testing.T, completion http.HandlerFunc) *llamaServerRunner {
- t.Helper()
- mux := http.NewServeMux()
- mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprint(w, `{"status":"ok"}`)
- })
- mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) {
- var req struct {
- Content string `json:"content"`
- AddSpecial bool `json:"add_special"`
- ParseSpecial *bool `json:"parse_special"`
- }
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- t.Error(err)
- return
- }
- if req.AddSpecial || req.ParseSpecial == nil {
- t.Error("scoring must explicitly parse special tokens without adding BOS/EOS")
- return
- }
- var tokens []int
- for _, c := range req.Content {
- tokens = append(tokens, int(c))
- }
- if *req.ParseSpecial && strings.HasSuffix(req.Content, "<special>") {
- tokens = append(tokens[:len(tokens)-len("<special>")], 1000)
- }
- // Model a tokenizer that merges the candidate with the prompt's last token.
- if strings.HasSuffix(req.Content, "~") && len(tokens) > 1 {
- tokens = append(tokens[:len(tokens)-2], 1001)
- }
- _ = json.NewEncoder(w).Encode(map[string]any{"tokens": tokens})
- })
- mux.HandleFunc("/completion", completion)
- srv := httptest.NewServer(mux)
- t.Cleanup(srv.Close)
- return &llamaServerRunner{
- port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(),
- cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}},
- sem: semaphore.NewWeighted(1),
- }
-}
-
-type scoreCompletionTestRequest struct {
- Prompt []int `json:"prompt"`
- Grammar string `json:"grammar"`
- Stream bool `json:"stream"`
- CachePrompt bool `json:"cache_prompt"`
- NPredict int `json:"n_predict"`
- NProbs int `json:"n_probs"`
- PostSamplingProbs *bool `json:"post_sampling_probs"`
- Temperature float64 `json:"temperature"`
- Samplers []string `json:"samplers"`
- TopK int `json:"top_k"`
- LogitBias [][2]float64 `json:"logit_bias"`
-}
-
-// scoreCompletionTestResponse returns the candidates' probabilities in the
-// order llama-server sorts them.
-func scoreCompletionTestResponse(prompt int, probs map[int]float64) map[string]any {
- ids := slices.Collect(maps.Keys(probs))
- slices.SortFunc(ids, func(a, b int) int { return cmp.Compare(probs[b], probs[a]) })
- var top []any
- for _, id := range ids {
- top = append(top, map[string]any{"id": id, "token": string(rune(id)), "prob": probs[id]})
- }
- return map[string]any{
- "tokens_predicted": 1, "tokens_evaluated": prompt,
- "completion_probabilities": []any{map[string]any{"id": ids[0], "token": string(rune(ids[0])), "prob": probs[ids[0]], "top_probs": top}},
- }
-}
-
-func checkScoreRowRequest(t *testing.T, req scoreCompletionTestRequest, candidates []int, bias float64) {
- t.Helper()
- var biased []int
- for _, b := range req.LogitBias {
- if b[1] != bias {
- t.Errorf("candidates must share bias %g: %v", bias, req.LogitBias)
- }
- biased = append(biased, int(b[0]))
- }
- if req.Stream || !req.CachePrompt || req.NPredict != 1 || req.Grammar != "" || req.NProbs != len(candidates) || req.PostSamplingProbs == nil || !*req.PostSamplingProbs ||
- req.Temperature != 1 || !slices.Equal(req.Samplers, []string{"top_k", "temperature"}) || req.TopK != len(candidates) || !slices.Equal(biased, candidates) {
- t.Errorf("request changes the candidate distribution: %+v", req)
- }
-}
-
-func TestLlamaServerScore(t *testing.T) {
- var calls int
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- var req scoreCompletionTestRequest
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- t.Error(err)
- return
- }
- calls++
- switch {
- case slices.Equal(req.Prompt, []int{'a', 'b', 'c'}):
- checkScoreRowRequest(t, req, []int{'Z', 'A'}, scoreBias)
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'Z': 0.25, 'A': 0.75}))
- case slices.Equal(req.Prompt, []int{'x', 'y'}):
- checkScoreRowRequest(t, req, []int{'B'}, scoreBias)
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'B': 1}))
- default:
- t.Errorf("rendered prompt changed: %v", req.Prompt)
- }
- })
- // Model generation defaults must not affect scoring.
- runner.options.Temperature = 0
- runner.options.Stop = []string{"A"}
- runner.options.RepeatPenalty = 2
- runner.options.NumCtx = 5 // The longest prompt leaves exactly two positions of headroom.
- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{
- {Prompt: "abc", Candidates: []string{"Z", "A"}},
- {Prompt: "xy", Candidates: []string{"B"}},
- }})
- if err != nil {
- t.Fatal(err)
- }
- want := [][]float32{{float32(math.Log(0.25)), float32(math.Log(0.75))}, {0}}
- if !reflect.DeepEqual(result.Logits, want) || result.InputTokens != 5 || result.OutputTokens != 2 || calls != 2 {
- t.Fatalf("incomplete or reordered scores/usage: %+v, calls=%d", result, calls)
- }
-}
-
-func TestLlamaServerScoreSharedPrefix(t *testing.T) {
- var prompts [][]int
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- var req scoreCompletionTestRequest
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- t.Error(err)
- return
- }
- prompts = append(prompts, req.Prompt)
- if len(prompts) == 1 {
- // The native server generates one token even with n_predict: 0.
- if req.NPredict != 0 || req.NProbs != 0 || len(req.LogitBias) != 0 || len(req.Samplers) != 0 || !req.CachePrompt {
- t.Errorf("unexpected primer request: %+v", req)
- }
- _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})
- return
- }
- candidate := int(req.LogitBias[0][0])
- checkScoreRowRequest(t, req, []int{candidate}, scoreBias)
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{candidate: 1}))
- })
- runner.options.NumCtx = 64
- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 32, Rows: []ScoreRow{
- {Prompt: "shared:first", Candidates: []string{"A"}},
- {Prompt: "shared:two", Candidates: []string{"B"}},
- {Prompt: "shared:x", Candidates: []string{"C"}},
- }})
- if err != nil {
- t.Fatal(err)
- }
- // The primer evaluates the shared prefix plus 4 tokens of the first row,
- // which continues from it; the other rows restore the checkpoint.
- want := [][]int{scoreTestTokens("shared:firs"), scoreTestTokens("shared:first"), scoreTestTokens("shared:two"), scoreTestTokens("shared:x")}
- if !reflect.DeepEqual(prompts, want) {
- t.Fatalf("got prompts %v, want %v", prompts, want)
- }
- if !reflect.DeepEqual(result.Logits, [][]float32{{0}, {0}, {0}}) || result.InputTokens != 30 || result.OutputTokens != 4 {
- t.Fatalf("unexpected result: %+v", result)
- }
-}
-
-func TestLlamaServerScorePrimer(t *testing.T) {
- for _, tt := range []struct {
- name string
- rows []string
- primer string
- }{
- {"one row", []string{"abcdefgh"}, ""},
- {"no shared prefix", []string{"abcdefgh", "xbcdefgh"}, ""},
- {"first row too short", []string{"abcdefg", "abcdwxyzuv"}, ""},
- {"identical prompts", []string{"abcdefgh", "abcdefgh"}, ""},
- {"shared prefix", []string{"abcdefghij", "abcxyzuvwt", "abcdefgh"}, "abcdefg"},
- } {
- t.Run(tt.name, func(t *testing.T) {
- var primer []int
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- var req scoreCompletionTestRequest
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- t.Error(err)
- return
- }
- if len(req.LogitBias) == 0 {
- primer = req.Prompt
- _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})
- return
- }
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'A': 1}))
- })
- runner.options.NumCtx = 64
- input := ScoreRequest{MaxTokens: 32}
- for _, row := range tt.rows {
- input.Rows = append(input.Rows, ScoreRow{Prompt: row, Candidates: []string{"A"}})
- }
- if _, err := runner.Score(t.Context(), input); err != nil {
- t.Fatal(err)
- }
- if !slices.Equal(primer, scoreTestTokens(tt.primer)) {
- t.Fatalf("primed %v, want %v", primer, scoreTestTokens(tt.primer))
- }
- })
- }
-}
-
-// scoreTestTokens tokenizes s the way the test tokenizer does: one token per rune.
-func scoreTestTokens(s string) []int {
- var tokens []int
- for _, c := range s {
- tokens = append(tokens, int(c))
- }
- return tokens
-}
-
-func TestLlamaServerScoreOutranked(t *testing.T) {
- for _, recovers := range []bool{true, false} {
- t.Run(fmt.Sprintf("recovers=%v", recovers), func(t *testing.T) {
- var calls int
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- var req scoreCompletionTestRequest
- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- t.Error(err)
- return
- }
- calls++
- bias := float64(scoreBias)
- if calls > 1 {
- bias = scoreRetryBias
- }
- checkScoreRowRequest(t, req, []int{'A', 'B'}, bias)
- probs := map[int]float64{'A': 0.6, 'x': 0.4}
- if recovers && calls > 1 {
- probs = map[int]float64{'A': 0.6, 'B': 0.4}
- }
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), probs))
- })
- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})
- if calls != 2 {
- t.Fatalf("got %d attempts, want 2", calls)
- }
- if !recovers {
- if err == nil {
- t.Fatal("accepted a distribution without every candidate")
- }
- return
- }
- if err != nil || result.OutputTokens != 2 || !reflect.DeepEqual(result.Logits, [][]float32{{float32(math.Log(0.6)), float32(math.Log(0.4))}}) {
- t.Fatalf("unexpected result: %+v, %v", result, err)
- }
- })
- }
-}
-
-func TestLlamaServerScoreUnderflow(t *testing.T) {
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- // llama-server omits candidates whose probability underflows.
- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(3, map[int]float64{'B': 1}))
- })
- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})
- if err != nil || !reflect.DeepEqual(result.Logits, [][]float32{{scoreFloorLogit, 0}}) {
- t.Fatalf("unexpected result: %+v, %v", result, err)
- }
-}
-
-func TestLlamaServerScoreValidation(t *testing.T) {
- for _, tt := range []struct {
- name string
- input ScoreRequest
- }{
- {"empty rows", ScoreRequest{MaxTokens: 7}},
- {"too many rows", ScoreRequest{MaxTokens: 7, Rows: make([]ScoreRow, 65)}},
- {"zero max tokens", ScoreRequest{Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},
- {"max exceeds context", ScoreRequest{MaxTokens: 9, Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},
- {"empty prompt", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Candidates: []string{"A"}}}}},
- {"oversize prompt", ScoreRequest{MaxTokens: 2, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}},
- {"no scoring token space", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefgh", Candidates: []string{"A"}}}}},
- {"no native scoring headroom", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefg", Candidates: []string{"A"}}}}},
- {"empty candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc"}}}},
- {"too many candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: make([]string, 27)}}}},
- {"multiple tokens", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"AB"}}}}},
- {"special token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"<special>"}}}}},
- {"changed prefix", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"~"}}}}},
- {"duplicate token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "A"}}}}},
- {"invalid later row", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}, {Prompt: "abc", Candidates: []string{"AB"}}}}},
- } {
- t.Run(tt.name, func(t *testing.T) {
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- t.Error("started inference before validating every row")
- })
- _, err := runner.Score(t.Context(), tt.input)
- var status api.StatusError
- if !errors.As(err, &status) || status.StatusCode != http.StatusBadRequest {
- t.Fatalf("expected validation error, got %v", err)
- }
- })
- }
-}
-
-func TestLlamaServerScoreResponse(t *testing.T) {
- const valid = `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`
- for _, tt := range []struct {
- name string
- body string
- status int
- }{
- {"missing probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A"}]`, 1), 200},
- {"missing token id", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"token":"A","prob":1}]`, 1), 200},
- {"zero probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":0}]`, 1), 200},
- {"probability above one", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":1.5}]`, 1), 200},
- {"duplicate candidate", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"id":65,"token":"A","prob":0.5},{"id":65,"token":"A","prob":0.5}]`, 1), 200},
- {"empty top probabilities", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[]`, 1), 200},
- {"incomplete prompt", strings.Replace(valid, `"tokens_evaluated":3`, `"tokens_evaluated":2`, 1), 200},
- {"truncated prompt", strings.Replace(valid, `"tokens_predicted":1`, `"truncated":true,"tokens_predicted":1`, 1), 200},
- {"extra generated token", strings.Replace(valid, `"tokens_predicted":1`, `"tokens_predicted":2`, 1), 200},
- {"empty probabilities", `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[]}`, 200},
- {"malformed JSON", `{`, 200},
- {"upstream error", `{"error":{"message":"out of memory"}}`, 500},
- } {
- t.Run(tt.name, func(t *testing.T) {
- var failed atomic.Bool
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- if failed.CompareAndSwap(false, true) {
- w.WriteHeader(tt.status)
- fmt.Fprint(w, tt.body)
- return
- }
- fmt.Fprint(w, valid)
- })
- input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}
- if _, err := runner.Score(t.Context(), input); err == nil {
- t.Fatal("accepted invalid scoring response")
- } else if tt.status == 500 {
- var status api.StatusError
- if !errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err) {
- t.Fatalf("lost upstream status/OOM diagnostic: %v", err)
- }
- }
- ctx, cancel := context.WithTimeout(t.Context(), time.Second)
- defer cancel()
- if _, err := runner.Score(ctx, input); err != nil {
- t.Fatalf("runner unusable after failed scoring: %v", err)
- }
- })
- }
-}
-
-func TestLlamaServerScoreCancellation(t *testing.T) {
- for _, queued := range []bool{true, false} {
- t.Run(fmt.Sprintf("queued=%v", queued), func(t *testing.T) {
- started := make(chan struct{})
- var block atomic.Bool
- block.Store(!queued)
- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {
- if block.CompareAndSwap(true, false) {
- _, _ = io.Copy(io.Discard, r.Body)
- close(started)
- <-r.Context().Done()
- return
- }
- fmt.Fprint(w, `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`)
- })
- input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}
- ctx, cancel := context.WithCancel(t.Context())
- defer cancel()
- if queued {
- if err := runner.sem.Acquire(t.Context(), 1); err != nil {
- t.Fatal(err)
- }
- cancel()
- } else {
- go func() {
- select {
- case <-started:
- cancel()
- case <-ctx.Done():
- }
- }()
- }
- _, err := runner.Score(ctx, input)
- if !errors.Is(err, context.Canceled) {
- t.Fatalf("cancellation lost: %v", err)
- }
- if queued {
- runner.sem.Release(1)
- }
- next, done := context.WithTimeout(t.Context(), time.Second)
- defer done()
- if _, err := runner.Score(next, input); err != nil {
- t.Fatalf("runner unusable after cancellation: %v", err)
- }
- })
- }
-}
-
-func TestLlamaServerScoreFields(t *testing.T) {
- for _, failure := range []string{"", "truncated", "missing head", "malformed JSON", "upstream error", "too long", "invalid span", "max exceeds context", "empty fields", "canceled", "image", "image over context", "invalid image", "invalid image position"} {
- t.Run(failure, func(t *testing.T) {
- calls := 0
- mux := http.NewServeMux()
- mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) })
- mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) {
- var input struct {
- Content string `json:"content"`
- AddSpecial bool `json:"add_special"`
- ParseSpecial bool `json:"parse_special"`
- }
- if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
- t.Fatal(err)
- }
- if input.AddSpecial || !input.ParseSpecial {
- t.Error("incorrect tokenizer settings")
- }
- ids := map[string][]int{"prefix": {1, 2}, "question": {3}, "one": {4, 5}, "two": {6}, "suffix": {7}}[input.Content]
- if ids == nil {
- t.Error("segments must be tokenized separately")
- }
- json.NewEncoder(w).Encode(map[string]any{"tokens": ids})
- })
- mux.HandleFunc("/embedding", func(w http.ResponseWriter, r *http.Request) {
- calls++
- var input struct {
- Input []int `json:"input"`
- Fields []ScoreField `json:"score_fields"`
- Images []api.ImageData `json:"images"`
- ImagePosition int `json:"image_position"`
- }
- if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
- t.Fatal(err)
- }
- if !reflect.DeepEqual(input.Input, []int{1, 2, 3, 4, 5, 6, 7}) || !reflect.DeepEqual(input.Fields, []ScoreField{{Type: 0, Question: [2]int{2, 3}, Options: [][2]int{{3, 5}, {5, 6}}}}) {
- t.Errorf("wrong token spans: %+v", input)
- }
- switch failure {
- case "image":
- if len(input.Images) != 1 || input.ImagePosition != 2 {
- t.Errorf("missing image or wrong image position: %+v", input)
- }
- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":11}]`)
- case "image over context":
- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":17}]`)
- case "upstream error":
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprint(w, `{"error":{"message":"out of memory"}}`)
- case "malformed JSON":
- fmt.Fprint(w, `{`)
- case "truncated":
- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":6}]`)
- case "missing head":
- fmt.Fprint(w, `[{"embedding":[[2,0]]}]`)
- default:
- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":7}]`)
- }
- })
- srv := httptest.NewServer(mux)
- defer srv.Close()
- runner := &llamaServerRunner{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(), cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}}, sem: semaphore.NewWeighted(1)}
- input := ScoreRequest{MaxTokens: 8, Segments: []string{"prefix", "question", "one", "two", "suffix"}, Fields: []ScoreField{{Type: 0, Question: [2]int{1, 2}, Options: [][2]int{{2, 3}, {3, 4}}}}}
- ctx, cancel := context.WithCancel(t.Context())
- defer cancel()
- invalid := true
- switch failure {
- case "image", "image over context", "invalid image", "invalid image position":
- input.Images = []api.ImageData{{137, 80, 78, 71, 13, 10, 26, 10}}
- input.ImagePosition = 1
- if failure == "invalid image" {
- input.Images[0] = []byte("not an image")
- }
- if failure == "invalid image position" {
- input.ImagePosition = 99
- }
- if failure == "image" || failure == "image over context" {
- invalid = false
- input.MaxTokens = 16
- runner.options.NumCtx = 16
- }
- case "too long":
- input.MaxTokens = 6
- case "invalid span":
- input.Fields[0].Question = [2]int{-1, 2}
- case "max exceeds context":
- input.MaxTokens = 9
- case "empty fields":
- input.Fields = nil
- case "canceled":
- cancel()
- default:
- invalid = false
- }
- result, err := runner.Score(ctx, input)
- if failure == "" || failure == "image" {
- wantTokens := 7
- if failure == "image" {
- wantTokens = 11
- }
- if err != nil || result.InputTokens != wantTokens || result.OutputTokens != 0 {
- t.Fatalf("result=%+v err=%v", result, err)
- }
- } else if err == nil {
- t.Fatal("expected error")
- }
- var status api.StatusError
- if failure == "upstream error" && (!errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err)) {
- t.Fatalf("lost upstream status/OOM diagnostic: %v", err)
- }
- if failure == "canceled" && !errors.Is(err, context.Canceled) {
- t.Fatalf("cancellation lost: %v", err)
- }
- if invalid && calls != 0 {
- t.Fatal("invalid request reached the model")
- }
- if !runner.sem.TryAcquire(1) {
- t.Fatal("scoring did not release the runner")
- }
- runner.sem.Release(1)
- })
- }
-}
diff --git a/llm/llama_server_systemone.go b/llm/llama_server_systemone.go
new file mode 100644
index 000000000..2e6373f8b
--- /dev/null
+++ b/llm/llama_server_systemone.go
@@ -0,0 +1,76 @@
+package llm
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+
+ "github.com/ollama/ollama/api"
+)
+
+func (s *llamaServerRunner) SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (answers, usage json.RawMessage, err error) {
+ request := struct {
+ State json.RawMessage `json:"state"`
+ Images []string `json:"images,omitempty"`
+ Questions json.RawMessage `json:"questions"`
+ }{State: state, Questions: questions}
+ for _, image := range images {
+ data, err := llamaServerMediaBytes(image)
+ if err != nil {
+ return nil, nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: err.Error()}
+ }
+ request.Images = append(request.Images, "data:"+http.DetectContentType(data)+";base64,"+base64.StdEncoding.EncodeToString(data))
+ }
+ data, err := json.Marshal(request)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if err := s.sem.Acquire(ctx, 1); err != nil {
+ return nil, nil, err
+ }
+ defer s.sem.Release(1)
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/v1/systemone", s.port), bytes.NewReader(data))
+ if err != nil {
+ return nil, nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ res, err := s.httpClient().Do(req)
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, nil, ctx.Err()
+ }
+ if msg := s.lastErrMsg(); msg != "" {
+ return nil, nil, fmt.Errorf("decision failed: %s: %w", msg, err)
+ }
+ return nil, nil, err
+ }
+ defer res.Body.Close()
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, nil, err
+ }
+ if res.StatusCode != http.StatusOK {
+ msg := s.statusErrorMessage(body)
+ // Decision models that read a whole prompt in one batch run with a batch
+ // as large as the context, so outgrowing it means outgrowing the context.
+ if strings.Contains(msg, "physical batch size") {
+ return nil, nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: promptTooLongMessage}
+ }
+ return nil, nil, api.StatusError{StatusCode: res.StatusCode, ErrorMessage: msg}
+ }
+ var response struct {
+ Answers json.RawMessage `json:"answers"`
+ Usage json.RawMessage `json:"usage"`
+ }
+ if err := json.Unmarshal(body, &response); err != nil {
+ return nil, nil, fmt.Errorf("invalid decision response: %w", err)
+ }
+ return response.Answers, response.Usage, nil
+}
diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go
index 2025e5a6c..62cdc755c 100644
--- a/llm/llama_server_test.go
+++ b/llm/llama_server_test.go
@@ -951,8 +951,8 @@ func TestLlamaServerStatusErrorMessageIncludesOOMStatus(t *testing.T) {
}
got := runner.statusErrorMessage([]byte(`{"error":{"message":"Compute error."}}`))
- if !strings.Contains(got, "Compute error") {
- t.Fatalf("expected original response body, got %q", got)
+ if !strings.HasPrefix(got, "Compute error.\n") {
+ t.Fatalf("expected llama-server's error message, got %q", got)
}
if !IsOutOfMemoryMessage(got) {
t.Fatalf("expected OOM status detail to be detectable, got %q", got)
@@ -1725,6 +1725,21 @@ func TestLlamaServerEmbedding(t *testing.T) {
}
}
+func TestLlamaServerSystemOnePromptOverContext(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, `{"error":{"code":500,"message":"input (9000 tokens) is too large to process. increase the physical batch size (current batch size: 8192)","type":"server_error"}}`)
+ }))
+ defer srv.Close()
+ runner := &llamaServerRunner{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(), cmd: fakeRunningCmd(), sem: semaphore.NewWeighted(1)}
+
+ _, _, err := runner.SystemOne(t.Context(), json.RawMessage(`"x"`), json.RawMessage(`{"q":{"type":"noul","instructions":"q"}}`), nil)
+ var serr api.StatusError
+ if !errors.As(err, &serr) || serr.StatusCode != http.StatusBadRequest || serr.ErrorMessage != promptTooLongMessage {
+ t.Fatalf("err = %v, want 400 %q", err, promptTooLongMessage)
+ }
+}
+
func TestLegacyEmbeddingsWereRaw(t *testing.T) {
tests := []struct {
name string
@@ -1915,6 +1930,40 @@ func TestEmbeddingBatchSize(t *testing.T) {
}
}
+func TestLlamaServerBatch(t *testing.T) {
+ tests := []struct {
+ name string
+ headBlocks, trainCtx uint64
+ numCtx, numBatch int
+ completion bool
+ wantBatch int
+ wantMemory uint64
+ }{
+ {"default generation", 0, 8192, 8192, 512, true, 512, 0},
+ {"medium generation", 0, 8192, 8192, 1024, true, 1024, 768 << 20},
+ {"large generation", 0, 8192, 8192, 2048, true, 2048, 2 << 30},
+ {"other non-completion models", 0, 8192, 8192, 2048, false, 2048, 0},
+ {"decision uses requested context", 4, 32768, 8192, 512, false, 8192, 2 << 30},
+ {"decision caps at trained context", 4, 1024, 8192, 512, false, 1024, 768 << 20},
+ {"decision without trained context", 4, 0, 8192, 512, false, 8192, 2 << 30},
+ {"decision with an already matching batch", 4, 8192, 8192, 8192, false, 8192, 2 << 30},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f := loadTestGGUF(t, gguftest.KV{
+ "general.architecture": "clef",
+ "clef.context_length": tt.trainCtx,
+ "clef.decision.block_count": tt.headBlocks,
+ })
+ opts := api.Options{Runner: api.Runner{NumCtx: tt.numCtx, NumBatch: tt.numBatch}}
+ batch, memory := LlamaServerBatch(f, opts, tt.completion)
+ if batch != tt.wantBatch || memory != tt.wantMemory {
+ t.Fatalf("batch=%d memory=%d, want batch=%d memory=%d", batch, memory, tt.wantBatch, tt.wantMemory)
+ }
+ })
+ }
+}
+
func TestAppendBatchArgs(t *testing.T) {
tests := []struct {
name string
diff --git a/server/capabilities_test.go b/server/capabilities_test.go
index 5541c6cad..f447cb123 100644
--- a/server/capabilities_test.go
+++ b/server/capabilities_test.go
@@ -126,6 +126,10 @@ func TestDecisionPublicCapabilities(t *testing.T) {
want := []model.Capability{model.CapabilityDecision}
if declaredVision {
cfg.Capabilities = append(cfg.Capabilities, "vision")
+ }
+ // The GGUF inherits vision from its base model; the safetensors
+ // fixture ships a projector.
+ if declaredVision || format == "safetensors" {
want = append(want, model.CapabilityVision)
}
_, digest := createBinFile(t, map[string]any{
diff --git a/server/images.go b/server/images.go
index 7df4dd179..07fc72d29 100644
--- a/server/images.go
+++ b/server/images.go
@@ -164,13 +164,14 @@ func (m *Model) Capabilities() []model.Capability {
return capabilities
}
-// publicCapabilities limits decision models to decision and explicitly declared
-// vision support. Serving still uses Capabilities.
+// publicCapabilities limits decision models to decision and the vision they
+// declare or ship a projector for, not vision inherited from a base model.
+// Serving still uses Capabilities.
func (m *Model) publicCapabilities() []model.Capability {
capabilities := m.Capabilities()
if slices.Contains(capabilities, model.CapabilityDecision) {
public := []model.Capability{model.CapabilityDecision}
- if slices.Contains(m.Config.Capabilities, "vision") && slices.Contains(capabilities, model.CapabilityVision) {
+ if slices.Contains(m.Config.Capabilities, "vision") || len(m.ProjectorPaths) > 0 {
public = append(public, model.CapabilityVision)
}
return public
@@ -859,7 +860,7 @@ func GetModelForRunner(name, runner string) (*Model, error) {
m.PreferChatTemplate = true
}
- if m.ModelPath != "" && m.isGGUF() && !modelHasPooling && !m.HasChatTemplate && (!m.HasGoTemplate || !envconfig.GoTemplate(true)) && m.Config.Renderer == "" && m.Config.Parser == "" && !usesHarmony {
+ if m.ModelPath != "" && m.isGGUF() && !modelHasPooling && !m.HasChatTemplate && (!m.HasGoTemplate || !envconfig.GoTemplate(true)) && m.Config.Renderer == "" && m.Config.Parser == "" && !usesHarmony && m.metadata.String("decision.type") == "" {
slog.Warn("model is missing tokenizer.chat_template and Go TEMPLATE support is unavailable; chat responses may be poorly formatted", "model", m.Name, "env", "OLLAMA_GO_TEMPLATE=1")
}
diff --git a/server/routes.go b/server/routes.go
index a93fda47f..95e863670 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -896,8 +896,66 @@ func (s *Server) GenerateHandler(c *gin.Context) {
streamResponse(c, ch)
}
-// SystemOneHandler compiles typed questions, scores their allowed answers, and
-// returns probabilities. Callers must select weights trained for the prompt format.
+func (s *Server) systemOneLlamaServer(c *gin.Context, m *Model, req decision.Request, body []byte) {
+ decisionType := m.metadata.String("decision.type")
+ // Old Clef GGUFs have a decision type but use qwen35; llama.cpp now
+ // requires the clef architecture and its converted decision head.
+ oldClef := decisionType == "clef" && m.metadata.String("general.architecture") == "qwen35"
+ if decisionType == "" || oldClef {
+ if oldClef || slices.Contains(m.Config.Capabilities, string(model.CapabilityDecision)) {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("'%s' was downloaded in a format that is no longer supported. To re-download, run 'ollama pull %s'", m.ShortName, m.ShortName)})
+ return
+ }
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q is not a llama.cpp decision model", req.Model)})
+ return
+ }
+ if err := decision.ValidateLimits(req); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ var raw struct {
+ State json.RawMessage `json:"state"`
+ Questions json.RawMessage `json:"questions"`
+ }
+ if err := json.Unmarshal(body, &raw); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ caps := []model.Capability{model.CapabilityDecision}
+ if len(req.Images) > 0 {
+ caps = append(caps, model.CapabilityVision)
+ }
+ r, _, _, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, nil, req.KeepAlive, nil)
+ if err != nil {
+ handleScheduleError(c, req.Model, err)
+ return
+ }
+ runner, ok := r.(interface {
+ SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (answers, usage json.RawMessage, err error)
+ })
+ if !ok {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring", req.Model)})
+ return
+ }
+ answers, usage, err := runner.SystemOne(c.Request.Context(), raw.State, raw.Questions, req.Images)
+ if err != nil {
+ s.sched.expireRunnersForRuntimeOOM(m, err)
+ status := http.StatusInternalServerError
+ var statusErr api.StatusError
+ if errors.As(err, &statusErr) {
+ status = statusErr.StatusCode
+ }
+ c.JSON(status, gin.H{"error": err.Error()})
+ return
+ }
+ c.JSON(http.StatusOK, struct {
+ Model string `json:"model"`
+ Answers json.RawMessage `json:"answers"`
+ Usage json.RawMessage `json:"usage"`
+ }{req.Model, answers, usage})
+}
+
+// SystemOneHandler answers typed questions with probabilities.
func (s *Server) SystemOneHandler(c *gin.Context) {
// TODO(parthsareen): Check token limits before copying state and schema into
// each question's prompt. This byte cap limits memory use until then.
@@ -952,8 +1010,12 @@ func (s *Server) SystemOneHandler(c *gin.Context) {
handleScheduleError(c, req.Model, err)
return
}
- encoding := m.metadata.String("decision.type")
- if encoding == "" && (m.Config.Renderer == "tev1" || m.Config.Renderer == "clef" || m.Config.Renderer == "strands") {
+ if m.isGGUF() {
+ s.systemOneLlamaServer(c, m, req, body)
+ return
+ }
+ var encoding string
+ if m.Config.Renderer == "tev1" || m.Config.Renderer == "clef" || m.Config.Renderer == "strands" {
encoding = m.Config.Renderer
}
compiled, err := decision.Compile(req, encoding)
@@ -979,11 +1041,7 @@ func (s *Server) SystemOneHandler(c *gin.Context) {
if m.System != "" {
messages = append([]api.Message{{Role: "system", Content: m.System}}, messages...)
}
- think := &api.ThinkValue{Value: false}
- if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative {
- return r.ApplyChatTemplate(c.Request.Context(), llm.ChatRequest{Messages: messages, Think: think})
- }
- return renderPrompt(m, messages, nil, think)
+ return renderPrompt(m, messages, nil, &api.ThinkValue{Value: false})
}); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
diff --git a/server/sched.go b/server/sched.go
index 5f5cb51b8..e4bb15a55 100644
--- a/server/sched.go
+++ b/server/sched.go
@@ -549,7 +549,9 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
flashAttention := llm.LlamaServerFlashAttention(loadGpus)
req.applyAutomaticGenerationBatch(completion, predictedCtx, predicted, availableForBatch, flashAttention, loadGpus)
launchOpts.NumBatch = req.opts.NumBatch
- predictedForLoad := predicted + generationBatchSurchargeForCompletion(completion, launchOpts.NumBatch)
+ var batchMemory uint64
+ launchOpts.NumBatch, batchMemory = llm.LlamaServerBatch(f, launchOpts, completion)
+ predictedForLoad := predicted + batchMemory
// Pre-flight check: estimate whether the model fits in remaining memory.
// llama-server auto-detects layers based on available VRAM, so if
@@ -831,13 +833,6 @@ func (req *LlmRequest) applyAutomaticGenerationBatch(completion bool, effectiveC
req.opts.NumBatch = automaticGenerationBatch(effectiveCtx, predictedVRAM, availableMemory, flashAttention, gpus)
}
-func generationBatchSurchargeForCompletion(completion bool, batch int) uint64 {
- if !completion {
- return 0
- }
- return generationBatchSurcharge(batch)
-}
-
func automaticGenerationBatch(effectiveCtx int, predictedVRAM, availableMemory uint64, flashAttention ml.FlashAttentionType, gpus []ml.DeviceInfo) int {
if flashAttention == ml.FlashAttentionDisabled && hasCUDADevice(gpus) {
if constrainedCUDAWithoutFlashAttention(effectiveCtx, gpus) {
@@ -899,7 +894,7 @@ func generationBatchFits(batch int, predictedVRAM, availableMemory uint64) bool
return false
}
- return generationBatchSurcharge(batch) <= threshold-predictedVRAM
+ return llm.LlamaServerBatchMemory(batch) <= threshold-predictedVRAM
}
func generationBatchHasHeadroom(batch int, predictedVRAM, availableMemory uint64) bool {
@@ -922,17 +917,6 @@ func nextLowerGenerationBatch(batch int) int {
}
}
-func generationBatchSurcharge(batch int) uint64 {
- switch {
- case batch >= llamaServerGenerationBatchLarge:
- return 2 * format.GibiByte
- case batch >= llamaServerGenerationBatchMedium:
- return 768 * format.MebiByte
- default:
- return 0
- }
-}
-
func nextLowerAutoNumCtx(numCtx int) (int, bool) {
switch {
case numCtx > 32768:
diff --git a/server/systemone_test.go b/server/systemone_test.go
index d4f8f855e..cd29ab6d2 100644
--- a/server/systemone_test.go
+++ b/server/systemone_test.go
@@ -28,6 +28,25 @@ type systemOneTestRunner struct {
request llm.ScoreRequest
err error
calls int
+ native bool
+ state json.RawMessage
+}
+
+func (r *systemOneTestRunner) SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (json.RawMessage, json.RawMessage, error) {
+ r.calls++
+ r.native = true
+ r.state = state
+ r.request = llm.ScoreRequest{Images: images}
+ var ids map[string]json.RawMessage
+ if err := json.Unmarshal(questions, &ids); err != nil {
+ return nil, nil, err
+ }
+ answers := map[string]any{}
+ for id := range ids {
+ answers[id] = map[string]any{"type": "noul", "noul": 0.9}
+ }
+ data, _ := json.Marshal(answers)
+ return data, json.RawMessage(`{"input_tokens":123,"output_tokens":0}`), r.err
}
func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest) (llm.ScoreResponse, error) {
@@ -36,7 +55,15 @@ func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest)
if len(input.Fields) > 0 {
return llm.ScoreResponse{Logits: [][]float32{{2, 0}}, InputTokens: 123}, r.err
}
- return llm.ScoreResponse{Logits: [][]float32{{0, 2}}, InputTokens: 123, OutputTokens: 2}, r.err
+ logits := [][]float32{{0, 2}}
+ if len(input.Rows) > 0 {
+ logits = make([][]float32, len(input.Rows))
+ for i, row := range input.Rows {
+ logits[i] = make([]float32, len(row.Candidates))
+ logits[i][len(row.Candidates)-1] = 2
+ }
+ }
+ return llm.ScoreResponse{Logits: logits, InputTokens: 123, OutputTokens: 2}, r.err
}
func TestDecisionModelRejectsCompletion(t *testing.T) {
@@ -95,61 +122,38 @@ func TestSystemOneHandler(t *testing.T) {
config.Renderer = "qwen3.5"
config.Capabilities = []string{"completion"}
createSafetensorsTestModel(t, "safetensors-undeclared", config, nil)
- for _, modelConfig := range []struct {
- name, architecture, renderer, system, template string
- contextLength int
- undeclared bool
- }{
- {"renamed-clef", "qwen35", "", "Ignored for the joint head", "", 1024, false},
- {"decision-test", "qwen35", "qwen3.5", "Model-specific scoring instructions.", "", 1024, false},
- {"gguf-tev1", "qwen35", "tev1", "Model-specific scoring instructions.", "", 1024, false},
- {"gguf-decision", "qwen35", "", "Native model scoring instructions.", "", 4096, false},
- {"go-template", "qwen35", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024, false},
- {"no-system", "qwen35", "qwen3.5", "", "", 1024, false},
- {"gguf-undeclared", "qwen35", "qwen3.5", "", "", 1024, true},
- {"gguf-other-architecture", "llama", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024, false},
+ config.Capabilities = []string{"completion", "decision"}
+ system, err := manifest.NewLayer(strings.NewReader("Model-specific scoring instructions."), "application/vnd.ollama.image.system")
+ if err != nil {
+ t.Fatal(err)
+ }
+ createSafetensorsTestModel(t, "decision-test", config, []manifest.Layer{params, system})
+ goTemplate, err := manifest.NewLayer(strings.NewReader("custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:"), "application/vnd.ollama.image.template")
+ if err != nil {
+ t.Fatal(err)
+ }
+ config.Renderer = ""
+ createSafetensorsTestModel(t, "go-template", config, []manifest.Layer{params, system, goTemplate})
+ for _, gguf := range []struct{ name, architecture, decisionType, renderer string }{
+ {"renamed-clef", "clef", "clef", ""},
+ {"old-clef", "qwen35", "clef", ""},
+ {"laya-native", "modern-bert", "laya", ""},
+ {"gguf-tev1", "qwen35", "", "tev1"},
} {
- kv := gguftest.KV{"general.architecture": modelConfig.architecture}
- if modelConfig.template == "" {
- kv["tokenizer.chat_template"] = "{{ messages }}"
+ kv := gguftest.KV{"general.architecture": gguf.architecture}
+ caps := []string{"decision"}
+ if gguf.decisionType != "" {
+ kv[gguf.architecture+".decision.type"] = gguf.decisionType
}
- if modelConfig.name == "renamed-clef" {
- kv[modelConfig.architecture+".decision.type"] = "clef"
+ if gguf.decisionType == "clef" {
+ caps = append(caps, "vision")
}
_, digest := createBinFile(t, kv, nil)
- caps := []string{"completion", "decision"}
- if modelConfig.name == "renamed-clef" {
- caps = []string{"decision", "vision"}
- }
- if modelConfig.undeclared {
- caps = []string{"completion"}
- }
- configLayer, err := createConfigLayer(model.ConfigV2{
- ModelFormat: "gguf", ModelFamily: modelConfig.architecture,
- Renderer: modelConfig.renderer, Capabilities: caps,
- })
+ configLayer, err := createConfigLayer(model.ConfigV2{ModelFormat: "gguf", ModelFamily: gguf.architecture, Renderer: gguf.renderer, Capabilities: caps})
if err != nil {
t.Fatal(err)
}
- layers := []manifest.Layer{{MediaType: "application/vnd.ollama.image.model", Digest: digest}}
- for _, layer := range []struct{ content, mediaType string }{
- {modelConfig.system, "application/vnd.ollama.image.system"},
- {fmt.Sprintf(`{"num_ctx":%d}`, modelConfig.contextLength), "application/vnd.ollama.image.params"},
- } {
- l, err := manifest.NewLayer(strings.NewReader(layer.content), layer.mediaType)
- if err != nil {
- t.Fatal(err)
- }
- layers = append(layers, l)
- }
- if modelConfig.template != "" {
- l, err := manifest.NewLayer(strings.NewReader(modelConfig.template), "application/vnd.ollama.image.template")
- if err != nil {
- t.Fatal(err)
- }
- layers = append(layers, l)
- }
- if err := manifest.WriteManifest(model.ParseName(modelConfig.name), *configLayer, layers); err != nil {
+ if err := manifest.WriteManifest(model.ParseName(gguf.name), *configLayer, []manifest.Layer{{MediaType: "application/vnd.ollama.image.model", Digest: digest}}); err != nil {
t.Fatal(err)
}
}
@@ -160,33 +164,41 @@ func TestSystemOneHandler(t *testing.T) {
image := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("image"), 16<<10))
const imagePrefix = `{"model":"safetensors-clef","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}},"images":["`
largeText := strings.Replace(prefix+strings.Repeat("x", stateLimit+1)+suffix, "decision-test", "safetensors-clef", 1)
- for _, tt := range []struct {
+ nativeQuestions := func(n int) string {
+ questions := []string{`"refund":{"type":"noul","instructions":"q"}`}
+ for i := 1; i < n; i++ {
+ questions = append(questions, fmt.Sprintf(`"q%d":{"type":"noul","instructions":"q"}`, i))
+ }
+ return `{"model":"laya-native","state":"x","questions":{` + strings.Join(questions, ",") + `}}`
+ }
+ type testCase struct {
name string
body string
err error
status int
calls int
expire bool
- }{
+ }
+ tests := []testCase{
{"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
{"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
{"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},
{"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"Clef videos unsupported", `{"model":"renamed-clef","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul"}}}`, nil, 400, 0, false},
{"candidate videos unsupported", `{"model":"decision-test","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
- {"Clef null state and default instructions", `{"model":"renamed-clef","state":null,"questions":{"refund":{"type":"noul"}}}`, nil, 200, 1, false},
- {"Clef joint head", `{"model":"renamed-clef","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
- {"GGUF success without renderer", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
+ {"Clef null state and default instructions", `{"model":"safetensors-clef","state":null,"questions":{"refund":{"type":"noul"}}}`, nil, 200, 1, false},
+ {"Clef joint head", `{"model":"safetensors-clef","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
+ {"llama.cpp decision model", `{"model":"laya-native","state":{"total":1250.0},"questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
{"Modelfile template", `{"model":"go-template","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
- {"no system prompt", `{"model":"no-system","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},
- {"template failure", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, errors.New("invalid model template"), 500, 0, false},
- {"GGUF capability without Qwen architecture", `{"model":"gguf-other-architecture","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
- {"GGUF missing capability", `{"model":"gguf-undeclared","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"runner validation", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, api.StatusError{StatusCode: 400, ErrorMessage: "prompt too long"}, 400, 1, false},
{"runner failure", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("runner failed"), 500, 1, false},
{"runtime OOM", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("out of memory"), 500, 1, true},
{"invalid schema", `{"model":"decision-test","state":"x","questions":{}}`, nil, 400, 0, false},
- {"invalid Clef schema", `{"model":"renamed-clef","state":"x","questions":{}}`, nil, 400, 0, false},
+ {"invalid Clef schema", `{"model":"safetensors-clef","state":"x","questions":{}}`, nil, 400, 0, false},
+ {"old-format decision GGUF", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
+ {"old-format Clef GGUF", `{"model":"old-clef","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
+ {"llama.cpp 65 questions", nativeQuestions(65), nil, 400, 0, false},
+ {"llama.cpp empty state", `{"model":"laya-native","state":" ","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"omitted model", `{"state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"blank model", `{"model":" ","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"missing model", `{"model":"missing","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 404, 0, false},
@@ -201,27 +213,29 @@ func TestSystemOneHandler(t *testing.T) {
{"invalid image base64", imagePrefix + `!not-base64"]}`, nil, 400, 0, false},
{"image data URL", imagePrefix + `data:image/png;base64,aW1hZ2U="]}`, nil, 400, 0, false},
{"Tev1 safetensors", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
- {"Tev1 GGUF", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},
- {"Tev1 too many candidates", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"score","instructions":"q","criteria":["x"` + strings.Repeat(`,"x"`, 24) + `]}}}`, nil, 400, 0, false},
- {"Tev1 empty description", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"choice","instructions":"q","criteria":{"a":"","b":"B"}}}}`, nil, 400, 0, false},
+ {"Tev1 too many score levels", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"score","instructions":"q","criteria":["x"` + strings.Repeat(`,"x"`, 24) + `]}}}`, nil, 400, 0, false},
+ {"Tev1 empty description", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"choice","instructions":"q","criteria":{"a":"","b":"B"}}}}`, nil, 400, 0, false},
{"safetensors missing capability", `{"model":"safetensors-undeclared","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},
{"bad JSON", `{`, nil, 400, 0, false},
{"at text limit", prefix + strings.Repeat("<", stateLimit) + suffix, nil, 200, 1, false},
{"over text limit", prefix + strings.Repeat("x", stateLimit+1) + suffix, nil, 413, 0, false},
{"image does not relax text limit", strings.TrimSuffix(largeText, "}") + `,"images":["aW1hZ2U="]}`, nil, 413, 0, false},
{"over transport limit", imagePrefix + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},
- } {
+ }
+ for _, model := range []string{"decision-test", "laya-native"} {
+ for _, levels := range []int{1, 2, 10, 11} {
+ status, calls := http.StatusBadRequest, 0
+ if levels >= 2 && levels <= 10 {
+ status, calls = http.StatusOK, 1
+ }
+ criteria := `["x"` + strings.Repeat(`,"x"`, levels-1) + `]`
+ body := fmt.Sprintf(`{"model":%q,"state":"x","questions":{"refund":{"type":"noul","instructions":"q"},"urgency":{"type":"score","instructions":"q","criteria":%s}}}`, model, criteria)
+ tests = append(tests, testCase{fmt.Sprintf("score limits/%s/%d", model, levels), body, nil, status, calls, false})
+ }
+ }
+ for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runner := &systemOneTestRunner{err: tt.err}
- runner.TemplateFn = func(_ context.Context, req llm.ChatRequest) (string, error) {
- if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[0].Content != "Native model scoring instructions." || req.Messages[1].Role != "user" {
- t.Fatalf("model system prompt was not passed to the native template: %+v", req.Messages)
- }
- if req.Think == nil || req.Think.Bool() {
- t.Fatal("scoring must disable thinking")
- }
- return "native:" + req.Messages[1].Content, tt.err
- }
ref := &runnerRef{llama: runner, refCount: 1, sessionDuration: time.Hour}
s := newServerWithMockRunner(t, &runner.mockRunner)
s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {
@@ -247,6 +261,19 @@ func TestSystemOneHandler(t *testing.T) {
if tt.calls == 0 && tt.err == nil && ref.model != nil {
t.Fatal("loaded a runner for a rejected request")
}
+ if strings.HasPrefix(tt.name, "score limits/") && tt.status == http.StatusBadRequest &&
+ !strings.Contains(w.Body.String(), "score criteria must be an array of 2–10 levels") {
+ t.Fatalf("expected shared score validation error: %s", w.Body)
+ }
+ if strings.HasPrefix(tt.name, "old-format ") {
+ var sent decision.Request
+ if err := json.Unmarshal([]byte(tt.body), &sent); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(w.Body.String(), "ollama pull "+sent.Model) {
+ t.Fatalf("old decision GGUFs must ask for a re-download: %s", w.Body)
+ }
+ }
if tt.name == "Clef decision only image" && !strings.Contains(w.Body.String(), "vision") {
t.Fatalf("expected missing vision capability: %s", w.Body)
}
@@ -288,12 +315,18 @@ func TestSystemOneHandler(t *testing.T) {
t.Fatal(err)
}
outputTokens := 2
- if len(runner.request.Fields) > 0 {
+ if len(runner.request.Fields) > 0 || runner.native {
outputTokens = 0
}
if response.Answers["refund"].Noul < 0.88 || response.Usage.InputTokens != 123 || response.Usage.OutputTokens != outputTokens {
t.Fatalf("incorrect scoring response: %s", w.Body)
}
+ if runner.native {
+ if tt.name == "llama.cpp decision model" && (string(runner.state) != `{"total":1250.0}` || !strings.Contains(w.Body.String(), `"model":"laya-native"`)) {
+ t.Fatalf("request did not pass through: state %s, response %s", runner.state, w.Body)
+ }
+ return
+ }
if ref.model.Config.Renderer == "strands" && (len(runner.request.PointerRows) != 1 || len(runner.request.Rows) != 0 || !strings.HasSuffix(runner.request.PointerRows[0].Prompt, "<answer>")) {
t.Fatalf("Strands did not receive pointer inputs: %+v", runner.request)
}
@@ -304,17 +337,10 @@ func TestSystemOneHandler(t *testing.T) {
if ref.model.Config.Renderer == "tev1" && (!strings.Contains(prompt, `"state": "x", "question": "q", "options":`) || strings.Contains(prompt, "Requested field:")) {
t.Fatalf("Tev1 did not receive its per-question prompt: %q", prompt)
}
- wantContext := 1024
- if !ref.model.isGGUF() {
- wantContext = 8192
- }
if len(runner.request.Fields) > 0 {
if len(runner.request.Rows) != 0 || len(runner.request.Fields) != 1 || len(runner.request.Segments) < 3 {
t.Fatal("Clef scoring must preserve the segmented schema")
}
- if ref.model.isGGUF() {
- wantContext = 2048
- }
} else if len(runner.request.PointerRows) > 0 {
if len(runner.request.Rows) != 0 {
t.Fatal("pointer head must bypass chat templates")
@@ -323,11 +349,6 @@ func TestSystemOneHandler(t *testing.T) {
if !strings.HasPrefix(prompt, "custom:system:Model-specific scoring instructions.\nuser:") || !strings.HasSuffix(prompt, "answer:") {
t.Fatalf("scorer did not receive the Modelfile template output: %q", prompt)
}
- } else if ref.model.Config.Renderer == "" {
- wantContext = 4096
- if !strings.HasPrefix(prompt, `native:{"context":`) {
- t.Fatalf("scorer did not receive the native template output: %q", prompt)
- }
} else {
wantPrefix := "<|im_start|>user\n"
if ref.model.System != "" {
@@ -337,8 +358,8 @@ func TestSystemOneHandler(t *testing.T) {
t.Fatalf("scorer did not receive the model's rendered prompt: %q", prompt)
}
}
- if runner.request.MaxTokens != wantContext {
- t.Fatalf("scoring context = %d, want model num_ctx %d", runner.request.MaxTokens, wantContext)
+ if runner.request.MaxTokens != 8192 {
+ t.Fatalf("scoring context = %d, want model num_ctx 8192", runner.request.MaxTokens)
}
}
})