Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
llama: update to b11521 and use native decision support
Jeffrey Morganadc4aae9d68c
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex 156916bbf..f7ea8bf56 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -46,7 +46,6 @@ jobs:'cmake/**/*' \'llama/server/**/*' \'llama/compat/**/*' \- 'llama/clef/**/*' \'LLAMA_CPP_VERSION' \'MLX_VERSION' \'MLX_C_VERSION' \diff --git a/Dockerfile b/Dockerfileindex 118082ba8..e68861205 100644--- a/Dockerfile+++ b/Dockerfile@@ -84,7 +84,6 @@ FROM cpu-deps AS llama-server-cpuCOPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset cpu \@@ -105,7 +104,6 @@ FROM cuda-12-deps AS llama-server-cuda_v12COPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset llama_cuda_v12_linux \@@ -119,7 +117,6 @@ FROM cuda-13-deps AS llama-server-cuda_v13COPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset llama_cuda_v13_linux \@@ -134,7 +131,6 @@ ENV CC=clang CXX=clang++ CXXFLAGS=--gcc-toolchain=/opt/rh/gcc-toolset-13/root/usCOPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset rocm_v7_2_linux \@@ -149,7 +145,6 @@ FROM vulkan-deps AS llama-server-vulkanCOPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset vulkan \@@ -175,7 +170,6 @@ ENV CMAKE_GENERATOR=NinjaCOPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset llama_cuda_jetpack5 \@@ -197,7 +191,6 @@ ENV CMAKE_GENERATOR=NinjaCOPY LLAMA_CPP_VERSION .COPY llama/server llama/serverCOPY llama/compat llama/compat-COPY llama/clef llama/clefCOPY cmake cmakeRUN --mount=type=cache,target=/root/.ccache \cmake -S llama/server --preset llama_cuda_jetpack6 \diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSIONindex a00ca832a..cf976c99f 100644--- a/LLAMA_CPP_VERSION+++ b/LLAMA_CPP_VERSION@@ -1 +1 @@-b11351+b11521diff --git a/docs/openapi.yaml b/docs/openapi.yamlindex b6bf8b777..6f2c2fe26 100644--- a/docs/openapi.yaml+++ b/docs/openapi.yaml@@ -899,12 +899,12 @@ components:model:type: stringpattern: '\S'- description: Local model trained for System One, such as nimble. Requires compatible GGUF weights and a scoring-capable runner; cloud and MLX/Safetensors models are not supported.+ description: Local model trained for System One, such as nimble.state:$ref: "#/components/schemas/SystemOneContent"images:type: array- description: Base64-encoded images shared by all questions, in request order. Requires Clef or Clef Flash with vision weights. URLs and data URLs are not supported.+ description: Base64-encoded images shared by all questions, in request order. Requires a model with vision, such as Clef. URLs and data URLs are not supported.items:type: stringformat: bytediff --git a/llama/clef/clef.cpp b/llama/clef/clef.cppdeleted file mode 100644index ef3b251c9..000000000--- a/llama/clef/clef.cpp+++ /dev/null@@ -1,388 +0,0 @@-// Clef joint schema head, ported from Cloudflare's Apache-2.0 reference.-// https://huggingface.co/Cloudflare/clef/blob/main/joint_schema_model.py-#include "clef.h"-#include "ggml-backend.h"-#include "ggml.h"-#include "gguf.h"-#include <algorithm>-#include <cmath>-#include <cstdio>-#include <cstring>-#include <map>-#include <stdexcept>--#if !defined(_WIN32)-#include <sys/types.h>-#endif--namespace {-struct matrix {- int rows = 0, cols = 0;- std::vector<float> data;- matrix() = default;- matrix(int r, int c) : rows(r), cols(c), data(size_t(r) * c) {}- float &at(int r, int c) { return data[size_t(r) * cols + c]; }- float at(int r, int c) const { return data[size_t(r) * cols + c]; }-};-void require(bool condition, const char *message) {- if (!condition)- throw std::runtime_error(message);-}-// Use the backend registry so builds with dynamically selected CPU variants-// work without linking to a particular ggml-cpu library.-struct cpu_backend {- ggml_backend_t handle = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);- cpu_backend() {- require(handle != nullptr, "Clef: CPU backend unavailable");- auto reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(handle));- auto set_threads = reinterpret_cast<ggml_backend_set_n_threads_t>(- ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"));- if (set_threads)- set_threads(handle, 8);- }- ~cpu_backend() { ggml_backend_free(handle); }-};-struct graph {- ggml_context *ctx;- explicit graph(size_t bytes) {- ctx = ggml_init({bytes + (2 << 20), nullptr, true});- require(ctx != nullptr, "Clef: cannot allocate graph");- }- ~graph() { ggml_free(ctx); }- ggml_tensor *input(const matrix &m) {- auto t = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, m.cols, m.rows);- t->data = const_cast<float *>(m.data.data());- return t;- }- matrix run(ggml_tensor *output, int rows, int cols) {- auto gf = ggml_new_graph(ctx);- ggml_build_forward_expand(gf, output);- static thread_local cpu_backend cpu;- require(ggml_backend_graph_compute(cpu.handle, gf) == GGML_STATUS_SUCCESS,- "Clef: graph evaluation failed");- matrix result(rows, cols);- memcpy(result.data.data(), output->data, result.data.size() * sizeof(float));- return result;- }-};-matrix linear(const matrix &x, const matrix &w) {- require(x.cols == w.cols, "Clef: linear shape mismatch");- graph g(size_t(x.rows) * w.rows * sizeof(float));- auto tx = g.input(x), tw = g.input(w);- ggml_set_no_alloc(g.ctx, false);- return g.run(ggml_mul_mat(g.ctx, tw, tx), x.rows, w.rows);-}-matrix add(matrix a, const matrix &b) {- require(a.cols == b.cols && (b.rows == 1 || a.rows == b.rows), "Clef: addition shape mismatch");- for (int r = 0; r < a.rows; ++r)- for (int c = 0; c < a.cols; ++c)- a.at(r, c) += b.at(b.rows == 1 ? 0 : r, c);- return a;-}-matrix slice(const matrix &m, int start, int end) {- require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span");- matrix out(end - start, m.cols);- std::copy(m.data.begin() + size_t(start) * m.cols, m.data.begin() + size_t(end) * m.cols,- out.data.begin());- return out;-}-matrix mean(const matrix &m, int start, int end) {- require(start >= 0 && start < end && end <= m.rows, "Clef: invalid span");- matrix out(1, m.cols);- for (int r = start; r < end; ++r)- for (int c = 0; c < m.cols; ++c)- out.at(0, c) += m.at(r, c) / (end - start);- return out;-}-void append(matrix &a, const matrix &b) {- require(a.rows == 0 || a.cols == b.cols, "Clef: append shape mismatch");- a.cols = b.cols;- a.rows += b.rows;- a.data.insert(a.data.end(), b.data.begin(), b.data.end());-}-float cosine(const matrix &a, int ar, const matrix &b, int br, double eps = 1e-8) {- require(a.cols == b.cols, "Clef: cosine shape mismatch");- double dot = 0, aa = 0, bb = 0;- for (int c = 0; c < a.cols; ++c) {- dot += double(a.at(ar, c)) * b.at(br, c);- aa += double(a.at(ar, c)) * a.at(ar, c);- bb += double(b.at(br, c)) * b.at(br, c);- }- return dot / (std::max(std::sqrt(aa), eps) * std::max(std::sqrt(bb), eps));-}-} // namespace--struct clef_head::impl {- FILE *file = nullptr;- gguf_context *meta = nullptr;- ggml_context *tensors = nullptr;- std::map<std::string, matrix> weights;- int hidden, width, heads, routing_layers, layers;- int output_index;- ggml_tensor *output;-- explicit impl(const std::string &path) : file(ggml_fopen(path.c_str(), "rb")) {- try {- meta = gguf_init_from_file(path.c_str(), {true, &tensors});- require(meta && tensors && file, "Clef: cannot open model");- const int arch_key = gguf_find_key(meta, "general.architecture");- require(arch_key >= 0 && gguf_get_kv_type(meta, arch_key) == GGUF_TYPE_STRING,- "Clef: missing architecture");- const std::string prefix = std::string(gguf_get_val_str(meta, arch_key)) + ".decision.";- auto integer = [&](const std::string &key) {- int i = gguf_find_key(meta, (prefix + key).c_str());- require(i >= 0 && gguf_get_kv_type(meta, i) == GGUF_TYPE_UINT32,- "Clef: missing head configuration");- return int(gguf_get_val_u32(meta, i));- };- hidden = integer("hidden_size");- width = integer("width");- heads = integer("heads");- routing_layers = integer("routing_layers");- layers = integer("layers");- require(hidden > 0 && hidden <= 16384 && width > 0 && width <= 4096 && heads > 0 &&- width % heads == 0 && routing_layers >= 0 && routing_layers <= 32 && layers >= 0 &&- layers <= 32,- "Clef: invalid head configuration");- for (int i = 0; i < gguf_get_n_tensors(meta); ++i) {- std::string name = gguf_get_tensor_name(meta, i);- if (name.rfind("clef.", 0) != 0)- continue;- auto t = ggml_get_tensor(tensors, name.c_str());- require(t->ne[2] == 1 && t->ne[3] == 1 && t->type == GGML_TYPE_F32,- "Clef: head tensors must be float32 matrices");- matrix m(t->ne[1], t->ne[0]);- read(i, 0, m.data.data(), m.data.size() * sizeof(float));- weights.emplace(name.substr(5), std::move(m));- }- output_index = gguf_find_tensor(meta, "output.weight");- require(output_index >= 0, "Clef: output embeddings are missing");- output = ggml_get_tensor(tensors, "output.weight");- require(output->ne[0] == hidden, "Clef: output embedding shape mismatch");- } catch (...) {- if (tensors)- ggml_free(tensors);- if (meta)- gguf_free(meta);- if (file)- std::fclose(file);- throw;- }- }- ~impl() {- ggml_free(tensors);- gguf_free(meta);- std::fclose(file);- }- void read(int index, size_t offset, void *data, size_t bytes) {- // The head can sit past 2 GiB, where iostream seeks truncate on some- // Windows C++ runtimes, so seek with an explicit 64-bit offset.- const size_t position = gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, index) + offset;-#if defined(_WIN32)- const int seek_rc = _fseeki64(file, static_cast<__int64>(position), SEEK_SET);-#else- const int seek_rc = fseeko(file, static_cast<off_t>(position), SEEK_SET);-#endif- require(seek_rc == 0 && std::fread(data, 1, bytes, file) == bytes, "Clef: truncated tensor data");- }- const matrix &w(const std::string &key) { return weights.at(key); }- matrix norm(const matrix &x, const std::string &key) {- const auto &weight = w(key + ".weight"), &bias = w(key + ".bias");- require(weight.data.size() == size_t(x.cols) && bias.data.size() == size_t(x.cols),- "Clef: layer norm shape mismatch");- matrix out = x;- for (int r = 0; r < x.rows; ++r) {- double m = 0, v = 0;- for (int c = 0; c < x.cols; ++c)- m += x.at(r, c) / x.cols;- for (int c = 0; c < x.cols; ++c)- v += (x.at(r, c) - m) * (x.at(r, c) - m) / x.cols;- for (int c = 0; c < x.cols; ++c)- out.at(r, c) = (x.at(r, c) - m) / std::sqrt(v + 1e-5) * weight.data[c] + bias.data[c];- }- return out;- }- matrix project(const matrix &x, const std::string &key, bool bias = false) {- matrix out = linear(x, w(key + ".weight"));- return bias ? add(std::move(out), w(key + ".bias")) : out;- }- matrix feedforward(matrix x, const std::string &first, const std::string &last) {- x = project(x, first, true);- for (auto &v : x.data)- v = 0.5f * v * (1.0f + std::erf(v / std::sqrt(2.0f)));- return project(x, last, true);- }- matrix attention(const matrix &query, const matrix &memory, const std::string &key) {- const auto &weight = w(key + ".in_proj_weight"), &bias = w(key + ".in_proj_bias");- require(weight.rows == 3 * width && weight.cols == width && bias.data.size() == size_t(3 * width),- "Clef: attention shape mismatch");- auto projection = [&](const matrix &x, int n) {- auto y = linear(x, slice(weight, n * width, (n + 1) * width));- for (int r = 0; r < y.rows; ++r)- for (int c = 0; c < width; ++c)- y.at(r, c) += bias.data[n * width + c];- return y;- };- auto q = projection(query, 0), k = projection(memory, 1), v = projection(memory, 2);- const int d = width / heads, nq = q.rows, nk = k.rows;- graph g((size_t(nq) * nk * heads * 3 + size_t(nq + nk) * width * 8) * sizeof(float));- auto tq = g.input(q), tk = g.input(k), tv = g.input(v);- ggml_set_no_alloc(g.ctx, false);- auto split = [&](ggml_tensor *t, int n) {- return ggml_cont(g.ctx, ggml_permute(g.ctx, ggml_reshape_3d(g.ctx, t, d, heads, n), 0, 2, 1, 3));- };- tq = split(tq, nq);- tk = split(tk, nk);- tv = split(tv, nk);- auto scores =- ggml_soft_max(g.ctx, ggml_scale(g.ctx, ggml_mul_mat(g.ctx, tk, tq), 1.0f / std::sqrt(float(d))));- tv = ggml_cont(g.ctx, ggml_transpose(g.ctx, tv));- auto out = ggml_mul_mat(g.ctx, tv, scores);- out = ggml_cont(g.ctx, ggml_permute(g.ctx, out, 0, 2, 1, 3));- auto result = g.run(ggml_reshape_2d(g.ctx, out, width, nq), nq, width);- return project(result, key + ".out_proj", true);- }- matrix lexical(const std::vector<int32_t> &tokens, int start, int end) {- require(start >= 0 && start < end && end <= int(tokens.size()), "Clef: invalid lexical span");- matrix result(1, hidden);- const auto traits = ggml_get_type_traits(output->type);- require(traits->to_float != nullptr || output->type == GGML_TYPE_F32,- "Clef: unsupported output embedding type");- size_t bytes = ggml_row_size(output->type, hidden);- std::vector<uint8_t> raw(bytes);- std::vector<float> row(hidden);- for (int i = start; i < end; ++i) {- require(tokens[i] >= 0 && tokens[i] < output->ne[1], "Clef: invalid token");- read(output_index, size_t(tokens[i]) * bytes, raw.data(), bytes);- if (output->type == GGML_TYPE_F32)- memcpy(row.data(), raw.data(), bytes);- else- traits->to_float(raw.data(), row.data(), hidden);- for (int c = 0; c < hidden; ++c)- result.at(0, c) += row[c] / (end - start);- }- return result;- }- std::vector<std::vector<float>> score(const std::vector<std::vector<float>> &states,- const std::vector<int32_t> &tokens, const common_json &fields) {- require(states.size() == tokens.size() && !states.empty(), "Clef: missing backbone hidden states");- require(fields.is_array() && !fields.empty() && fields.size() <= 64, "Clef: expected 1-64 fields");- matrix x;- for (const auto &row : states) {- require(row.size() == size_t(hidden), "Clef: hidden size mismatch");- matrix m(1, hidden);- m.data = row;- append(x, m);- }- x = norm(x, "hidden_norm");- auto memory = project(x, "memory_projection"), global = slice(x, x.rows - 1, x.rows);- matrix questions, lexical_options, queries;- std::vector<int> counts, types;- auto span = [](const common_json &value) {- require(value.is_array() && value.size() == 2 && value[0].is_number_integer() &&- value[1].is_number_integer(),- "Clef: malformed span");- return std::pair<int, int>{value[0].get<int>(), value[1].get<int>()};- };- for (const auto &field : fields) {- int type = field.at("type").get<int>();- require(type >= 0 && type < 3, "Clef: invalid question type");- types.push_back(type);- auto [qs, qe] = span(field.at("question"));- auto question = mean(x, qs, qe);- append(questions, question);- const auto &options = field.at("options");- require(options.is_array() && options.size() >= 2 && options.size() <= 26,- "Clef: expected 2-26 options");- counts.push_back(options.size());- matrix contexts, lex;- for (const auto &option : options) {- auto [s, e] = span(option);- append(contexts, mean(x, s, e));- append(lex, lexical(tokens, s, e));- }- append(lexical_options, lex);- append(queries, add(add(project(contexts, "option_context_projection"),- project(lex, "option_lexical_projection")),- project(question, "option_question_projection")));- }- for (int i = 0; i < routing_layers; ++i) {- auto key = "evidence_layers." + std::to_string(i);- queries = add(queries, attention(norm(queries, key + ".query_norm"),- norm(memory, key + ".memory_norm"), key + ".attention"));- queries = add(queries, feedforward(norm(queries, key + ".feedforward_norm"),- key + ".feedforward.0", key + ".feedforward.3"));- }- auto base = project(questions, "question_projection");- matrix summaries;- int offset = 0;- for (int f = 0; f < base.rows; ++f) {- std::vector<float> scores(counts[f]);- for (int j = 0; j < counts[f]; ++j)- for (int c = 0; c < width; ++c)- scores[j] += queries.at(offset + j, c) * base.at(f, c) / std::sqrt(float(width));- float peak = *std::max_element(scores.begin(), scores.end()), sum = 0;- for (auto &v : scores) {- v = std::exp(v - peak);- sum += v;- }- matrix summary(1, width);- for (int j = 0; j < counts[f]; ++j)- for (int c = 0; c < width; ++c)- summary.at(0, c) += scores[j] / sum * queries.at(offset + j, c);- append(summaries, summary);- offset += counts[f];- }- auto fs =- add(add(base, norm(summaries, "option_summary_norm")), project(global, "global_projection"));- const auto &te = w("type_embedding.weight");- require(te.rows == 3 && te.cols == width, "Clef: type embedding shape mismatch");- for (int f = 0; f < fs.rows; ++f)- for (int c = 0; c < width; ++c)- fs.at(f, c) += te.at(types[f], c);- for (int i = 0; i < layers; ++i) {- auto key = "layers." + std::to_string(i);- auto normalized = norm(fs, key + ".norm1");- fs = add(fs, attention(normalized, normalized, key + ".self_attn"));- fs = add(fs, attention(norm(fs, key + ".norm2"), memory, key + ".multihead_attn"));- fs = add(fs, feedforward(norm(fs, key + ".norm3"), key + ".linear1", key + ".linear2"));- }- fs = norm(fs, "field_norm");- auto options = norm(queries, "option_norm");- float prior_scale = std::exp(std::min(w("prior_logit_scale").data.at(0), std::log(100.0f)));- float joint_scale = std::exp(std::min(w("joint_logit_scale").data.at(0), std::log(100.0f)));- float gate = 1.0f / (1.0f + std::exp(-w("residual_gate").data.at(0)));- std::vector<std::vector<float>> result;- offset = 0;- for (int f = 0; f < fs.rows; ++f) {- matrix features(counts[f], 4 * width);- for (int j = 0; j < counts[f]; ++j)- for (int c = 0; c < width; ++c) {- float a = fs.at(f, c), b = options.at(offset + j, c);- features.at(j, c) = a;- features.at(j, width + c) = b;- features.at(j, 2 * width + c) = a * b;- features.at(j, 3 * width + c) = std::abs(a - b);- }- auto residual = feedforward(features, "residual_scorer.0", "residual_scorer.3");- auto anchor = add(slice(questions, f, f + 1), global);- std::vector<float> logits;- for (int j = 0; j < counts[f]; ++j) {- float v = prior_scale * cosine(lexical_options, offset + j, anchor, 0, 1e-12) +- gate * (joint_scale * cosine(fs, f, options, offset + j) + residual.at(j, 0));- require(std::isfinite(v), "Clef: non-finite logit");- logits.push_back(v);- }- result.push_back(std::move(logits));- offset += counts[f];- }- return result;- }-};-clef_head::clef_head(const std::string &path) : p(new impl(path)) {}-clef_head::~clef_head() = default;-std::vector<std::vector<float>> clef_head::score(const std::vector<std::vector<float>> &hidden,- const std::vector<int32_t> &tokens,- const common_json &fields) {- return p->score(hidden, tokens, fields);-}diff --git a/llama/clef/clef.h b/llama/clef/clef.hdeleted file mode 100644index ea2607d32..000000000--- a/llama/clef/clef.h+++ /dev/null@@ -1,20 +0,0 @@-#pragma once-#include "json.h"-#include <cstdint>-#include <memory>-#include <string>-#include <vector>--// The backbone is evaluated by llama-server. This head consumes every final-// hidden state and the token spans produced by Cloudflare's record format.-class clef_head {- public:- explicit clef_head(const std::string &path);- ~clef_head();- std::vector<std::vector<float>> score(const std::vector<std::vector<float>> &hidden,- const std::vector<int32_t> &tokens, const common_json &fields);-- private:- struct impl;- std::unique_ptr<impl> p;-};diff --git a/llama/clef/convert.py b/llama/clef/convert.pydeleted file mode 100644index ac18c32b2..000000000--- a/llama/clef/convert.py+++ /dev/null@@ -1,47 +0,0 @@-#!/usr/bin/env python3-"""Convert Cloudflare Clef with its trained joint head, using pinned llama.cpp.--Usage: python convert.py --llama-cpp build/llama-src MODEL --outtype q8_0 --outfile MODEL.gguf-For the vision projector, add --mmproj --outtype f16 and use a separate output file.-All remaining arguments are passed to convert_hf_to_gguf.py.-"""-import argparse-import json-from pathlib import Path-import sys--parser = argparse.ArgumentParser(add_help=False)-parser.add_argument('--llama-cpp', type=Path, required=True)-args, remaining = parser.parse_known_args()-sys.path[:0] = [str(args.llama_cpp.resolve()), str((args.llama_cpp / 'gguf-py').resolve())]--import numpy as np-from safetensors import safe_open-from conversion.qwen import Qwen3_5TextModel-import convert_hf_to_gguf--original_tensors = Qwen3_5TextModel.prepare_tensors-original_metadata = Qwen3_5TextModel.set_gguf_parameters---def prepare_tensors(self):- original_tensors(self)- with safe_open(self.dir_model / 'joint_head.safetensors', framework='pt', device='cpu') as head:- for name in head.keys():- # Keep the small head in float32, including its scalar gates.- data = head.get_tensor(name).float().numpy()- self.gguf_writer.add_tensor('clef.' + name, np.atleast_1d(data))---def set_gguf_parameters(self):- original_metadata(self)- config = json.loads((self.dir_model / 'joint_head_config.json').read_text())- self.gguf_writer.add_string(f'{self.gguf_writer.arch}.decision.type', 'clef')- for name, value in config.items():- self.gguf_writer.add_uint32(f'{self.gguf_writer.arch}.decision.{name}', value)---Qwen3_5TextModel.prepare_tensors = prepare_tensors-Qwen3_5TextModel.set_gguf_parameters = set_gguf_parameters-sys.argv = [sys.argv[0], '--no-mtp', *remaining]-convert_hf_to_gguf.main()diff --git a/llama/compat/002-clef.patch b/llama/compat/002-clef.patchdeleted file mode 100644index 9614e0fb5..000000000--- a/llama/compat/002-clef.patch+++ /dev/null@@ -1,321 +0,0 @@-diff --git a/common/common.cpp b/common/common.cpp-index 7ebfa2610..695a1bfff 100644---- a/common/common.cpp-+++ b/common/common.cpp-@@ -1192,6 +1192,18 @@ struct common_init_result::impl {- std::vector<llama_sampler_seq_config> samplers_seq_config;- };--+common_decision_type common_get_decision_type(const llama_model * model) {-+ char value[128];-+ if (llama_model_meta_val_str(model, "general.architecture", value, sizeof(value)) < 0) {-+ return COMMON_DECISION_TYPE_NONE;-+ }-+ const std::string key = std::string(value) + ".decision.type";-+ if (llama_model_meta_val_str(model, key.c_str(), value, sizeof(value)) < 0) {-+ return COMMON_DECISION_TYPE_NONE;-+ }-+ return std::strcmp(value, "clef") == 0 ? COMMON_DECISION_TYPE_CLEF : COMMON_DECISION_TYPE_UNKNOWN;-+}-+- common_init_result::common_init_result(common_params & params, bool model_only) :- pimpl(new impl{}) {- auto mparams = common_model_params_to_llama(params);-@@ -1244,6 +1256,18 @@ common_init_result::common_init_result(common_params & params, bool model_only)-- const llama_vocab * vocab = llama_model_get_vocab(model);--+ if (common_get_decision_type(model) == COMMON_DECISION_TYPE_CLEF) {-+ params.embedding = cparams.embeddings = true;-+ cparams.ctx_type = LLAMA_CONTEXT_TYPE_DECISION;-+ params.pooling_type = cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;-+ // The joint head consumes every hidden state; retain the whole input.-+ const auto n_ctx = cparams.n_ctx ? cparams.n_ctx : llama_model_n_ctx_train(model);-+ params.n_batch = cparams.n_batch = n_ctx;-+ params.n_ubatch = cparams.n_ubatch = n_ctx;-+ params.n_outputs_max = cparams.n_outputs_max = n_ctx;-+ params.n_outputs_max_per_seq = cparams.n_outputs_max_per_seq = 1;-+ }-+- // load and optionally apply lora adapters- for (auto & la : params.lora_adapters) {- llama_adapter_lora_ptr lora;-diff --git a/common/common.h b/common/common.h-index fd90c3b95..641f2dc6f 100644---- a/common/common.h-+++ b/common/common.h-@@ -953,6 +953,14 @@ bool tty_can_use_colors();-- struct common_sampler;--+enum common_decision_type {-+ COMMON_DECISION_TYPE_NONE,-+ COMMON_DECISION_TYPE_CLEF,-+ COMMON_DECISION_TYPE_UNKNOWN,-+};-+-+common_decision_type common_get_decision_type(const struct llama_model * model);-+- // note: defines the model, context, samplers, ets. lifetimes- struct common_init_result {- common_init_result(common_params & params, bool model_only = false);-diff --git a/include/llama.h b/include/llama.h-index dea4d4606..720b45140 100644---- a/include/llama.h-+++ b/include/llama.h-@@ -224,6 +224,7 @@ extern "C" {- enum llama_context_type {- LLAMA_CONTEXT_TYPE_DEFAULT = 0,- LLAMA_CONTEXT_TYPE_MTP = 1,-+ LLAMA_CONTEXT_TYPE_DECISION = 2, // hidden states without vocabulary logits- };-- // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)-diff --git a/src/llama-context.cpp b/src/llama-context.cpp-index 4f04db0ee..fa115ffe4 100644---- a/src/llama-context.cpp-+++ b/src/llama-context.cpp-@@ -27,6 +27,7 @@-- static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {- switch (ctx_type) {-+ case LLAMA_CONTEXT_TYPE_DECISION:- case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;- case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP;- }-@@ -2124,7 +2125,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) {- const auto n_embd = hparams.n_embd;- const auto n_embd_out = hparams.n_embd_out();--- bool has_logits = true;-+ bool has_logits = cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION;- bool has_embd = cparams.embeddings;- bool has_embd_nextn = cparams.embeddings_nextn;--diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp-index a1e263500..751c8a72e 100644---- a/src/models/qwen35.cpp-+++ b/src/models/qwen35.cpp-@@ -215,11 +215,12 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para- cb(cur, "result_norm", -1);- res->t_embd = cur;--- // LM head-- cur = build_lora_mm(model.output, cur, model.output_s);---- cb(cur, "result_output", -1);-- res->t_logits = cur;-+ // Decision heads consume the hidden states, not vocabulary logits.-+ if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_DECISION) {-+ cur = build_lora_mm(model.output, cur, model.output_s);-+ cb(cur, "result_output", -1);-+ res->t_logits = cur;-+ }-- ggml_build_forward_expand(gf, cur);- }-diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp-index 470fbd977..e2ac37eb7 100644---- a/tools/server/server-context.cpp-+++ b/tools/server/server-context.cpp-@@ -1,4 +1,5 @@- #include "server-context.h"-+#include "clef.h"- #include "server-chat.h"- #include "server-common.h"- #include "server-http.h"-@@ -206,6 +207,7 @@ struct server_slot {- // multimodal- mtmd_context * mctx = nullptr;- mtmd::batch_ptr mbatch = nullptr;-+ std::vector<std::vector<float>> score_embeddings;-- // speculative decoding- common_speculative * spec;-@@ -329,6 +331,7 @@ struct server_slot {- SLT_DBG(*this, "%s", "\n");-- spec_is_replay = false;-+ score_embeddings.clear();-- last_nl_pos = 0;- generated_text = "";-@@ -389,6 +392,18 @@ struct server_slot {- (ggml_time_us() - t_start) / 1000.0, n_text, (int) prompt.tokens.size());- }--+ void collect_score_embeddings(const common_batch & batch) {-+ if (!task || task->score_fields.is_null()) return;-+ // Embedding mode returns every token, including image tokens with logits disabled.-+ const int width = llama_model_n_embd_out(llama_get_model(ctx_tgt));-+ for (int32_t i = 0; i < batch.size(); ++i) {-+ if (batch.tokens[i].seq_id != id) continue;-+ const float * row = llama_get_embeddings_ith(ctx_tgt, i);-+ if (!row) throw std::runtime_error("decision model did not return hidden states");-+ score_embeddings.emplace_back(row, row + width);-+ }-+ }-+- bool need_embd() const {- GGML_ASSERT(task);- return task->need_embd();-@@ -707,9 +722,7 @@ struct server_slot {- // returns 0 on success- // caller need to update prompt.tokens after a successful call to keep track of the processing progress- // note: this is not a member of server_slot because we want to run it inside yield_to_queue--// slot is passed as const to avoid accidental modification of the slot state--// some pointers are allowed to be used, they are not used by to_json()--static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) {-+static int process_mtmd_chunk(server_slot & slot, mtmd::batch_ptr & mbatch, size_t idx, size_t & n_tokens_out) {- GGML_ASSERT(slot.mctx);- const auto & mctx = slot.mctx;- const auto & input_tokens = slot.task->tokens;-@@ -721,9 +734,9 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch- float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get());- if (embd) {- struct cb_data_t {-- common_speculative * spec;-+ server_slot * slot;- llama_context * ctx;-- } cb_data = { slot.spec, slot.ctx_tgt };-+ } cb_data = { &slot, slot.ctx_tgt };-- static auto cb = [](const mtmd_helper_embd_batch * b, void * user_data) {- const auto * data = static_cast<cb_data_t *>(user_data);-@@ -737,7 +750,9 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch- batch.add_embd({ b->embd + (size_t) i * b->n_embd, 1, (size_t) b->n_embd }, pos, b->seq_id, false);- }--- return common_speculative_process(data->spec, batch) ? 0 : 1;-+ if (!common_speculative_process(data->slot->spec, batch)) return 1;-+ data->slot->collect_score_embeddings(batch);-+ return 0;- };-- llama_pos new_n_past; // unused for now-@@ -819,6 +834,7 @@ public:- // - when not in sleeping state- // - and, with thread-safe APIs (e.g., tokenizer calls)- llama_model * model_tgt = nullptr;-+ std::unique_ptr<clef_head> clef;-- mtmd_context * mctx = nullptr;- // note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct-@@ -2156,6 +2172,20 @@ private:- res->n_tokens = slot.task->n_tokens();- res->res_type = slot.task->params.res_type;--+ if (!slot.task->score_fields.is_null()) {-+ try {-+ if (!clef) clef.reset(new clef_head(params_base.model.path));-+ llama_tokens tokens;-+ for (size_t i = 0; i < slot.task->tokens.size(); ++i) tokens.push_back(slot.task->tokens[i]);-+ res->score_logits = clef->score(slot.score_embeddings, tokens, slot.task->score_fields);-+ } catch (const std::exception & e) {-+ send_error(slot, e.what(), ERROR_TYPE_SERVER);-+ return;-+ }-+ queue_results.send(std::move(res));-+ return;-+ }-+- const int n_embd_out = llama_model_n_embd_out(model_tgt);-- std::vector<float> embd_res(n_embd_out, 0.0f);-@@ -3803,6 +3833,7 @@ private:- };-- iterate(slots, [&](server_slot & slot) {-+ slot.collect_score_embeddings(batch.view);- // optionally send prompt processing progress- if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) {- if (slot.task->params.stream && slot.task->params.return_progress) {-@@ -4263,6 +4294,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(- GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL);-- auto res = create_response();-+ if (common_get_decision_type(ctx_server.model_tgt) == COMMON_DECISION_TYPE_CLEF) {-+ res->error(format_error_response("This model requires a decision request", ERROR_TYPE_INVALID_REQUEST));-+ return res;-+ }- auto completion_id = gen_chatcmplid();- auto & rd = res->rd;- auto & params = this->params;-@@ -5475,6 +5510,43 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons- // OAI-compat- task.params.res_type = res_type;- task.params.embd_normalize = embd_normalize;-+ if (body.contains("score_fields")) {-+ task.score_fields = body.at("score_fields");-+ task.params.cache_prompt = false; // the head needs every hidden state-+ if (body.contains("images") && !body.at("images").empty()) {-+ if (!ctx_server.mctx) throw std::invalid_argument("model does not support images");-+ const auto & text = task.tokens.get_tokens();-+ const int position = body.at("image_position").get<int>();-+ if (position < 0 || size_t(position) > text.size()) throw std::invalid_argument("invalid image position");-+ std::string markers;-+ for (size_t j = 0; j < body.at("images").size(); ++j) markers += mtmd_get_marker(ctx_server.mctx);-+ auto media_inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx,-+ json{{"prompt_string", markers + "\n"}, {"multimodal_data", body.at("images")}}, true, true, ctx_server.init_opt);-+ auto & media = media_inputs[0];-+ if (text.size() + media.size() > size_t(meta->slot_n_ctx)) throw std::invalid_argument("decision prompt exceeds the model context (input is never truncated)");-+ server_tokens input(llama_tokens(text.begin(), text.begin() + position), true);-+ for (size_t j = 0; j < media.size();) {-+ if (media[j] == LLAMA_TOKEN_NULL) {-+ const auto & chunk = media.find_chunk(j);-+ input.push_back(chunk.get());-+ j += mtmd_input_chunk_get_n_tokens(chunk.get());-+ } else {-+ input.push_back(media[j++]);-+ }-+ }-+ input.insert(llama_tokens(text.begin() + position, text.end()));-+ for (auto & field : task.score_fields) {-+ auto shift = [&](json & span) {-+ const int start = span.at(0).get<int>(), end = span.at(1).get<int>();-+ if (start < position && end > position) throw std::invalid_argument("image splits a scoring span");-+ if (start >= position) span = json::array({start + media.size(), end + media.size()});-+ };-+ shift(field.at("question"));-+ for (auto & option : field.at("options")) shift(option);-+ }-+ task.tokens = std::move(input);-+ }-+ }-- tasks.push_back(std::move(task));- }-diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp-index 0d3beb313..f5f3f2e18 100644---- a/tools/server/server-task.cpp-+++ b/tools/server/server-task.cpp-@@ -1464,6 +1464,9 @@ json server_task_result_cmpl_partial::to_json_anthropic() {- // server_task_result_embd- //- json server_task_result_embd::to_json() {-+ if (!score_logits.is_null()) {-+ return json {{"logits", score_logits}, {"tokens_evaluated", n_tokens}};-+ }- return res_type == TASK_RESPONSE_TYPE_OAI_EMBD- ? to_json_oaicompat()- : to_json_non_oaicompat();-diff --git a/tools/server/server-task.h b/tools/server/server-task.h-index 9c99143f8..94c724ed1 100644---- a/tools/server/server-task.h-+++ b/tools/server/server-task.h-@@ -152,6 +152,7 @@ struct server_task {- // used by SERVER_TASK_TYPE_INFERENCE- task_params params;- server_tokens tokens;-+ common_json score_fields; // optional decision question and option spans-- // only used by CLI, this allow tokenizing CLI inputs on server side- // we need this because mtmd_context and vocab are not accessible outside of server_context-@@ -452,6 +453,7 @@ struct server_task_result_cmpl_partial : server_task_result {- };-- struct server_task_result_embd : server_task_result {-+ common_json score_logits;- std::vector<std::vector<float>> embedding;-- int32_t n_tokens;diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cppindex 848b84fa0..3e1cbddd4 100644--- a/llama/compat/llama-ollama-compat.cpp+++ b/llama/compat/llama-ollama-compat.cpp@@ -3242,12 +3242,6 @@ bool translate_metadata(const llama_model_loader * ml,std::lock_guard<std::mutex> lk(g_loader_path_mutex);g_loader_paths[ml] = fname ? fname : "";}- // Clef's joint head is consumed by the server after backbone evaluation.- const int decision_key = gguf_find_key(meta, (arch_name + ".decision.type").c_str());- if (decision_key >= 0 && gguf_get_kv_type(meta, decision_key) == GGUF_TYPE_STRING &&- std::strcmp(gguf_get_val_str(meta, decision_key), "clef") == 0) {- add_skip_prefix(ml, "clef.");- }// embeddinggemma must run before gemma3: it switches arch_name to// "gemma-embedding", which is what later checks (and the loader's KV// prefix) need to see.diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txtindex aabc1a28d..9153c9bf0 100644--- a/llama/server/CMakeLists.txt+++ b/llama/server/CMakeLists.txt@@ -186,8 +186,6 @@ FetchContent_Declare(${_ollama_compat_patch_cmd})FetchContent_MakeAvailable(llama_cpp)-target_sources(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef/clef.cpp)-target_include_directories(server-context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../clef)if(NOT COMMAND license_add_file)message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update")diff --git a/llm/llama_server.go b/llm/llama_server.goindex c7cd91518..746d389f7 100644--- a/llm/llama_server.go+++ b/llm/llama_server.go@@ -105,6 +105,38 @@ func WithDefaultEmbeddingNumBatch(opts api.Options) api.Options {return opts}+// LlamaServerBatch resolves the batch size and extra memory allowance used+// when deciding whether to load a model alongside other models.+func LlamaServerBatch(f *gguf.Model, opts api.Options, completion bool) (int, uint64) {+ batch := opts.NumBatch+ // These decision heads read the whole prompt without a KV cache, so the+ // physical batch must hold the entire effective context.+ wholePrompt := f.KV().Uint("decision.block_count") > 0+ if wholePrompt {+ batch = opts.NumCtx+ if trainCtx := int(f.KV().ContextLength()); trainCtx > 0 {+ batch = min(batch, trainCtx)+ }+ }+ if !completion && !wholePrompt {+ return batch, 0+ }+ return batch, LlamaServerBatchMemory(batch)+}++// LlamaServerBatchMemory is the conservative extra memory allowance for a+// generation or whole-prompt decision batch above the default batch size.+func LlamaServerBatchMemory(batch int) uint64 {+ switch {+ case batch >= 2048:+ return 2 << 30+ case batch >= 1024:+ return 768 << 20+ default:+ return 0+ }+}+func boundedNumPredict(numPredict, numCtx int) int {if numCtx <= 0 {return numPredict@@ -183,6 +215,7 @@ type llamaServerLaunchConfig struct {numParallel intkvCacheType stringembedding bool+ decision boolconfig LlamaServerConfiggpus []ml.DeviceInfogpuLibs []string@@ -285,7 +318,7 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, reqreturn prompt, nil}- tokens, err := s.tokenize(ctx, prompt, true, nil)+ tokens, err := s.tokenize(ctx, prompt, true)if err != nil {return nil, err}@@ -296,10 +329,7 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req}if !s.launch.config.ContextShift {- return nil, api.StatusError{- StatusCode: http.StatusBadRequest,- ErrorMessage: "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length",- }+ return nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: promptTooLongMessage}}nKeep := req.Options.NumKeep@@ -376,7 +406,14 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec."--no-webui","--offline","-c", strconv.Itoa(launch.opts.NumCtx * launch.numParallel),- "-np", strconv.Itoa(launch.numParallel),+ }+ // The questions of a decision request share their prompt only while they+ // run in parallel slots. Their prompts rarely repeat across requests, and+ // at trace verbosity llama-server logs its whole prompt cache after each one.+ if launch.decision {+ params = append(params, "-np", "16", "--kv-unified", "--cache-ram", "0")+ } else {+ params = append(params, "-np", strconv.Itoa(launch.numParallel))}params = appendLlamaServerLogArgs(params)params = appendJinjaArgs(params, launch.config)@@ -863,6 +900,7 @@ func NewLlamaServerRunner(// Check if this is an embedding modelarch := f.KV().Architecture()isEmbedding := f.KV().Has("pooling_type")+ decisionType := f.KV().String("decision.type")// Older Ollama-format GGUFs store vision tensors (v.*, mm.*) inline in// the main model file rather than in a separate projector layer. When@@ -943,6 +981,7 @@ func NewLlamaServerRunner(numParallel: numParallel,kvCacheType: kvCacheType,embedding: isEmbedding,+ decision: decisionType != "",config: config,gpus: slices.Clone(gpus),gpuLibs: slices.Clone(gpuLibs),@@ -1886,7 +1925,10 @@ func llamaServerStreamLimitError(label string, err error) error {}func (s *llamaServerRunner) statusErrorMessage(body []byte) string {- errMsg := strings.TrimSpace(string(body))+ errMsg := extractLlamaServerErrorMessage(body)+ if errMsg == "" {+ errMsg = strings.TrimSpace(string(body))+ }statusMsg := s.lastErrMsg()if statusMsg == "" {return errMsg@@ -2579,6 +2621,8 @@ func extractLlamaServerErrorMessage(body []byte) string {return ""}+const promptTooLongMessage = "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length"+func isEmbeddingInputLimitError(errMsg string) bool {msg := strings.ToLower(errMsg)return strings.Contains(msg, "too large") ||@@ -2588,15 +2632,13 @@ func isEmbeddingInputLimitError(errMsg string) bool {strings.Contains(msg, "exceeds the available context")}-func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecial bool, parseSpecial *bool) ([]int, error) {+func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecial bool) ([]int, error) {req := struct {- Content any `json:"content"`- AddSpecial bool `json:"add_special,omitempty"`- ParseSpecial *bool `json:"parse_special,omitempty"`+ Content any `json:"content"`+ AddSpecial bool `json:"add_special,omitempty"`}{- Content: content,- AddSpecial: addSpecial,- ParseSpecial: parseSpecial,+ Content: content,+ AddSpecial: addSpecial,}data, err := json.Marshal(req)@@ -2637,7 +2679,7 @@ func (s *llamaServerRunner) tokenize(ctx context.Context, content any, addSpecia// Tokenize calls llama-server's /tokenize endpoint.func (s *llamaServerRunner) Tokenize(ctx context.Context, content string) ([]int, error) {- return s.tokenize(ctx, content, false, nil)+ return s.tokenize(ctx, content, false)}// Detokenize calls llama-server's /detokenize endpoint.diff --git a/llm/llama_server_score.go b/llm/llama_server_score.godeleted file mode 100644index fc3474d06..000000000--- a/llm/llama_server_score.go+++ /dev/null@@ -1,370 +0,0 @@-package llm--import (- "bytes"- "context"- "encoding/json"- "fmt"- "io"- "math"- "net/http"- "slices"- "strings"-- "github.com/ollama/ollama/api"-)--var _ Scorer = (*llamaServerRunner)(nil)--const (- // Candidates are lifted above every other token so that top_k keeps exactly- // them; a shared bias does not change their softmax. A larger bias is- // retried if another token still outranks a candidate.- scoreBias = 100- scoreRetryBias = 1000-)--// A candidate whose probability underflows float32 is omitted by llama-server.-var scoreFloorLogit = float32(math.Log(math.SmallestNonzeroFloat32))--type scoreRowTokens struct {- tokens, candidates []int-}--func (s *llamaServerRunner) Score(ctx context.Context, input ScoreRequest) (ScoreResponse, error) {- var result ScoreResponse- badRequest := func(format string, args ...any) (ScoreResponse, error) {- return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: fmt.Sprintf(format, args...)}- }- if len(input.Segments) == 0 && len(input.Fields) == 0 && (len(input.Rows) < 1 || len(input.Rows) > 64) {- return badRequest("scoring requires 1–64 prompts")- }- if input.MaxTokens < 1 || input.MaxTokens > s.ContextLength() {- return badRequest("max_tokens must be between 1 and %d", s.ContextLength())- }- if err := s.sem.Acquire(ctx, 1); err != nil {- return result, err- }- defer s.sem.Release(1)- status, err := s.getServerStatusRetry(ctx)- if err != nil {- return result, err- } else if status != ServerStatusReady {- return result, fmt.Errorf("unexpected server status: %s", status)- }-- if len(input.Segments) > 0 || len(input.Fields) > 0 {- return s.scoreFields(ctx, input)- }-- rows := make([]scoreRowTokens, len(input.Rows))- parseSpecial, ordinary := true, false- for i, row := range input.Rows {- if len(row.Candidates) < 1 || len(row.Candidates) > 26 {- return badRequest("prompt %d requires 1–26 candidates", i)- }- tokens, err := s.tokenize(ctx, row.Prompt, false, &parseSpecial)- if err != nil {- return result, err- }- if len(tokens) == 0 || len(tokens) > input.MaxTokens {- return badRequest("prompt %d has %d tokens; expected 1–%d (input is never truncated)", i, len(tokens), input.MaxTokens)- }- // llama-server marks the result truncated when prompt+1 reaches the- // context limit, so scoring needs two positions of headroom.- if len(tokens)+2 > s.ContextLength() {- return badRequest("prompt %d requires %d context tokens for scoring; model has %d", i, len(tokens)+2, s.ContextLength())- }- rows[i].tokens = tokens- result.InputTokens += len(tokens)- for _, candidate := range row.Candidates {- joined, err := s.tokenize(ctx, row.Prompt+candidate, false, &parseSpecial)- if err != nil {- return result, err- }- literal, err := s.tokenize(ctx, candidate, false, &ordinary)- if err != nil {- return result, err- }- if len(joined) != len(tokens)+1 || !slices.Equal(joined[:len(tokens)], tokens) || len(literal) != 1 || literal[0] != joined[len(tokens)] {- return badRequest("candidate %q must append exactly one ordinary token to prompt %d", candidate, i)- }- id := joined[len(tokens)]- if id < 0 || slices.Contains(rows[i].candidates, id) {- return badRequest("prompt %d has invalid or duplicate candidate tokens", i)- }- rows[i].candidates = append(rows[i].candidates, id)- }- }-- result.OutputTokens, err = s.primeSharedPrefix(ctx, rows)- if err != nil {- return ScoreResponse{}, err- }-- // Rows are scored in order: the first row continues from the primer.- result.Logits = make([][]float32, len(rows))- for i, row := range rows {- logits, outputTokens, err := s.scoreRow(ctx, row)- if err != nil {- return ScoreResponse{}, err- }- result.Logits[i] = logits- result.OutputTokens += outputTokens- }- return result, nil-}--// llama-server saves a checkpoint of state it cannot roll back, from recurrent-// and sliding-window layers, this many tokens before the end of each prompt it-// evaluates. See https://github.com/ggml-org/llama.cpp/pull/20288.-const scoreCheckpointOffset = 4--// primeSharedPrefix makes llama-server save a checkpoint at the end of the-// tokens every row shares, so each row evaluates only its own last few tokens-// instead of its whole prompt.-//-// Rows share the rendered context and schema and differ only near the end.-// llama-server usually reuses a cached prefix by trimming its cache, but it-// cannot trim the state of recurrent layers, as in hybrid models like Qwen 3.5,-// or of sliding-window layers. Such models can only resume from a checkpoint,-// and the checkpoints llama-server saves on its own sit near the end of each-// prompt, past the point where the next row differs. Without a primer, every row would be-// evaluated from the start.-//-// The primer evaluates the shared tokens plus the first row's next-// scoreCheckpointOffset tokens into the cache, so llama-server saves its-// checkpoint exactly at the end of the shared tokens. The pinned server still-// generates one token with n_predict 0; it is discarded but counted in usage.-// The first row continues from the primer; later rows restore the checkpoint.-// With 8 questions, scoring is about 3x faster.-//-// Priming only changes speed: if llama-server stops saving that checkpoint,-// rows are evaluated from the start and scores stay the same. It can be-// removed once llama-server can save a checkpoint at a requested position or-// score rows as one batch.-func (s *llamaServerRunner) primeSharedPrefix(ctx context.Context, rows []scoreRowTokens) (int, error) {- if len(rows) < 2 {- return 0, nil- }- first := rows[0].tokens- shared := len(first)- for _, row := range rows[1:] {- shared = min(shared, len(row.tokens))- for i := range shared {- if row.tokens[i] != first[i] {- shared = i- break- }- }- }- // Nothing is shared, or the first row is too short to continue from the- // primer.- if shared == 0 || len(first) <= shared+scoreCheckpointOffset {- return 0, nil- }- tokens := first[:shared+scoreCheckpointOffset]- var output scoreCompletionResponse- if err := s.scoreRequest(ctx, "/completion", scoreCompletionRequest{Prompt: tokens, CachePrompt: true, NPredict: 0}, &output); err != nil {- return 0, err- }- if output.Truncated || output.TokensEvaluated != len(tokens) {- return 0, fmt.Errorf("scoring did not evaluate the complete shared prefix")- }- return output.TokensPredicted, nil-}--// scoreCompletionRequest is the part of llama-server's /completion request-// that scoring uses.-type scoreCompletionRequest struct {- Prompt []int `json:"prompt"`- Stream bool `json:"stream"`- CachePrompt bool `json:"cache_prompt"`- NPredict int `json:"n_predict"`- NProbs int `json:"n_probs,omitempty"`- PostSamplingProbs bool `json:"post_sampling_probs"`- Samplers []string `json:"samplers,omitempty"`- TopK int `json:"top_k"`- Temperature float32 `json:"temperature"`- LogitBias [][2]float64 `json:"logit_bias,omitempty"`-}--// scoreCompletionResponse is the part of llama-server's /completion response-// that scoring reads.-type scoreCompletionResponse struct {- TokensPredicted int `json:"tokens_predicted"`- TokensEvaluated int `json:"tokens_evaluated"`- Truncated bool `json:"truncated"`- Probabilities []struct {- TopProbs []struct {- ID *int `json:"id"`- Prob *float64 `json:"prob"`- } `json:"top_probs"`- } `json:"completion_probabilities"`-}--// scoreRow reads every candidate from one next-token distribution. Biasing the-// candidates by the same amount keeps their relative logits, so the returned-// probabilities are the softmax over the candidates' original logits.-func (s *llamaServerRunner) scoreRow(ctx context.Context, row scoreRowTokens) ([]float32, int, error) {- var outputTokens int- for _, bias := range []float64{scoreBias, scoreRetryBias} {- req := scoreCompletionRequest{- Prompt: row.tokens, CachePrompt: true, NPredict: 1,- NProbs: len(row.candidates), PostSamplingProbs: true,- Samplers: []string{"top_k", "temperature"}, TopK: len(row.candidates), Temperature: 1,- }- for _, id := range row.candidates {- req.LogitBias = append(req.LogitBias, [2]float64{float64(id), bias})- }- var output scoreCompletionResponse- if err := s.scoreRequest(ctx, "/completion", req, &output); err != nil {- return nil, 0, err- }- if output.Truncated || output.TokensEvaluated != len(row.tokens) || output.TokensPredicted != 1 || len(output.Probabilities) != 1 || len(output.Probabilities[0].TopProbs) == 0 {- return nil, 0, fmt.Errorf("scoring did not evaluate the complete prompt and return one distribution")- }- outputTokens += output.TokensPredicted- logits := slices.Repeat([]float32{scoreFloorLogit}, len(row.candidates))- seen := make([]bool, len(row.candidates))- outranked := false- for _, p := range output.Probabilities[0].TopProbs {- if p.ID == nil || p.Prob == nil || math.IsNaN(*p.Prob) || *p.Prob <= 0 || *p.Prob > 1 {- return nil, 0, fmt.Errorf("scoring returned an invalid candidate probability")- }- j := slices.Index(row.candidates, *p.ID)- if j < 0 {- outranked = true- break- }- if seen[j] {- return nil, 0, fmt.Errorf("scoring returned a candidate twice")- }- seen[j] = true- logits[j] = max(float32(math.Log(*p.Prob)), scoreFloorLogit)- }- if !outranked {- return logits, outputTokens, nil- }- }- return nil, 0, fmt.Errorf("scoring candidates were outranked by other tokens")-}--func (s *llamaServerRunner) scoreRequest(ctx context.Context, path string, input, output any) error {- data, err := json.Marshal(input)- if err != nil {- return err- }- req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d%s", s.port, path), bytes.NewReader(data))- if err != nil {- return err- }- req.Header.Set("Content-Type", "application/json")- res, err := s.httpClient().Do(req)- if err != nil {- if ctx.Err() != nil {- return ctx.Err()- }- if msg := s.lastErrMsg(); msg != "" {- return fmt.Errorf("scoring failed: %s: %w", msg, err)- }- return err- }- defer res.Body.Close()- body, err := io.ReadAll(res.Body)- if err != nil {- return err- }- if res.StatusCode != http.StatusOK {- return api.StatusError{StatusCode: res.StatusCode, ErrorMessage: s.statusErrorMessage(body)}- }- if err := json.Unmarshal(body, output); err != nil {- return fmt.Errorf("invalid scoring response: %w", err)- }- return nil-}--func (s *llamaServerRunner) scoreFields(ctx context.Context, input ScoreRequest) (ScoreResponse, error) {- bad := func(message string) (ScoreResponse, error) {- return ScoreResponse{}, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: message}- }- if len(input.Fields) < 1 || len(input.Fields) > 64 || len(input.Segments) == 0 {- return bad("invalid decision scoring request")- }- var err error- // Do not concatenate before tokenizing: BPE may merge across the reference- // encoder's segment boundaries, changing both embeddings and span offsets.- var tokens []int- offsets := []int{0}- cache := map[string][]int{}- special := true- for _, segment := range input.Segments {- ids, ok := cache[segment]- if !ok {- ids, err = s.tokenize(ctx, segment, false, &special)- if err != nil {- return ScoreResponse{}, err- }- cache[segment] = ids- }- tokens = append(tokens, ids...)- offsets = append(offsets, len(tokens))- if len(tokens) > input.MaxTokens {- return bad("decision prompt exceeds the model context (input is never truncated)")- }- }- if len(tokens) == 0 {- return bad("decision prompt must not be empty")- }- fields := make([]ScoreField, len(input.Fields))- for i, f := range input.Fields {- if f.Type < 0 || f.Type > 2 || len(f.Options) < 2 || len(f.Options) > 26 {- return bad("invalid decision field")- }- // Map the question and its options from segment indexes to token offsets.- spans := append([][2]int{f.Question}, f.Options...)- for j, span := range spans {- start, end := span[0], span[1]- if start < 0 || start >= end || end >= len(offsets) || offsets[start] == offsets[end] {- return bad("invalid decision token span")- }- spans[j] = [2]int{offsets[start], offsets[end]}- }- fields[i] = ScoreField{Type: f.Type, Question: spans[0], Options: spans[1:]}- }- var images []api.ImageData- for _, image := range input.Images {- data, err := llamaServerMediaBytes(image)- if err != nil {- return bad(err.Error())- }- if !strings.HasPrefix(http.DetectContentType(data), "image/") {- return bad("invalid decision image")- }- images = append(images, data)- }- imagePosition := 0- if len(images) > 0 {- if input.ImagePosition < 0 || input.ImagePosition >= len(offsets) {- return bad("invalid image position")- }- imagePosition = offsets[input.ImagePosition]- }- request := struct {- Images []api.ImageData `json:"images,omitempty"`- ImagePosition int `json:"image_position"`- Input []int `json:"input"`- Fields []ScoreField `json:"score_fields"`- }{images, imagePosition, tokens, fields}- var result []struct {- Logits [][]float32 `json:"logits"`- Tokens int `json:"tokens_evaluated"`- }- if err := s.scoreRequest(ctx, "/embedding", request, &result); err != nil {- return ScoreResponse{}, err- }- if len(result) != 1 || result[0].Tokens > input.MaxTokens || (len(images) == 0 && result[0].Tokens != len(tokens)) || (len(images) > 0 && result[0].Tokens <= len(tokens)) || len(result[0].Logits) != len(fields) {- return ScoreResponse{}, fmt.Errorf("decision runner did not score the complete request")- }- return ScoreResponse{Logits: result[0].Logits, InputTokens: result[0].Tokens}, nil-}diff --git a/llm/llama_server_score_test.go b/llm/llama_server_score_test.godeleted file mode 100644index 9995ee289..000000000--- a/llm/llama_server_score_test.go+++ /dev/null@@ -1,545 +0,0 @@-package llm--import (- "cmp"- "context"- "encoding/json"- "errors"- "fmt"- "io"- "maps"- "math"- "net"- "net/http"- "net/http/httptest"- "reflect"- "slices"- "strings"- "sync/atomic"- "testing"- "time"-- "github.com/ollama/ollama/api"- "golang.org/x/sync/semaphore"-)--func newLlamaScoreTestRunner(t *testing.T, completion http.HandlerFunc) *llamaServerRunner {- t.Helper()- mux := http.NewServeMux()- mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {- fmt.Fprint(w, `{"status":"ok"}`)- })- mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) {- var req struct {- Content string `json:"content"`- AddSpecial bool `json:"add_special"`- ParseSpecial *bool `json:"parse_special"`- }- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- t.Error(err)- return- }- if req.AddSpecial || req.ParseSpecial == nil {- t.Error("scoring must explicitly parse special tokens without adding BOS/EOS")- return- }- var tokens []int- for _, c := range req.Content {- tokens = append(tokens, int(c))- }- if *req.ParseSpecial && strings.HasSuffix(req.Content, "<special>") {- tokens = append(tokens[:len(tokens)-len("<special>")], 1000)- }- // Model a tokenizer that merges the candidate with the prompt's last token.- if strings.HasSuffix(req.Content, "~") && len(tokens) > 1 {- tokens = append(tokens[:len(tokens)-2], 1001)- }- _ = json.NewEncoder(w).Encode(map[string]any{"tokens": tokens})- })- mux.HandleFunc("/completion", completion)- srv := httptest.NewServer(mux)- t.Cleanup(srv.Close)- return &llamaServerRunner{- port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(),- cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}},- sem: semaphore.NewWeighted(1),- }-}--type scoreCompletionTestRequest struct {- Prompt []int `json:"prompt"`- Grammar string `json:"grammar"`- Stream bool `json:"stream"`- CachePrompt bool `json:"cache_prompt"`- NPredict int `json:"n_predict"`- NProbs int `json:"n_probs"`- PostSamplingProbs *bool `json:"post_sampling_probs"`- Temperature float64 `json:"temperature"`- Samplers []string `json:"samplers"`- TopK int `json:"top_k"`- LogitBias [][2]float64 `json:"logit_bias"`-}--// scoreCompletionTestResponse returns the candidates' probabilities in the-// order llama-server sorts them.-func scoreCompletionTestResponse(prompt int, probs map[int]float64) map[string]any {- ids := slices.Collect(maps.Keys(probs))- slices.SortFunc(ids, func(a, b int) int { return cmp.Compare(probs[b], probs[a]) })- var top []any- for _, id := range ids {- top = append(top, map[string]any{"id": id, "token": string(rune(id)), "prob": probs[id]})- }- return map[string]any{- "tokens_predicted": 1, "tokens_evaluated": prompt,- "completion_probabilities": []any{map[string]any{"id": ids[0], "token": string(rune(ids[0])), "prob": probs[ids[0]], "top_probs": top}},- }-}--func checkScoreRowRequest(t *testing.T, req scoreCompletionTestRequest, candidates []int, bias float64) {- t.Helper()- var biased []int- for _, b := range req.LogitBias {- if b[1] != bias {- t.Errorf("candidates must share bias %g: %v", bias, req.LogitBias)- }- biased = append(biased, int(b[0]))- }- if req.Stream || !req.CachePrompt || req.NPredict != 1 || req.Grammar != "" || req.NProbs != len(candidates) || req.PostSamplingProbs == nil || !*req.PostSamplingProbs ||- req.Temperature != 1 || !slices.Equal(req.Samplers, []string{"top_k", "temperature"}) || req.TopK != len(candidates) || !slices.Equal(biased, candidates) {- t.Errorf("request changes the candidate distribution: %+v", req)- }-}--func TestLlamaServerScore(t *testing.T) {- var calls int- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- var req scoreCompletionTestRequest- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- t.Error(err)- return- }- calls++- switch {- case slices.Equal(req.Prompt, []int{'a', 'b', 'c'}):- checkScoreRowRequest(t, req, []int{'Z', 'A'}, scoreBias)- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'Z': 0.25, 'A': 0.75}))- case slices.Equal(req.Prompt, []int{'x', 'y'}):- checkScoreRowRequest(t, req, []int{'B'}, scoreBias)- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'B': 1}))- default:- t.Errorf("rendered prompt changed: %v", req.Prompt)- }- })- // Model generation defaults must not affect scoring.- runner.options.Temperature = 0- runner.options.Stop = []string{"A"}- runner.options.RepeatPenalty = 2- runner.options.NumCtx = 5 // The longest prompt leaves exactly two positions of headroom.- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{- {Prompt: "abc", Candidates: []string{"Z", "A"}},- {Prompt: "xy", Candidates: []string{"B"}},- }})- if err != nil {- t.Fatal(err)- }- want := [][]float32{{float32(math.Log(0.25)), float32(math.Log(0.75))}, {0}}- if !reflect.DeepEqual(result.Logits, want) || result.InputTokens != 5 || result.OutputTokens != 2 || calls != 2 {- t.Fatalf("incomplete or reordered scores/usage: %+v, calls=%d", result, calls)- }-}--func TestLlamaServerScoreSharedPrefix(t *testing.T) {- var prompts [][]int- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- var req scoreCompletionTestRequest- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- t.Error(err)- return- }- prompts = append(prompts, req.Prompt)- if len(prompts) == 1 {- // The native server generates one token even with n_predict: 0.- if req.NPredict != 0 || req.NProbs != 0 || len(req.LogitBias) != 0 || len(req.Samplers) != 0 || !req.CachePrompt {- t.Errorf("unexpected primer request: %+v", req)- }- _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})- return- }- candidate := int(req.LogitBias[0][0])- checkScoreRowRequest(t, req, []int{candidate}, scoreBias)- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{candidate: 1}))- })- runner.options.NumCtx = 64- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 32, Rows: []ScoreRow{- {Prompt: "shared:first", Candidates: []string{"A"}},- {Prompt: "shared:two", Candidates: []string{"B"}},- {Prompt: "shared:x", Candidates: []string{"C"}},- }})- if err != nil {- t.Fatal(err)- }- // The primer evaluates the shared prefix plus 4 tokens of the first row,- // which continues from it; the other rows restore the checkpoint.- want := [][]int{scoreTestTokens("shared:firs"), scoreTestTokens("shared:first"), scoreTestTokens("shared:two"), scoreTestTokens("shared:x")}- if !reflect.DeepEqual(prompts, want) {- t.Fatalf("got prompts %v, want %v", prompts, want)- }- if !reflect.DeepEqual(result.Logits, [][]float32{{0}, {0}, {0}}) || result.InputTokens != 30 || result.OutputTokens != 4 {- t.Fatalf("unexpected result: %+v", result)- }-}--func TestLlamaServerScorePrimer(t *testing.T) {- for _, tt := range []struct {- name string- rows []string- primer string- }{- {"one row", []string{"abcdefgh"}, ""},- {"no shared prefix", []string{"abcdefgh", "xbcdefgh"}, ""},- {"first row too short", []string{"abcdefg", "abcdwxyzuv"}, ""},- {"identical prompts", []string{"abcdefgh", "abcdefgh"}, ""},- {"shared prefix", []string{"abcdefghij", "abcxyzuvwt", "abcdefgh"}, "abcdefg"},- } {- t.Run(tt.name, func(t *testing.T) {- var primer []int- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- var req scoreCompletionTestRequest- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- t.Error(err)- return- }- if len(req.LogitBias) == 0 {- primer = req.Prompt- _ = json.NewEncoder(w).Encode(map[string]any{"tokens_predicted": 1, "tokens_evaluated": len(req.Prompt)})- return- }- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), map[int]float64{'A': 1}))- })- runner.options.NumCtx = 64- input := ScoreRequest{MaxTokens: 32}- for _, row := range tt.rows {- input.Rows = append(input.Rows, ScoreRow{Prompt: row, Candidates: []string{"A"}})- }- if _, err := runner.Score(t.Context(), input); err != nil {- t.Fatal(err)- }- if !slices.Equal(primer, scoreTestTokens(tt.primer)) {- t.Fatalf("primed %v, want %v", primer, scoreTestTokens(tt.primer))- }- })- }-}--// scoreTestTokens tokenizes s the way the test tokenizer does: one token per rune.-func scoreTestTokens(s string) []int {- var tokens []int- for _, c := range s {- tokens = append(tokens, int(c))- }- return tokens-}--func TestLlamaServerScoreOutranked(t *testing.T) {- for _, recovers := range []bool{true, false} {- t.Run(fmt.Sprintf("recovers=%v", recovers), func(t *testing.T) {- var calls int- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- var req scoreCompletionTestRequest- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- t.Error(err)- return- }- calls++- bias := float64(scoreBias)- if calls > 1 {- bias = scoreRetryBias- }- checkScoreRowRequest(t, req, []int{'A', 'B'}, bias)- probs := map[int]float64{'A': 0.6, 'x': 0.4}- if recovers && calls > 1 {- probs = map[int]float64{'A': 0.6, 'B': 0.4}- }- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(len(req.Prompt), probs))- })- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})- if calls != 2 {- t.Fatalf("got %d attempts, want 2", calls)- }- if !recovers {- if err == nil {- t.Fatal("accepted a distribution without every candidate")- }- return- }- if err != nil || result.OutputTokens != 2 || !reflect.DeepEqual(result.Logits, [][]float32{{float32(math.Log(0.6)), float32(math.Log(0.4))}}) {- t.Fatalf("unexpected result: %+v, %v", result, err)- }- })- }-}--func TestLlamaServerScoreUnderflow(t *testing.T) {- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- // llama-server omits candidates whose probability underflows.- _ = json.NewEncoder(w).Encode(scoreCompletionTestResponse(3, map[int]float64{'B': 1}))- })- result, err := runner.Score(t.Context(), ScoreRequest{MaxTokens: 5, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "B"}}}})- if err != nil || !reflect.DeepEqual(result.Logits, [][]float32{{scoreFloorLogit, 0}}) {- t.Fatalf("unexpected result: %+v, %v", result, err)- }-}--func TestLlamaServerScoreValidation(t *testing.T) {- for _, tt := range []struct {- name string- input ScoreRequest- }{- {"empty rows", ScoreRequest{MaxTokens: 7}},- {"too many rows", ScoreRequest{MaxTokens: 7, Rows: make([]ScoreRow, 65)}},- {"zero max tokens", ScoreRequest{Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},- {"max exceeds context", ScoreRequest{MaxTokens: 9, Rows: []ScoreRow{{Prompt: "a", Candidates: []string{"A"}}}}},- {"empty prompt", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Candidates: []string{"A"}}}}},- {"oversize prompt", ScoreRequest{MaxTokens: 2, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}},- {"no scoring token space", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefgh", Candidates: []string{"A"}}}}},- {"no native scoring headroom", ScoreRequest{MaxTokens: 8, Rows: []ScoreRow{{Prompt: "abcdefg", Candidates: []string{"A"}}}}},- {"empty candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc"}}}},- {"too many candidates", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: make([]string, 27)}}}},- {"multiple tokens", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"AB"}}}}},- {"special token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"<special>"}}}}},- {"changed prefix", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"~"}}}}},- {"duplicate token", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A", "A"}}}}},- {"invalid later row", ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}, {Prompt: "abc", Candidates: []string{"AB"}}}}},- } {- t.Run(tt.name, func(t *testing.T) {- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- t.Error("started inference before validating every row")- })- _, err := runner.Score(t.Context(), tt.input)- var status api.StatusError- if !errors.As(err, &status) || status.StatusCode != http.StatusBadRequest {- t.Fatalf("expected validation error, got %v", err)- }- })- }-}--func TestLlamaServerScoreResponse(t *testing.T) {- const valid = `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`- for _, tt := range []struct {- name string- body string- status int- }{- {"missing probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A"}]`, 1), 200},- {"missing token id", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"token":"A","prob":1}]`, 1), 200},- {"zero probability", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":0}]`, 1), 200},- {"probability above one", strings.Replace(valid, `"token":"A","prob":1}]`, `"token":"A","prob":1.5}]`, 1), 200},- {"duplicate candidate", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[{"id":65,"token":"A","prob":0.5},{"id":65,"token":"A","prob":0.5}]`, 1), 200},- {"empty top probabilities", strings.Replace(valid, `[{"id":65,"token":"A","prob":1}]`, `[]`, 1), 200},- {"incomplete prompt", strings.Replace(valid, `"tokens_evaluated":3`, `"tokens_evaluated":2`, 1), 200},- {"truncated prompt", strings.Replace(valid, `"tokens_predicted":1`, `"truncated":true,"tokens_predicted":1`, 1), 200},- {"extra generated token", strings.Replace(valid, `"tokens_predicted":1`, `"tokens_predicted":2`, 1), 200},- {"empty probabilities", `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[]}`, 200},- {"malformed JSON", `{`, 200},- {"upstream error", `{"error":{"message":"out of memory"}}`, 500},- } {- t.Run(tt.name, func(t *testing.T) {- var failed atomic.Bool- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- if failed.CompareAndSwap(false, true) {- w.WriteHeader(tt.status)- fmt.Fprint(w, tt.body)- return- }- fmt.Fprint(w, valid)- })- input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}- if _, err := runner.Score(t.Context(), input); err == nil {- t.Fatal("accepted invalid scoring response")- } else if tt.status == 500 {- var status api.StatusError- if !errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err) {- t.Fatalf("lost upstream status/OOM diagnostic: %v", err)- }- }- ctx, cancel := context.WithTimeout(t.Context(), time.Second)- defer cancel()- if _, err := runner.Score(ctx, input); err != nil {- t.Fatalf("runner unusable after failed scoring: %v", err)- }- })- }-}--func TestLlamaServerScoreCancellation(t *testing.T) {- for _, queued := range []bool{true, false} {- t.Run(fmt.Sprintf("queued=%v", queued), func(t *testing.T) {- started := make(chan struct{})- var block atomic.Bool- block.Store(!queued)- runner := newLlamaScoreTestRunner(t, func(w http.ResponseWriter, r *http.Request) {- if block.CompareAndSwap(true, false) {- _, _ = io.Copy(io.Discard, r.Body)- close(started)- <-r.Context().Done()- return- }- fmt.Fprint(w, `{"tokens_predicted":1,"tokens_evaluated":3,"completion_probabilities":[{"id":65,"token":"A","prob":1,"top_probs":[{"id":65,"token":"A","prob":1}]}]}`)- })- input := ScoreRequest{MaxTokens: 7, Rows: []ScoreRow{{Prompt: "abc", Candidates: []string{"A"}}}}- ctx, cancel := context.WithCancel(t.Context())- defer cancel()- if queued {- if err := runner.sem.Acquire(t.Context(), 1); err != nil {- t.Fatal(err)- }- cancel()- } else {- go func() {- select {- case <-started:- cancel()- case <-ctx.Done():- }- }()- }- _, err := runner.Score(ctx, input)- if !errors.Is(err, context.Canceled) {- t.Fatalf("cancellation lost: %v", err)- }- if queued {- runner.sem.Release(1)- }- next, done := context.WithTimeout(t.Context(), time.Second)- defer done()- if _, err := runner.Score(next, input); err != nil {- t.Fatalf("runner unusable after cancellation: %v", err)- }- })- }-}--func TestLlamaServerScoreFields(t *testing.T) {- for _, failure := range []string{"", "truncated", "missing head", "malformed JSON", "upstream error", "too long", "invalid span", "max exceeds context", "empty fields", "canceled", "image", "image over context", "invalid image", "invalid image position"} {- t.Run(failure, func(t *testing.T) {- calls := 0- mux := http.NewServeMux()- mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) })- mux.HandleFunc("/tokenize", func(w http.ResponseWriter, r *http.Request) {- var input struct {- Content string `json:"content"`- AddSpecial bool `json:"add_special"`- ParseSpecial bool `json:"parse_special"`- }- if err := json.NewDecoder(r.Body).Decode(&input); err != nil {- t.Fatal(err)- }- if input.AddSpecial || !input.ParseSpecial {- t.Error("incorrect tokenizer settings")- }- ids := map[string][]int{"prefix": {1, 2}, "question": {3}, "one": {4, 5}, "two": {6}, "suffix": {7}}[input.Content]- if ids == nil {- t.Error("segments must be tokenized separately")- }- json.NewEncoder(w).Encode(map[string]any{"tokens": ids})- })- mux.HandleFunc("/embedding", func(w http.ResponseWriter, r *http.Request) {- calls++- var input struct {- Input []int `json:"input"`- Fields []ScoreField `json:"score_fields"`- Images []api.ImageData `json:"images"`- ImagePosition int `json:"image_position"`- }- if err := json.NewDecoder(r.Body).Decode(&input); err != nil {- t.Fatal(err)- }- if !reflect.DeepEqual(input.Input, []int{1, 2, 3, 4, 5, 6, 7}) || !reflect.DeepEqual(input.Fields, []ScoreField{{Type: 0, Question: [2]int{2, 3}, Options: [][2]int{{3, 5}, {5, 6}}}}) {- t.Errorf("wrong token spans: %+v", input)- }- switch failure {- case "image":- if len(input.Images) != 1 || input.ImagePosition != 2 {- t.Errorf("missing image or wrong image position: %+v", input)- }- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":11}]`)- case "image over context":- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":17}]`)- case "upstream error":- w.WriteHeader(http.StatusInternalServerError)- fmt.Fprint(w, `{"error":{"message":"out of memory"}}`)- case "malformed JSON":- fmt.Fprint(w, `{`)- case "truncated":- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":6}]`)- case "missing head":- fmt.Fprint(w, `[{"embedding":[[2,0]]}]`)- default:- fmt.Fprint(w, `[{"logits":[[2,0]],"tokens_evaluated":7}]`)- }- })- srv := httptest.NewServer(mux)- defer srv.Close()- runner := &llamaServerRunner{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(), cmd: fakeRunningCmd(), options: api.Options{Runner: api.Runner{NumCtx: 8}}, sem: semaphore.NewWeighted(1)}- input := ScoreRequest{MaxTokens: 8, Segments: []string{"prefix", "question", "one", "two", "suffix"}, Fields: []ScoreField{{Type: 0, Question: [2]int{1, 2}, Options: [][2]int{{2, 3}, {3, 4}}}}}- ctx, cancel := context.WithCancel(t.Context())- defer cancel()- invalid := true- switch failure {- case "image", "image over context", "invalid image", "invalid image position":- input.Images = []api.ImageData{{137, 80, 78, 71, 13, 10, 26, 10}}- input.ImagePosition = 1- if failure == "invalid image" {- input.Images[0] = []byte("not an image")- }- if failure == "invalid image position" {- input.ImagePosition = 99- }- if failure == "image" || failure == "image over context" {- invalid = false- input.MaxTokens = 16- runner.options.NumCtx = 16- }- case "too long":- input.MaxTokens = 6- case "invalid span":- input.Fields[0].Question = [2]int{-1, 2}- case "max exceeds context":- input.MaxTokens = 9- case "empty fields":- input.Fields = nil- case "canceled":- cancel()- default:- invalid = false- }- result, err := runner.Score(ctx, input)- if failure == "" || failure == "image" {- wantTokens := 7- if failure == "image" {- wantTokens = 11- }- if err != nil || result.InputTokens != wantTokens || result.OutputTokens != 0 {- t.Fatalf("result=%+v err=%v", result, err)- }- } else if err == nil {- t.Fatal("expected error")- }- var status api.StatusError- if failure == "upstream error" && (!errors.As(err, &status) || status.StatusCode != 500 || !IsOutOfMemory(err)) {- t.Fatalf("lost upstream status/OOM diagnostic: %v", err)- }- if failure == "canceled" && !errors.Is(err, context.Canceled) {- t.Fatalf("cancellation lost: %v", err)- }- if invalid && calls != 0 {- t.Fatal("invalid request reached the model")- }- if !runner.sem.TryAcquire(1) {- t.Fatal("scoring did not release the runner")- }- runner.sem.Release(1)- })- }-}diff --git a/llm/llama_server_systemone.go b/llm/llama_server_systemone.gonew file mode 100644index 000000000..2e6373f8b--- /dev/null+++ b/llm/llama_server_systemone.go@@ -0,0 +1,76 @@+package llm++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strings"++ "github.com/ollama/ollama/api"+)++func (s *llamaServerRunner) SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (answers, usage json.RawMessage, err error) {+ request := struct {+ State json.RawMessage `json:"state"`+ Images []string `json:"images,omitempty"`+ Questions json.RawMessage `json:"questions"`+ }{State: state, Questions: questions}+ for _, image := range images {+ data, err := llamaServerMediaBytes(image)+ if err != nil {+ return nil, nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: err.Error()}+ }+ request.Images = append(request.Images, "data:"+http.DetectContentType(data)+";base64,"+base64.StdEncoding.EncodeToString(data))+ }+ data, err := json.Marshal(request)+ if err != nil {+ return nil, nil, err+ }++ if err := s.sem.Acquire(ctx, 1); err != nil {+ return nil, nil, err+ }+ defer s.sem.Release(1)++ req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/v1/systemone", s.port), bytes.NewReader(data))+ if err != nil {+ return nil, nil, err+ }+ req.Header.Set("Content-Type", "application/json")+ res, err := s.httpClient().Do(req)+ if err != nil {+ if ctx.Err() != nil {+ return nil, nil, ctx.Err()+ }+ if msg := s.lastErrMsg(); msg != "" {+ return nil, nil, fmt.Errorf("decision failed: %s: %w", msg, err)+ }+ return nil, nil, err+ }+ defer res.Body.Close()+ body, err := io.ReadAll(res.Body)+ if err != nil {+ return nil, nil, err+ }+ if res.StatusCode != http.StatusOK {+ msg := s.statusErrorMessage(body)+ // Decision models that read a whole prompt in one batch run with a batch+ // as large as the context, so outgrowing it means outgrowing the context.+ if strings.Contains(msg, "physical batch size") {+ return nil, nil, api.StatusError{StatusCode: http.StatusBadRequest, ErrorMessage: promptTooLongMessage}+ }+ return nil, nil, api.StatusError{StatusCode: res.StatusCode, ErrorMessage: msg}+ }+ var response struct {+ Answers json.RawMessage `json:"answers"`+ Usage json.RawMessage `json:"usage"`+ }+ if err := json.Unmarshal(body, &response); err != nil {+ return nil, nil, fmt.Errorf("invalid decision response: %w", err)+ }+ return response.Answers, response.Usage, nil+}diff --git a/llm/llama_server_test.go b/llm/llama_server_test.goindex 2025e5a6c..62cdc755c 100644--- a/llm/llama_server_test.go+++ b/llm/llama_server_test.go@@ -951,8 +951,8 @@ func TestLlamaServerStatusErrorMessageIncludesOOMStatus(t *testing.T) {}got := runner.statusErrorMessage([]byte(`{"error":{"message":"Compute error."}}`))- if !strings.Contains(got, "Compute error") {- t.Fatalf("expected original response body, got %q", got)+ if !strings.HasPrefix(got, "Compute error.\n") {+ t.Fatalf("expected llama-server's error message, got %q", got)}if !IsOutOfMemoryMessage(got) {t.Fatalf("expected OOM status detail to be detectable, got %q", got)@@ -1725,6 +1725,21 @@ func TestLlamaServerEmbedding(t *testing.T) {}}+func TestLlamaServerSystemOnePromptOverContext(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"error":{"code":500,"message":"input (9000 tokens) is too large to process. increase the physical batch size (current batch size: 8192)","type":"server_error"}}`)+ }))+ defer srv.Close()+ runner := &llamaServerRunner{port: srv.Listener.Addr().(*net.TCPAddr).Port, client: srv.Client(), cmd: fakeRunningCmd(), sem: semaphore.NewWeighted(1)}++ _, _, err := runner.SystemOne(t.Context(), json.RawMessage(`"x"`), json.RawMessage(`{"q":{"type":"noul","instructions":"q"}}`), nil)+ var serr api.StatusError+ if !errors.As(err, &serr) || serr.StatusCode != http.StatusBadRequest || serr.ErrorMessage != promptTooLongMessage {+ t.Fatalf("err = %v, want 400 %q", err, promptTooLongMessage)+ }+}+func TestLegacyEmbeddingsWereRaw(t *testing.T) {tests := []struct {name string@@ -1915,6 +1930,40 @@ func TestEmbeddingBatchSize(t *testing.T) {}}+func TestLlamaServerBatch(t *testing.T) {+ tests := []struct {+ name string+ headBlocks, trainCtx uint64+ numCtx, numBatch int+ completion bool+ wantBatch int+ wantMemory uint64+ }{+ {"default generation", 0, 8192, 8192, 512, true, 512, 0},+ {"medium generation", 0, 8192, 8192, 1024, true, 1024, 768 << 20},+ {"large generation", 0, 8192, 8192, 2048, true, 2048, 2 << 30},+ {"other non-completion models", 0, 8192, 8192, 2048, false, 2048, 0},+ {"decision uses requested context", 4, 32768, 8192, 512, false, 8192, 2 << 30},+ {"decision caps at trained context", 4, 1024, 8192, 512, false, 1024, 768 << 20},+ {"decision without trained context", 4, 0, 8192, 512, false, 8192, 2 << 30},+ {"decision with an already matching batch", 4, 8192, 8192, 8192, false, 8192, 2 << 30},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ f := loadTestGGUF(t, gguftest.KV{+ "general.architecture": "clef",+ "clef.context_length": tt.trainCtx,+ "clef.decision.block_count": tt.headBlocks,+ })+ opts := api.Options{Runner: api.Runner{NumCtx: tt.numCtx, NumBatch: tt.numBatch}}+ batch, memory := LlamaServerBatch(f, opts, tt.completion)+ if batch != tt.wantBatch || memory != tt.wantMemory {+ t.Fatalf("batch=%d memory=%d, want batch=%d memory=%d", batch, memory, tt.wantBatch, tt.wantMemory)+ }+ })+ }+}+func TestAppendBatchArgs(t *testing.T) {tests := []struct {name stringdiff --git a/server/capabilities_test.go b/server/capabilities_test.goindex 5541c6cad..f447cb123 100644--- a/server/capabilities_test.go+++ b/server/capabilities_test.go@@ -126,6 +126,10 @@ func TestDecisionPublicCapabilities(t *testing.T) {want := []model.Capability{model.CapabilityDecision}if declaredVision {cfg.Capabilities = append(cfg.Capabilities, "vision")+ }+ // The GGUF inherits vision from its base model; the safetensors+ // fixture ships a projector.+ if declaredVision || format == "safetensors" {want = append(want, model.CapabilityVision)}_, digest := createBinFile(t, map[string]any{diff --git a/server/images.go b/server/images.goindex 7df4dd179..07fc72d29 100644--- a/server/images.go+++ b/server/images.go@@ -164,13 +164,14 @@ func (m *Model) Capabilities() []model.Capability {return capabilities}-// publicCapabilities limits decision models to decision and explicitly declared-// vision support. Serving still uses Capabilities.+// publicCapabilities limits decision models to decision and the vision they+// declare or ship a projector for, not vision inherited from a base model.+// Serving still uses Capabilities.func (m *Model) publicCapabilities() []model.Capability {capabilities := m.Capabilities()if slices.Contains(capabilities, model.CapabilityDecision) {public := []model.Capability{model.CapabilityDecision}- if slices.Contains(m.Config.Capabilities, "vision") && slices.Contains(capabilities, model.CapabilityVision) {+ if slices.Contains(m.Config.Capabilities, "vision") || len(m.ProjectorPaths) > 0 {public = append(public, model.CapabilityVision)}return public@@ -859,7 +860,7 @@ func GetModelForRunner(name, runner string) (*Model, error) {m.PreferChatTemplate = true}- if m.ModelPath != "" && m.isGGUF() && !modelHasPooling && !m.HasChatTemplate && (!m.HasGoTemplate || !envconfig.GoTemplate(true)) && m.Config.Renderer == "" && m.Config.Parser == "" && !usesHarmony {+ if m.ModelPath != "" && m.isGGUF() && !modelHasPooling && !m.HasChatTemplate && (!m.HasGoTemplate || !envconfig.GoTemplate(true)) && m.Config.Renderer == "" && m.Config.Parser == "" && !usesHarmony && m.metadata.String("decision.type") == "" {slog.Warn("model is missing tokenizer.chat_template and Go TEMPLATE support is unavailable; chat responses may be poorly formatted", "model", m.Name, "env", "OLLAMA_GO_TEMPLATE=1")}diff --git a/server/routes.go b/server/routes.goindex a93fda47f..95e863670 100644--- a/server/routes.go+++ b/server/routes.go@@ -896,8 +896,66 @@ func (s *Server) GenerateHandler(c *gin.Context) {streamResponse(c, ch)}-// SystemOneHandler compiles typed questions, scores their allowed answers, and-// returns probabilities. Callers must select weights trained for the prompt format.+func (s *Server) systemOneLlamaServer(c *gin.Context, m *Model, req decision.Request, body []byte) {+ decisionType := m.metadata.String("decision.type")+ // Old Clef GGUFs have a decision type but use qwen35; llama.cpp now+ // requires the clef architecture and its converted decision head.+ oldClef := decisionType == "clef" && m.metadata.String("general.architecture") == "qwen35"+ if decisionType == "" || oldClef {+ if oldClef || slices.Contains(m.Config.Capabilities, string(model.CapabilityDecision)) {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("'%s' was downloaded in a format that is no longer supported. To re-download, run 'ollama pull %s'", m.ShortName, m.ShortName)})+ return+ }+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q is not a llama.cpp decision model", req.Model)})+ return+ }+ if err := decision.ValidateLimits(req); err != nil {+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ }+ var raw struct {+ State json.RawMessage `json:"state"`+ Questions json.RawMessage `json:"questions"`+ }+ if err := json.Unmarshal(body, &raw); err != nil {+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})+ return+ }+ caps := []model.Capability{model.CapabilityDecision}+ if len(req.Images) > 0 {+ caps = append(caps, model.CapabilityVision)+ }+ r, _, _, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, nil, req.KeepAlive, nil)+ if err != nil {+ handleScheduleError(c, req.Model, err)+ return+ }+ runner, ok := r.(interface {+ SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (answers, usage json.RawMessage, err error)+ })+ if !ok {+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring", req.Model)})+ return+ }+ answers, usage, err := runner.SystemOne(c.Request.Context(), raw.State, raw.Questions, req.Images)+ if err != nil {+ s.sched.expireRunnersForRuntimeOOM(m, err)+ status := http.StatusInternalServerError+ var statusErr api.StatusError+ if errors.As(err, &statusErr) {+ status = statusErr.StatusCode+ }+ c.JSON(status, gin.H{"error": err.Error()})+ return+ }+ c.JSON(http.StatusOK, struct {+ Model string `json:"model"`+ Answers json.RawMessage `json:"answers"`+ Usage json.RawMessage `json:"usage"`+ }{req.Model, answers, usage})+}++// SystemOneHandler answers typed questions with probabilities.func (s *Server) SystemOneHandler(c *gin.Context) {// TODO(parthsareen): Check token limits before copying state and schema into// each question's prompt. This byte cap limits memory use until then.@@ -952,8 +1010,12 @@ func (s *Server) SystemOneHandler(c *gin.Context) {handleScheduleError(c, req.Model, err)return}- encoding := m.metadata.String("decision.type")- if encoding == "" && (m.Config.Renderer == "tev1" || m.Config.Renderer == "clef" || m.Config.Renderer == "strands") {+ if m.isGGUF() {+ s.systemOneLlamaServer(c, m, req, body)+ return+ }+ var encoding string+ if m.Config.Renderer == "tev1" || m.Config.Renderer == "clef" || m.Config.Renderer == "strands" {encoding = m.Config.Renderer}compiled, err := decision.Compile(req, encoding)@@ -979,11 +1041,7 @@ func (s *Server) SystemOneHandler(c *gin.Context) {if m.System != "" {messages = append([]api.Message{{Role: "system", Content: m.System}}, messages...)}- think := &api.ThinkValue{Value: false}- if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative {- return r.ApplyChatTemplate(c.Request.Context(), llm.ChatRequest{Messages: messages, Think: think})- }- return renderPrompt(m, messages, nil, think)+ return renderPrompt(m, messages, nil, &api.ThinkValue{Value: false})}); err != nil {c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})returndiff --git a/server/sched.go b/server/sched.goindex 5f5cb51b8..e4bb15a55 100644--- a/server/sched.go+++ b/server/sched.go@@ -549,7 +549,9 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.DeflashAttention := llm.LlamaServerFlashAttention(loadGpus)req.applyAutomaticGenerationBatch(completion, predictedCtx, predicted, availableForBatch, flashAttention, loadGpus)launchOpts.NumBatch = req.opts.NumBatch- predictedForLoad := predicted + generationBatchSurchargeForCompletion(completion, launchOpts.NumBatch)+ var batchMemory uint64+ launchOpts.NumBatch, batchMemory = llm.LlamaServerBatch(f, launchOpts, completion)+ predictedForLoad := predicted + batchMemory// Pre-flight check: estimate whether the model fits in remaining memory.// llama-server auto-detects layers based on available VRAM, so if@@ -831,13 +833,6 @@ func (req *LlmRequest) applyAutomaticGenerationBatch(completion bool, effectiveCreq.opts.NumBatch = automaticGenerationBatch(effectiveCtx, predictedVRAM, availableMemory, flashAttention, gpus)}-func generationBatchSurchargeForCompletion(completion bool, batch int) uint64 {- if !completion {- return 0- }- return generationBatchSurcharge(batch)-}-func automaticGenerationBatch(effectiveCtx int, predictedVRAM, availableMemory uint64, flashAttention ml.FlashAttentionType, gpus []ml.DeviceInfo) int {if flashAttention == ml.FlashAttentionDisabled && hasCUDADevice(gpus) {if constrainedCUDAWithoutFlashAttention(effectiveCtx, gpus) {@@ -899,7 +894,7 @@ func generationBatchFits(batch int, predictedVRAM, availableMemory uint64) boolreturn false}- return generationBatchSurcharge(batch) <= threshold-predictedVRAM+ return llm.LlamaServerBatchMemory(batch) <= threshold-predictedVRAM}func generationBatchHasHeadroom(batch int, predictedVRAM, availableMemory uint64) bool {@@ -922,17 +917,6 @@ func nextLowerGenerationBatch(batch int) int {}}-func generationBatchSurcharge(batch int) uint64 {- switch {- case batch >= llamaServerGenerationBatchLarge:- return 2 * format.GibiByte- case batch >= llamaServerGenerationBatchMedium:- return 768 * format.MebiByte- default:- return 0- }-}-func nextLowerAutoNumCtx(numCtx int) (int, bool) {switch {case numCtx > 32768:diff --git a/server/systemone_test.go b/server/systemone_test.goindex d4f8f855e..cd29ab6d2 100644--- a/server/systemone_test.go+++ b/server/systemone_test.go@@ -28,6 +28,25 @@ type systemOneTestRunner struct {request llm.ScoreRequesterr errorcalls int+ native bool+ state json.RawMessage+}++func (r *systemOneTestRunner) SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (json.RawMessage, json.RawMessage, error) {+ r.calls+++ r.native = true+ r.state = state+ r.request = llm.ScoreRequest{Images: images}+ var ids map[string]json.RawMessage+ if err := json.Unmarshal(questions, &ids); err != nil {+ return nil, nil, err+ }+ answers := map[string]any{}+ for id := range ids {+ answers[id] = map[string]any{"type": "noul", "noul": 0.9}+ }+ data, _ := json.Marshal(answers)+ return data, json.RawMessage(`{"input_tokens":123,"output_tokens":0}`), r.err}func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest) (llm.ScoreResponse, error) {@@ -36,7 +55,15 @@ func (r *systemOneTestRunner) Score(ctx context.Context, input llm.ScoreRequest)if len(input.Fields) > 0 {return llm.ScoreResponse{Logits: [][]float32{{2, 0}}, InputTokens: 123}, r.err}- return llm.ScoreResponse{Logits: [][]float32{{0, 2}}, InputTokens: 123, OutputTokens: 2}, r.err+ logits := [][]float32{{0, 2}}+ if len(input.Rows) > 0 {+ logits = make([][]float32, len(input.Rows))+ for i, row := range input.Rows {+ logits[i] = make([]float32, len(row.Candidates))+ logits[i][len(row.Candidates)-1] = 2+ }+ }+ return llm.ScoreResponse{Logits: logits, InputTokens: 123, OutputTokens: 2}, r.err}func TestDecisionModelRejectsCompletion(t *testing.T) {@@ -95,61 +122,38 @@ func TestSystemOneHandler(t *testing.T) {config.Renderer = "qwen3.5"config.Capabilities = []string{"completion"}createSafetensorsTestModel(t, "safetensors-undeclared", config, nil)- for _, modelConfig := range []struct {- name, architecture, renderer, system, template string- contextLength int- undeclared bool- }{- {"renamed-clef", "qwen35", "", "Ignored for the joint head", "", 1024, false},- {"decision-test", "qwen35", "qwen3.5", "Model-specific scoring instructions.", "", 1024, false},- {"gguf-tev1", "qwen35", "tev1", "Model-specific scoring instructions.", "", 1024, false},- {"gguf-decision", "qwen35", "", "Native model scoring instructions.", "", 4096, false},- {"go-template", "qwen35", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024, false},- {"no-system", "qwen35", "qwen3.5", "", "", 1024, false},- {"gguf-undeclared", "qwen35", "qwen3.5", "", "", 1024, true},- {"gguf-other-architecture", "llama", "", "Model-specific scoring instructions.", "custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:", 1024, false},+ config.Capabilities = []string{"completion", "decision"}+ system, err := manifest.NewLayer(strings.NewReader("Model-specific scoring instructions."), "application/vnd.ollama.image.system")+ if err != nil {+ t.Fatal(err)+ }+ createSafetensorsTestModel(t, "decision-test", config, []manifest.Layer{params, system})+ goTemplate, err := manifest.NewLayer(strings.NewReader("custom:{{ range .Messages }}{{ .Role }}:{{ .Content }}\n{{ end }}answer:"), "application/vnd.ollama.image.template")+ if err != nil {+ t.Fatal(err)+ }+ config.Renderer = ""+ createSafetensorsTestModel(t, "go-template", config, []manifest.Layer{params, system, goTemplate})+ for _, gguf := range []struct{ name, architecture, decisionType, renderer string }{+ {"renamed-clef", "clef", "clef", ""},+ {"old-clef", "qwen35", "clef", ""},+ {"laya-native", "modern-bert", "laya", ""},+ {"gguf-tev1", "qwen35", "", "tev1"},} {- kv := gguftest.KV{"general.architecture": modelConfig.architecture}- if modelConfig.template == "" {- kv["tokenizer.chat_template"] = "{{ messages }}"+ kv := gguftest.KV{"general.architecture": gguf.architecture}+ caps := []string{"decision"}+ if gguf.decisionType != "" {+ kv[gguf.architecture+".decision.type"] = gguf.decisionType}- if modelConfig.name == "renamed-clef" {- kv[modelConfig.architecture+".decision.type"] = "clef"+ if gguf.decisionType == "clef" {+ caps = append(caps, "vision")}_, digest := createBinFile(t, kv, nil)- caps := []string{"completion", "decision"}- if modelConfig.name == "renamed-clef" {- caps = []string{"decision", "vision"}- }- if modelConfig.undeclared {- caps = []string{"completion"}- }- configLayer, err := createConfigLayer(model.ConfigV2{- ModelFormat: "gguf", ModelFamily: modelConfig.architecture,- Renderer: modelConfig.renderer, Capabilities: caps,- })+ configLayer, err := createConfigLayer(model.ConfigV2{ModelFormat: "gguf", ModelFamily: gguf.architecture, Renderer: gguf.renderer, Capabilities: caps})if err != nil {t.Fatal(err)}- layers := []manifest.Layer{{MediaType: "application/vnd.ollama.image.model", Digest: digest}}- for _, layer := range []struct{ content, mediaType string }{- {modelConfig.system, "application/vnd.ollama.image.system"},- {fmt.Sprintf(`{"num_ctx":%d}`, modelConfig.contextLength), "application/vnd.ollama.image.params"},- } {- l, err := manifest.NewLayer(strings.NewReader(layer.content), layer.mediaType)- if err != nil {- t.Fatal(err)- }- layers = append(layers, l)- }- if modelConfig.template != "" {- l, err := manifest.NewLayer(strings.NewReader(modelConfig.template), "application/vnd.ollama.image.template")- if err != nil {- t.Fatal(err)- }- layers = append(layers, l)- }- if err := manifest.WriteManifest(model.ParseName(modelConfig.name), *configLayer, layers); err != nil {+ if err := manifest.WriteManifest(model.ParseName(gguf.name), *configLayer, []manifest.Layer{{MediaType: "application/vnd.ollama.image.model", Digest: digest}}); err != nil {t.Fatal(err)}}@@ -160,33 +164,41 @@ func TestSystemOneHandler(t *testing.T) {image := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("image"), 16<<10))const imagePrefix = `{"model":"safetensors-clef","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}},"images":["`largeText := strings.Replace(prefix+strings.Repeat("x", stateLimit+1)+suffix, "decision-test", "safetensors-clef", 1)- for _, tt := range []struct {+ nativeQuestions := func(n int) string {+ questions := []string{`"refund":{"type":"noul","instructions":"q"}`}+ for i := 1; i < n; i++ {+ questions = append(questions, fmt.Sprintf(`"q%d":{"type":"noul","instructions":"q"}`, i))+ }+ return `{"model":"laya-native","state":"x","questions":{` + strings.Join(questions, ",") + `}}`+ }+ type testCase struct {name stringbody stringerr errorstatus intcalls intexpire bool- }{+ }+ tests := []testCase{{"success", `{"model":"decision-test","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},{"Clef image above text body limit", `{"model":"renamed-clef","state":"x","images":["` + base64.StdEncoding.EncodeToString(make([]byte, 70<<10)) + `"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},{"over image body limit", `{"model":"renamed-clef","images":["` + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},{"images require a supported encoding", `{"model":"decision-test","state":"x","images":["aW1hZ2U="],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"Clef videos unsupported", `{"model":"renamed-clef","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul"}}}`, nil, 400, 0, false},{"candidate videos unsupported", `{"model":"decision-test","state":"x","videos":["video.mp4"],"questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},- {"Clef null state and default instructions", `{"model":"renamed-clef","state":null,"questions":{"refund":{"type":"noul"}}}`, nil, 200, 1, false},- {"Clef joint head", `{"model":"renamed-clef","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},- {"GGUF success without renderer", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"Clef null state and default instructions", `{"model":"safetensors-clef","state":null,"questions":{"refund":{"type":"noul"}}}`, nil, 200, 1, false},+ {"Clef joint head", `{"model":"safetensors-clef","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},+ {"llama.cpp decision model", `{"model":"laya-native","state":{"total":1250.0},"questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},{"Modelfile template", `{"model":"go-template","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},- {"no system prompt", `{"model":"no-system","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, nil, 200, 1, false},- {"template failure", `{"model":"gguf-decision","state":"refund please","questions":{"refund":{"type":"noul","instructions":"Refund requested?"}}}`, errors.New("invalid model template"), 500, 0, false},- {"GGUF capability without Qwen architecture", `{"model":"gguf-other-architecture","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},- {"GGUF missing capability", `{"model":"gguf-undeclared","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"runner validation", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, api.StatusError{StatusCode: 400, ErrorMessage: "prompt too long"}, 400, 1, false},{"runner failure", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("runner failed"), 500, 1, false},{"runtime OOM", `{"model":"decision-test","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, errors.New("out of memory"), 500, 1, true},{"invalid schema", `{"model":"decision-test","state":"x","questions":{}}`, nil, 400, 0, false},- {"invalid Clef schema", `{"model":"renamed-clef","state":"x","questions":{}}`, nil, 400, 0, false},+ {"invalid Clef schema", `{"model":"safetensors-clef","state":"x","questions":{}}`, nil, 400, 0, false},+ {"old-format decision GGUF", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"old-format Clef GGUF", `{"model":"old-clef","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},+ {"llama.cpp 65 questions", nativeQuestions(65), nil, 400, 0, false},+ {"llama.cpp empty state", `{"model":"laya-native","state":" ","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"omitted model", `{"state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"blank model", `{"model":" ","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"missing model", `{"model":"missing","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 404, 0, false},@@ -201,27 +213,29 @@ func TestSystemOneHandler(t *testing.T) {{"invalid image base64", imagePrefix + `!not-base64"]}`, nil, 400, 0, false},{"image data URL", imagePrefix + `data:image/png;base64,aW1hZ2U="]}`, nil, 400, 0, false},{"Tev1 safetensors", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},- {"Tev1 GGUF", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"noul","instructions":"q"}}}`, nil, 200, 1, false},- {"Tev1 too many candidates", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"score","instructions":"q","criteria":["x"` + strings.Repeat(`,"x"`, 24) + `]}}}`, nil, 400, 0, false},- {"Tev1 empty description", `{"model":"gguf-tev1","state":"x","questions":{"refund":{"type":"choice","instructions":"q","criteria":{"a":"","b":"B"}}}}`, nil, 400, 0, false},+ {"Tev1 too many score levels", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"score","instructions":"q","criteria":["x"` + strings.Repeat(`,"x"`, 24) + `]}}}`, nil, 400, 0, false},+ {"Tev1 empty description", `{"model":"safetensors-tev1","state":"x","questions":{"refund":{"type":"choice","instructions":"q","criteria":{"a":"","b":"B"}}}}`, nil, 400, 0, false},{"safetensors missing capability", `{"model":"safetensors-undeclared","state":"x","questions":{"x":{"type":"noul","instructions":"q"}}}`, nil, 400, 0, false},{"bad JSON", `{`, nil, 400, 0, false},{"at text limit", prefix + strings.Repeat("<", stateLimit) + suffix, nil, 200, 1, false},{"over text limit", prefix + strings.Repeat("x", stateLimit+1) + suffix, nil, 413, 0, false},{"image does not relax text limit", strings.TrimSuffix(largeText, "}") + `,"images":["aW1hZ2U="]}`, nil, 413, 0, false},{"over transport limit", imagePrefix + strings.Repeat("A", 32<<20) + `"]}`, nil, 413, 0, false},- } {+ }+ for _, model := range []string{"decision-test", "laya-native"} {+ for _, levels := range []int{1, 2, 10, 11} {+ status, calls := http.StatusBadRequest, 0+ if levels >= 2 && levels <= 10 {+ status, calls = http.StatusOK, 1+ }+ criteria := `["x"` + strings.Repeat(`,"x"`, levels-1) + `]`+ body := fmt.Sprintf(`{"model":%q,"state":"x","questions":{"refund":{"type":"noul","instructions":"q"},"urgency":{"type":"score","instructions":"q","criteria":%s}}}`, model, criteria)+ tests = append(tests, testCase{fmt.Sprintf("score limits/%s/%d", model, levels), body, nil, status, calls, false})+ }+ }+ for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {runner := &systemOneTestRunner{err: tt.err}- runner.TemplateFn = func(_ context.Context, req llm.ChatRequest) (string, error) {- if len(req.Messages) != 2 || req.Messages[0].Role != "system" || req.Messages[0].Content != "Native model scoring instructions." || req.Messages[1].Role != "user" {- t.Fatalf("model system prompt was not passed to the native template: %+v", req.Messages)- }- if req.Think == nil || req.Think.Bool() {- t.Fatal("scoring must disable thinking")- }- return "native:" + req.Messages[1].Content, tt.err- }ref := &runnerRef{llama: runner, refCount: 1, sessionDuration: time.Hour}s := newServerWithMockRunner(t, &runner.mockRunner)s.sched.loadFn = func(req *LlmRequest, _ ml.SystemInfo, _ []ml.DeviceInfo, _ bool) bool {@@ -247,6 +261,19 @@ func TestSystemOneHandler(t *testing.T) {if tt.calls == 0 && tt.err == nil && ref.model != nil {t.Fatal("loaded a runner for a rejected request")}+ if strings.HasPrefix(tt.name, "score limits/") && tt.status == http.StatusBadRequest &&+ !strings.Contains(w.Body.String(), "score criteria must be an array of 2–10 levels") {+ t.Fatalf("expected shared score validation error: %s", w.Body)+ }+ if strings.HasPrefix(tt.name, "old-format ") {+ var sent decision.Request+ if err := json.Unmarshal([]byte(tt.body), &sent); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(w.Body.String(), "ollama pull "+sent.Model) {+ t.Fatalf("old decision GGUFs must ask for a re-download: %s", w.Body)+ }+ }if tt.name == "Clef decision only image" && !strings.Contains(w.Body.String(), "vision") {t.Fatalf("expected missing vision capability: %s", w.Body)}@@ -288,12 +315,18 @@ func TestSystemOneHandler(t *testing.T) {t.Fatal(err)}outputTokens := 2- if len(runner.request.Fields) > 0 {+ if len(runner.request.Fields) > 0 || runner.native {outputTokens = 0}if response.Answers["refund"].Noul < 0.88 || response.Usage.InputTokens != 123 || response.Usage.OutputTokens != outputTokens {t.Fatalf("incorrect scoring response: %s", w.Body)}+ if runner.native {+ if tt.name == "llama.cpp decision model" && (string(runner.state) != `{"total":1250.0}` || !strings.Contains(w.Body.String(), `"model":"laya-native"`)) {+ t.Fatalf("request did not pass through: state %s, response %s", runner.state, w.Body)+ }+ return+ }if ref.model.Config.Renderer == "strands" && (len(runner.request.PointerRows) != 1 || len(runner.request.Rows) != 0 || !strings.HasSuffix(runner.request.PointerRows[0].Prompt, "<answer>")) {t.Fatalf("Strands did not receive pointer inputs: %+v", runner.request)}@@ -304,17 +337,10 @@ func TestSystemOneHandler(t *testing.T) {if ref.model.Config.Renderer == "tev1" && (!strings.Contains(prompt, `"state": "x", "question": "q", "options":`) || strings.Contains(prompt, "Requested field:")) {t.Fatalf("Tev1 did not receive its per-question prompt: %q", prompt)}- wantContext := 1024- if !ref.model.isGGUF() {- wantContext = 8192- }if len(runner.request.Fields) > 0 {if len(runner.request.Rows) != 0 || len(runner.request.Fields) != 1 || len(runner.request.Segments) < 3 {t.Fatal("Clef scoring must preserve the segmented schema")}- if ref.model.isGGUF() {- wantContext = 2048- }} else if len(runner.request.PointerRows) > 0 {if len(runner.request.Rows) != 0 {t.Fatal("pointer head must bypass chat templates")@@ -323,11 +349,6 @@ func TestSystemOneHandler(t *testing.T) {if !strings.HasPrefix(prompt, "custom:system:Model-specific scoring instructions.\nuser:") || !strings.HasSuffix(prompt, "answer:") {t.Fatalf("scorer did not receive the Modelfile template output: %q", prompt)}- } else if ref.model.Config.Renderer == "" {- wantContext = 4096- if !strings.HasPrefix(prompt, `native:{"context":`) {- t.Fatalf("scorer did not receive the native template output: %q", prompt)- }} else {wantPrefix := "<|im_start|>user\n"if ref.model.System != "" {@@ -337,8 +358,8 @@ func TestSystemOneHandler(t *testing.T) {t.Fatalf("scorer did not receive the model's rendered prompt: %q", prompt)}}- if runner.request.MaxTokens != wantContext {- t.Fatalf("scoring context = %d, want model num_ctx %d", runner.request.MaxTokens, wantContext)+ if runner.request.MaxTokens != 8192 {+ t.Fatalf("scoring context = %d, want model num_ctx 8192", runner.request.MaxTokens)}}})