LLM inference in C/C++
C++ ggml
model: support vllm decision 2 model (#30378)
Xuan-Son Nguyen097cf33d9499
diff --git a/common/common.cpp b/common/common.cppindex 9a71a33ffe..9d2578424c 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1170,8 +1170,10 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA{ COMMON_DECISION_TYPE_LAYA, "laya" },{ COMMON_DECISION_TYPE_CLEF, "clef" },{ COMMON_DECISION_TYPE_PPLX_DECIDER, "pplx-decider" },+ { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },+ // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"};static common_decision_type common_decision_type_from_string(const std::string & str) {@@ -1289,7 +1291,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)// TODO: maybe improve this in the futureconst auto decision_type = common_get_decision_type(model);if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||- decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {+ decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {params.embedding = true;params.pooling_type = LLAMA_POOLING_TYPE_NONE;diff --git a/common/common.h b/common/common.hindex d89f803af9..9513a04fa9 100644--- a/common/common.h+++ b/common/common.h@@ -963,6 +963,7 @@ enum common_decision_type {COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings outputCOMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row iCOMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters+ COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batchCOMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question typeCOMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layoutCOMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supporteddiff --git a/conversion/__init__.py b/conversion/__init__.pyindex 2a62dbac99..c2a8fb92ce 100644--- a/conversion/__init__.py+++ b/conversion/__init__.py@@ -291,6 +291,7 @@ TEXT_MODEL_MAP: dict[str, str] = {"UltravoxModel": "ultravox","UnlimitedOCRForCausalLM": "deepseek","VLlama3ForCausalLM": "llama",+ "VllmDecision2Model": "vllm_decision2","VoxtralForConditionalGeneration": "llama","WavTokenizerDec": "wavtokenizer","XLMRobertaForSequenceClassification": "bert",diff --git a/conversion/vllm_decision2.py b/conversion/vllm_decision2.pynew file mode 100644index 0000000000..3e7085f14e--- /dev/null+++ b/conversion/vllm_decision2.py@@ -0,0 +1,107 @@+from __future__ import annotations++import json++from pathlib import Path+from typing import Any, Callable, Iterator, TYPE_CHECKING++if TYPE_CHECKING:+ from torch import Tensor++from .base import ModelBase, gguf, logger+from .qwen import Qwen3Model+++def _is_vllm_decision2_checkpoint(dir_model: Path) -> bool:+ if not (dir_model / "config.json").is_file():+ return False+ with open(dir_model / "config.json", encoding="utf-8") as f:+ return json.load(f).get("decision_format") == "vllm-sr-decision"++[email protected]_hparams_loader(_is_vllm_decision2_checkpoint)+def _load_vllm_decision2_hparams(dir_model: Path) -> dict[str, Any]:+ logger.info("gguf: detected vllm-sr Decision 2.0 checkpoint")+ with open(dir_model / "config.json", encoding="utf-8") as f:+ package = json.load(f)+ hparams = ModelBase.load_hparams((dir_model / package["backbone"]["config"]).parent, False, guess=False)+ hparams["architectures"] = ["VllmDecision2Model"]+ with open(dir_model / package["model_config"], encoding="utf-8") as f:+ hparams["decision"] = json.load(f)+ with open(dir_model / "score_bias.json", encoding="utf-8") as f:+ hparams["decision"]["score_bias"] = json.load(f)["offsets"]+ return hparams++[email protected]("VllmDecision2Model")[email protected]("vllm-sr/Decision-2.0-Kai-0.6B")+class VllmDecision2Model(Qwen3Model):+ model_arch = gguf.MODEL_ARCH.VLLM_DECISION2++ # torch.nn.LayerNorm default, used by the head+ _HEAD_NORM_EPS = 1e-5++ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ head = self.hparams["decision"]+ if head["architecture"] != "qwen3-text-endpoints-global-query-shared-bilinear-mlp" or head["head_variant"] != "shared":+ raise ValueError(f"unsupported decision head: {head['architecture']}")++ def index_tensors(self, *args, **kwargs) -> dict[str, Callable[[], Tensor]]:+ # the backbone weights are in their own directory+ dir_model, self.dir_model = self.dir_model, self.dir_model / "backbone"+ try:+ return super().index_tensors(*args, **kwargs)+ finally:+ self.dir_model = dir_model++ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:+ yield from super().get_tensors()+ from safetensors.torch import load_file+ for name, data in load_file(self.dir_model / "decision_head.safetensors").items():+ yield "decision_head." + name, data++ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:+ # the head is computed in F32+ if new_name.startswith("decision."):+ return gguf.GGMLQuantizationType.F32+ return super().tensor_force_quant(name, new_name, bid, n_dims)++ def set_vocab(self):+ super().set_vocab()+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])++ @classmethod+ def _systemone_template(cls) -> str:+ # prompt follows decision2/_vendor/dev2model/decision_model.py (segments) of the model repo+ # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)+ # and the text that starts the span of a question or of an option (mark_question, mark_option)+ # the head reads the last token of each option and the last token of the question+ # the start of an option is a piece of its own, so that the spans of two options are not joined+ # the keys of JSON objects are given in sorted order+ def text(value: str) -> str:+ return "{{ " + json.dumps(value) + " }}"++ def render(name: str) -> str:+ return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"++ option = (+ "{% set d = o.description %}"+ "{% if q.type == 'noul' and d is none %}{% set d = 'Yes' if o.key == 'true' else 'No' %}{% endif %}"+ "{{ {'description': d, 'key': o.key} | tojson(separators=[',', ':']) }}"+ )+ return (+ "{% for q in questions %}"+ + text("Context:\n") + render("state")+ + text("\n\nTask type: ") + "{{ q.type }}" + text("\nQuestion:\n") + render("q.instructions") + text("\nOptions:")+ + "{% for o in q.options %}{{ sep }}" + text("\n<option>\n") + "{{ sep }}{{ mark_option }}" + option + text("\n</option>") + "{% endfor %}"+ + "{{ sep }}{{ mark_question }}" + text("\n\nSelect the single option best supported by the context and instructions.\nDecision:")+ + "{% endfor %}"+ )++ def set_gguf_parameters(self):+ super().set_gguf_parameters()+ self.gguf_writer.add_decision_type(gguf.DecisionType.VLLM_DECISION2)+ self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)+ for offsets in self.hparams["decision"]["score_bias"].values():+ self.gguf_writer.add_decision_score_bias(offsets)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 5adb402050..a569afc96c 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -332,6 +332,7 @@ class Keys:HEAD_COUNT = "{arch}.decision.head_count"MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"+ SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levelsclass Tokenizer:MODEL = "tokenizer.ggml.model"@@ -559,6 +560,7 @@ class MODEL_ARCH(IntEnum):QWEN35 = auto()QWEN35MOE = auto()CLEF = auto()+ VLLM_DECISION2 = auto()QWEN4EXP = auto()PHI2 = auto()PHI3 = auto()@@ -923,6 +925,11 @@ class MODEL_TENSOR(IntEnum):DECISION_SCALES = auto()DECISION_SCORER = auto()DECISION_SCORER_OUT = auto()+ DECISION_QUERY_NORM = auto()+ DECISION_KEY = auto()+ DECISION_QUERY = auto()+ DECISION_OPTION_MLP = auto()+ DECISION_QUERY_MLP = auto()CONV1D = auto()CONVNEXT_DW = auto()CONVNEXT_NORM = auto()@@ -1343,6 +1350,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {MODEL_ARCH.QWEN35: "qwen35",MODEL_ARCH.QWEN35MOE: "qwen35moe",MODEL_ARCH.CLEF: "clef",+ MODEL_ARCH.VLLM_DECISION2: "vllm-decision2",MODEL_ARCH.QWEN4EXP: "qwen4exp",MODEL_ARCH.PHI2: "phi2",MODEL_ARCH.PHI3: "phi3",@@ -1706,6 +1714,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {MODEL_TENSOR.DECISION_SCALES: "decision.scales",MODEL_TENSOR.DECISION_SCORER: "decision.scorer",MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",+ MODEL_TENSOR.DECISION_QUERY_NORM: "decision.query_norm",+ MODEL_TENSOR.DECISION_KEY: "decision.key",+ MODEL_TENSOR.DECISION_QUERY: "decision.query",+ MODEL_TENSOR.DECISION_OPTION_MLP: "decision.option_mlp",+ MODEL_TENSOR.DECISION_QUERY_MLP: "decision.query_mlp",MODEL_TENSOR.CONV1D: "conv1d",MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",@@ -3034,6 +3047,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {MODEL_TENSOR.DECISION_SCORER,MODEL_TENSOR.DECISION_SCORER_OUT,],+ MODEL_ARCH.VLLM_DECISION2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ # decision head+ MODEL_TENSOR.DECISION_OPTION_NORM,+ MODEL_TENSOR.DECISION_QUERY_NORM,+ MODEL_TENSOR.DECISION_KEY,+ MODEL_TENSOR.DECISION_QUERY,+ MODEL_TENSOR.DECISION_OPTION_MLP,+ MODEL_TENSOR.DECISION_QUERY_MLP,+ MODEL_TENSOR.DECISION_SCORER_OUT,+ ],MODEL_ARCH.QWEN35MOE: [MODEL_TENSOR.TOKEN_EMBD,MODEL_TENSOR.OUTPUT_NORM,@@ -6100,6 +6136,7 @@ class DecisionType:NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the requestCLEF = "clef" # joint head over all questions, one score per optionPPLX_DECIDER = "pplx-decider" # same as openjev, label codes of 1 or 2 letters+ VLLM_DECISION2 = "vllm-decision2" # head over the end token of each option and the last token, one score per optionLFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question typeLFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layoutdiff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.pyindex 86d4d6e2f6..cda59a6285 100644--- a/gguf-py/gguf/gguf_writer.py+++ b/gguf-py/gguf/gguf_writer.py@@ -1365,6 +1365,10 @@ class GGUFWriter:def add_decision_temperature(self, name: str, value: float) -> None:self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)+ def add_decision_score_bias(self, values: Sequence[float]) -> None:+ for i, value in enumerate(values):+ self.add_float32(Keys.Decision.SCORE_BIAS.format(arch=self.arch, n=len(values), i=i), value)+# for vision modelsdef add_clip_has_vision_encoder(self, value: bool) -> None:diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.pyindex 566b7348a7..6445784a35 100644--- a/gguf-py/gguf/tensor_mapping.py+++ b/gguf-py/gguf/tensor_mapping.py@@ -1328,7 +1328,8 @@ class TensorNameMap:),MODEL_TENSOR.DECISION_OPTION_NORM: (- "joint_head.option_norm", # clef+ "joint_head.option_norm", # clef+ "decision_head.candidate_norm", # vllm-decision2),MODEL_TENSOR.DECISION_SCORER: (@@ -1337,6 +1338,27 @@ class TensorNameMap:MODEL_TENSOR.DECISION_SCORER_OUT: ("joint_head.residual_scorer.3", # clef+ "decision_head.scalar", # vllm-decision2+ ),++ MODEL_TENSOR.DECISION_QUERY_NORM: (+ "decision_head.query_norm", # vllm-decision2+ ),++ MODEL_TENSOR.DECISION_KEY: (+ "decision_head.key", # vllm-decision2+ ),++ MODEL_TENSOR.DECISION_QUERY: (+ "decision_head.query", # vllm-decision2+ ),++ MODEL_TENSOR.DECISION_OPTION_MLP: (+ "decision_head.candidate_mlp", # vllm-decision2+ ),++ MODEL_TENSOR.DECISION_QUERY_MLP: (+ "decision_head.query_mlp", # vllm-decision2),MODEL_TENSOR.ENC_ATTN_NORM: (diff --git a/src/llama-arch.cpp b/src/llama-arch.cppindex 88eab7b4dc..0dc4cc961b 100644--- a/src/llama-arch.cpp+++ b/src/llama-arch.cpp@@ -41,6 +41,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_QWEN35, "qwen35" },{ LLM_ARCH_QWEN35MOE, "qwen35moe" },{ LLM_ARCH_CLEF, "clef" },+ { LLM_ARCH_VLLM_DECISION2, "vllm-decision2" },{ LLM_ARCH_QWEN4EXP, "qwen4exp" },{ LLM_ARCH_PHI2, "phi2" },{ LLM_ARCH_PHI3, "phi3" },@@ -524,6 +525,11 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {{ LLM_TENSOR_DECISION_SCALES, "decision.scales" },{ LLM_TENSOR_DECISION_SCORER, "decision.scorer" },{ LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" },+ { LLM_TENSOR_DECISION_QUERY_NORM, "decision.query_norm" },+ { LLM_TENSOR_DECISION_KEY, "decision.key" },+ { LLM_TENSOR_DECISION_QUERY, "decision.query" },+ { LLM_TENSOR_DECISION_OPTION_MLP, "decision.option_mlp" },+ { LLM_TENSOR_DECISION_QUERY_MLP, "decision.query_mlp" },{ LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },{ LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },{ LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" },@@ -789,6 +795,11 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {{LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},{LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},{LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DECISION_QUERY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},+ {LLM_TENSOR_DECISION_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DECISION_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DECISION_OPTION_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},+ {LLM_TENSOR_DECISION_QUERY_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},{LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output{LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output{LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},diff --git a/src/llama-arch.h b/src/llama-arch.hindex 2d8ad6b077..8f8cc27ed2 100644--- a/src/llama-arch.h+++ b/src/llama-arch.h@@ -46,6 +46,7 @@ enum llm_arch {LLM_ARCH_QWEN35,LLM_ARCH_QWEN35MOE,LLM_ARCH_CLEF,+ LLM_ARCH_VLLM_DECISION2,LLM_ARCH_QWEN4EXP,LLM_ARCH_PHI2,LLM_ARCH_PHI3,@@ -695,6 +696,11 @@ enum llm_tensor {LLM_TENSOR_DECISION_SCALES,LLM_TENSOR_DECISION_SCORER,LLM_TENSOR_DECISION_SCORER_OUT,+ LLM_TENSOR_DECISION_QUERY_NORM,+ LLM_TENSOR_DECISION_KEY,+ LLM_TENSOR_DECISION_QUERY,+ LLM_TENSOR_DECISION_OPTION_MLP,+ LLM_TENSOR_DECISION_QUERY_MLP,LLM_TENSOR_CONV1D,LLM_TENSOR_CONVNEXT_DW,LLM_TENSOR_CONVNEXT_NORM,diff --git a/src/llama-ext.h b/src/llama-ext.hindex fcc78f47e4..faf4beb696 100644--- a/src/llama-ext.h+++ b/src/llama-ext.h@@ -100,7 +100,7 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value// chain multiple trained NextN heads. Default 0 (first head).LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);-// Marks the entries that a joint decision head (clef) reads, the default is 0+// Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0// See https://github.com/ggml-org/llama.cpp/pull/29831 for details// A run of entries with the same value is one span, spans must be separated by entries with value 0// An option belongs to the last question before itdiff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cppindex 1c7cbd9005..47e36d3743 100644--- a/src/llama-model-saver.cpp+++ b/src/llama-model-saver.cpp@@ -21,6 +21,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {case LLM_ARCH_APERTUS:case LLM_ARCH_STEP35:case LLM_ARCH_CLEF: // the head tensors are not saved+ case LLM_ARCH_VLLM_DECISION2:return false;default:return true;diff --git a/src/llama-model.cpp b/src/llama-model.cppindex b3e88322b0..6872589fa4 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -330,6 +330,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_paramsreturn new llama_model_qwen35moe(params);case LLM_ARCH_CLEF:return new llama_model_clef(params);+ case LLM_ARCH_VLLM_DECISION2:+ return new llama_model_vllm_decision2(params);case LLM_ARCH_QWEN4EXP:return new llama_model_qwen4exp(params);case LLM_ARCH_MISTRAL3:@@ -3534,6 +3536,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {case LLM_ARCH_DREAM:case LLM_ARCH_QWEN2MOE:case LLM_ARCH_QWEN3:+ case LLM_ARCH_VLLM_DECISION2:case LLM_ARCH_QWEN3MOE:case LLM_ARCH_LLADA_MOE:case LLM_ARCH_RND1:diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex 821b4798c2..9c60667963 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -346,6 +346,11 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * paramquantize &= name.find("attn_v_b.weight") == std::string::npos;}+ // the decision head of vllm-decision2 is small and computed in F32+ if (arch == LLM_ARCH_VLLM_DECISION2) {+ quantize &= name.compare(0, 9, "decision.") != 0;+ }+// do not quantize RWKV's small yet 2D weightsquantize &= name.find("time_mix_first.weight") == std::string::npos;quantize &= name.find("time_mix_w0.weight") == std::string::npos;diff --git a/src/models/models.h b/src/models/models.hindex d6ddf9d167..a55983cfbc 100644--- a/src/models/models.h+++ b/src/models/models.h@@ -583,6 +583,44 @@ struct llama_model_qwen3 : public llama_model_base {struct graph : public llm_graph_context {graph(const llama_model & model, const llm_graph_params & params);+ protected:+ // the helpers alone, a derived graph builds its own body+ struct no_build {};+ graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {}++ // the layers and the final norm: [n_embd, n_outputs]+ ggml_tensor * build_backbone(const llama_model & model);+ };++ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;+};++// Qwen3 with the decision head of vllm-sr Decision 2.0: one score per option+struct llama_model_vllm_decision2 : public llama_model_qwen3 {+ llama_model_vllm_decision2(const struct llama_model_params & params) : llama_model_qwen3(params) {}+ void load_arch_hparams(llama_model_loader & ml) override;+ void load_arch_tensors(llama_model_loader & ml) override;++ ggml_tensor * option_norm = nullptr;+ ggml_tensor * option_norm_b = nullptr;+ ggml_tensor * query_norm = nullptr;+ ggml_tensor * query_norm_b = nullptr;+ ggml_tensor * key = nullptr;+ ggml_tensor * query = nullptr;+ ggml_tensor * option_mlp = nullptr;+ ggml_tensor * option_mlp_b = nullptr;+ ggml_tensor * query_mlp = nullptr;+ ggml_tensor * scorer_out = nullptr;++ class input_decision;++ struct graph : public llama_model_qwen3::graph {+ graph(const llama_model & model, const llm_graph_params & params);+ private:+ // hidden: [n_embd, n_outputs], returns one score per option: [1, n_options]+ ggml_tensor * build_head(ggml_tensor * hidden, input_decision * inp);++ const llama_model_vllm_decision2 & model;};std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cppindex f4b2a2aebe..a565e3e688 100644--- a/src/models/qwen3.cpp+++ b/src/models/qwen3.cpp@@ -51,6 +51,18 @@ std::unique_ptr<llm_graph_context> llama_model_qwen3::build_arch_graph(const llm}llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {+ ggml_tensor * cur = build_backbone(model);++ // lm_head+ cur = build_lora_mm(model.output, cur, model.output_s);++ cb(cur, "result_output", -1);+ res->t_logits = cur;++ ggml_build_forward_expand(gf, cur);+}++ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model) {const int64_t n_embd_head = hparams.n_embd_head_v();GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());@@ -149,11 +161,5 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_paramcb(cur, "result_norm", -1);res->t_embd = cur;- // lm_head- cur = build_lora_mm(model.output, cur, model.output_s);-- cb(cur, "result_output", -1);- res->t_logits = cur;-- ggml_build_forward_expand(gf, cur);+ return cur;}diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cppnew file mode 100644index 0000000000..bbac808de1--- /dev/null+++ b/src/models/vllm-decision2.cpp@@ -0,0 +1,179 @@+#include "models.h"++#include "llama-ext.h"++#include <cmath>++void llama_model_vllm_decision2::load_arch_hparams(llama_model_loader & ml) {+ llama_model_qwen3::load_arch_hparams(ml);++ // used by the head+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);++ // the output is one score per token, see llama_batch_ext_set_decision_order()+ hparams.n_embd_out_impl = 1;+}++void llama_model_vllm_decision2::load_arch_tensors(llama_model_loader & ml) {+ llama_model_qwen3::load_arch_tensors(ml);++ LLAMA_LOAD_LOCALS;++ const auto * w_key = ml.get_weight(tn(LLM_TENSOR_DECISION_KEY, "weight").str().c_str());+ if (w_key == nullptr) {+ throw std::runtime_error("the decision head is missing");+ }+ const int64_t n_embd_h = w_key->tensor->ne[1];++ option_norm = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "weight"), {n_embd}, 0);+ option_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "bias"), {n_embd}, 0);+ query_norm = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "weight"), {n_embd}, 0);+ query_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "bias"), {n_embd}, 0);++ key = create_tensor(tn(LLM_TENSOR_DECISION_KEY, "weight"), {n_embd, n_embd_h}, 0);+ query = create_tensor(tn(LLM_TENSOR_DECISION_QUERY, "weight"), {n_embd, n_embd_h}, 0);+ option_mlp = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "weight"), {n_embd, n_embd_h}, 0);+ option_mlp_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "bias"), {n_embd_h}, 0);+ query_mlp = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_MLP, "weight"), {n_embd, n_embd_h}, 0);+ scorer_out = create_tensor(tn(LLM_TENSOR_DECISION_SCORER_OUT, "weight"), {n_embd_h, 1}, 0);+}++std::unique_ptr<llm_graph_context> llama_model_vllm_decision2::build_arch_graph(const llm_graph_params & params) const {+ return std::make_unique<graph>(*this, params);+}++// output rows read by the head: the last token of each option span and the last token of the question span+struct decision2_rows {+ std::vector<int32_t> options;+ int32_t query = 0;+ bool valid = false;+};++// see llama_batch_ext_set_decision_order()+// if the batch has no usable order, returns one option at row 0+static decision2_rows decision2_get_rows(const llama_ubatch & ubatch) {+ decision2_rows res;++ const int32_t n_tokens = ubatch.n_tokens;+ bool ok = ubatch.decision_order != nullptr && ubatch.n_seqs_unq == 1;++ // output row of each token, -1 if the token has no output+ std::vector<int32_t> rows(n_tokens, -1);+ for (int32_t i = 0, n = 0; i < n_tokens; i++) {+ if (ubatch.output[i]) {+ rows[i] = n++;+ }+ }++ int32_t n_questions = 0;+ for (int32_t i = 0; ok && i < n_tokens;) {+ const int32_t order = ubatch.decision_order[i];+ int32_t end = i + 1;+ while (end < n_tokens && ubatch.decision_order[end] == order) {+ end++;+ }+ switch (order) {+ case LLAMA_DECISION_ORDER_NONE:+ break;+ case LLAMA_DECISION_ORDER_QUESTION_NOUL:+ case LLAMA_DECISION_ORDER_QUESTION_CHOICE:+ case LLAMA_DECISION_ORDER_QUESTION_SCORE:+ res.query = rows[end - 1];+ n_questions++;+ break;+ case LLAMA_DECISION_ORDER_OPTION:+ res.options.push_back(rows[end - 1]);+ break;+ default:+ ok = false;+ }+ ok = ok && (order == LLAMA_DECISION_ORDER_NONE || rows[end - 1] >= 0);+ i = end;+ }++ res.valid = ok && n_questions == 1 && !res.options.empty();+ if (!res.valid) {+ res.options.assign(1, 0);+ res.query = 0;+ }+ return res;+}++class llama_model_vllm_decision2::input_decision : public llm_graph_input_i {+public:+ input_decision(const llama_ubatch & ubatch) : n_options(decision2_get_rows(ubatch).options.size()) {}++ void set_input(const llama_ubatch * ubatch) override {+ const auto rows = decision2_get_rows(*ubatch);+ GGML_ASSERT(rows.options.size() == n_options);++ // the scores are NaN if the batch has a decision order that cannot be used+ const float status_data = rows.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;++ ggml_backend_tensor_set(options, rows.options.data(), 0, ggml_nbytes(options));+ ggml_backend_tensor_set(query, &rows.query, 0, ggml_nbytes(query));+ ggml_backend_tensor_set(status, &status_data, 0, ggml_nbytes(status));+ }++ bool can_reuse(const llm_graph_params & params) override {+ // the values are computed again in set_input(), only the shapes must match+ return decision2_get_rows(params.ubatch).options.size() == n_options;+ }++ ggml_tensor * options = nullptr; // I32 [n_options]+ ggml_tensor * query = nullptr; // I32 [1]+ ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input++ const size_t n_options;+};++llama_model_vllm_decision2::graph::graph(const llama_model & model_base, const llm_graph_params & params) :+ llama_model_qwen3::graph(params, no_build()), model(static_cast<const llama_model_vllm_decision2 &>(model_base)) {+ auto inp_decision_ptr = std::make_unique<input_decision>(ubatch);+ auto * inp_decision = inp_decision_ptr.get();+ res->add_input(std::move(inp_decision_ptr));++ ggml_tensor * cur = build_backbone(model);++ // the head is always evaluated, so that the graph has the same nodes for every batch+ cur = build_head(cur, inp_decision);++ // row i of the output is the score of option i+ cur = ggml_pad(ctx0, cur, 0, n_outputs - cur->ne[1], 0, 0);+ cur = ggml_add(ctx0, cur, inp_decision->status);+ cb(cur, "result_decision", -1);++ res->t_embd = cur;++ ggml_build_forward_expand(gf, cur);+}++ggml_tensor * llama_model_vllm_decision2::graph::build_head(ggml_tensor * hidden, input_decision * inp) {+ inp->options = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, inp->n_options);+ inp->query = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);+ inp->status = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);+ for (ggml_tensor * t : { inp->options, inp->query, inp->status }) {+ ggml_set_input(t);+ }++ ggml_tensor * options = ggml_get_rows(ctx0, hidden, inp->options); // [n_embd, n_options]+ ggml_tensor * query = ggml_get_rows(ctx0, hidden, inp->query); // [n_embd, 1]++ options = build_norm(options, model.option_norm, model.option_norm_b, LLM_NORM, -1);+ query = build_norm(query, model.query_norm, model.query_norm_b, LLM_NORM, -1);++ // scaled dot product of the option key and the query+ ggml_tensor * k = ggml_mul_mat(ctx0, model.key, options); // [n_embd_h, n_options]+ ggml_tensor * q = ggml_mul_mat(ctx0, model.query, query); // [n_embd_h, 1]+ ggml_tensor * bilinear = ggml_sum_rows(ctx0, ggml_mul(ctx0, k, q));+ bilinear = ggml_scale(ctx0, bilinear, 1.0f / sqrtf((float) k->ne[0])); // [1, n_options]++ ggml_tensor * mlp = ggml_add(ctx0, ggml_mul_mat(ctx0, model.option_mlp, options), model.option_mlp_b);+ mlp = ggml_add(ctx0, mlp, ggml_mul_mat(ctx0, model.query_mlp, query));+ mlp = ggml_gelu_erf(ctx0, mlp);+ ggml_tensor * nonlinear = ggml_mul_mat(ctx0, model.scorer_out, mlp); // [1, n_options]++ ggml_tensor * scores = ggml_add(ctx0, bilinear, nonlinear);+ cb(scores, "decision_scores", -1);+ return scores;+}diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cppindex 78128ebf03..1e8e915b15 100644--- a/tests/test-llama-archs.cpp+++ b/tests/test-llama-archs.cpp@@ -757,7 +757,7 @@ static bool arch_supported(const llm_arch arch) {if (arch == LLM_ARCH_PLM) {return false; // TODO tensor shapes}- if (arch == LLM_ARCH_CLEF) {+ if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) {return false; // TODO decision head tensors}if (arch == LLM_ARCH_DEEPSEEK2OCR) {diff --git a/tools/server/README.md b/tools/server/README.mdindex 0c66504123..c7cf50d5a1 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1718,9 +1718,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppoThe questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.-For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. A server that runs clef only serves this endpoint, text generation is not available.+For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.*Image input:*diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex 5f77d43d05..9f43e7a2c9 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -156,6 +156,7 @@ std::vector<std::string> server_model_output_modalities(common_decision_type deccase COMMON_DECISION_TYPE_LAYA:case COMMON_DECISION_TYPE_CLEF:case COMMON_DECISION_TYPE_PPLX_DECIDER:+ case COMMON_DECISION_TYPE_VLLM_DECISION2:case COMMON_DECISION_TYPE_LFM2_D1:case COMMON_DECISION_TYPE_LFM2_D1_OMNI:return {"decisions"};diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 2a25b75d83..3014f52cfe 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -3604,6 +3604,11 @@ private:n_past = std::min(n_past, slot.task->n_tokens_shared);}+ // the outputs of a decision are read from the batch, so its first position must be evaluated+ if (n_decision_first >= 0) {+ n_past = std::min(n_past, n_decision_first);+ }+// if there is an alora invoked, don't cache after the invocation startif (slot.alora_invocation_start > 0) {SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start);diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cppindex 11ad66a19e..6b7d2c16eb 100644--- a/tools/server/server-decision.cpp+++ b/tools/server/server-decision.cpp@@ -66,6 +66,28 @@ void server_decision_context::init(const llama_model * model) {temperatures[key + prefix_temp.size()] = temp;}+ // "<n_levels>.<level>" -> offset+ const std::string prefix_bias = prefix + "score_bias.";+ for (int32_t i = 0; i < llama_model_meta_count(model); i++) {+ char key[256];+ char val[64];+ if (llama_model_meta_key_by_index(model, i, key, sizeof(key)) < 0 || !string_starts_with(key, prefix_bias)) {+ continue;+ }+ if (llama_model_meta_val_str_by_index(model, i, val, sizeof(val)) < 0) {+ continue;+ }+ const auto parts = string_split<std::string>(key + prefix_bias.size(), '.');+ const size_t n = parts.size() == 2 ? std::strtoul(parts[0].c_str(), nullptr, 10) : 0;+ const size_t level = parts.size() == 2 ? std::strtoul(parts[1].c_str(), nullptr, 10) : 0;+ if (n < 2 || n > 255 || level >= n) {+ throw std::runtime_error(string_format("invalid decision score bias: %s", key));+ }+ auto & bias = score_bias[n];+ bias.resize(n, 0.0f);+ bias[level] = std::strtof(val, nullptr);+ }+if (model_type == COMMON_DECISION_TYPE_OPENJEV) {// one letter per option, each must be a single tokenconst std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";@@ -123,6 +145,10 @@ void server_decision_context::init(const llama_model * model) {n_options_max = 255;noul_true_first = true;choice_sorted = true;+ } else if (model_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {+ n_options_max = 255;+ noul_criteria_order = true;+ strict_payload = true;} else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) {n_options_max = 255;noul_true_first = true;@@ -144,11 +170,18 @@ void server_decision_context::init(const llama_model * model) {// request parsing//+static bool decision_is_payload(const json & val) {+ return val.is_string() || val.is_object() || val.is_array();+}+std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {// lfm2-d1 and lfm2-d1-omni accept a null state, for example to ask about images onlyif (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI)) {throw std::invalid_argument("\"state\" must be provided");}+ if (strict_payload && !decision_is_payload(body.at("state"))) {+ throw std::invalid_argument("\"state\" must be text, an object or an array");+ }if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) {throw std::invalid_argument("\"questions\" must be a non-empty object");}@@ -164,6 +197,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(cif (!q.contains("instructions") || q.at("instructions").is_null()) {throw err("\"instructions\" must be provided");}+ if (strict_payload && (!decision_is_payload(q.at("instructions")) || q.at("instructions") == "")) {+ throw err("\"instructions\" must be non-empty text, an object or an array");+ }server_decision_question question;question.id = id;@@ -178,8 +214,14 @@ std::vector<server_decision_question> server_decision_context::parse_questions(cthrow err("\"criteria\" must be a non-empty object");}for (const auto & [key, description] : criteria.items()) {+ if (strict_payload && (key.empty() || !(description.is_null() || decision_is_payload(description)))) {+ throw err("option keys must be non-empty, descriptions must be null, text, an object or an array");+ }question.options.push_back({key, description});}+ if (strict_payload && question.options.size() < 2) {+ throw err("\"criteria\" must have at least 2 options");+ }if (choice_sorted) {std::sort(question.options.begin(), question.options.end(), [](const auto & a, const auto & b) {return a.key < b.key;@@ -191,6 +233,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(cthrow err("\"criteria\" must be an array of 2 to 10 levels");}for (size_t i = 0; i < criteria.size(); i++) {+ if (strict_payload && !decision_is_payload(criteria.at(i))) {+ throw err("level descriptions must be text, an object or an array");+ }question.options.push_back({std::to_string(i), criteria.at(i)});}} else if (type_name == "noul") {@@ -198,6 +243,13 @@ std::vector<server_decision_question> server_decision_context::parse_questions(cif (!criteria.is_null() && !criteria.is_object()) {throw err("\"criteria\" must be an object");}+ if (strict_payload && criteria.is_object()) {+ for (const auto & [key, description] : criteria.items()) {+ if ((key != "false" && key != "true") || !decision_is_payload(description)) {+ throw err("\"criteria\" may only describe \"false\" and \"true\" with text, an object or an array");+ }+ }+ }for (const char * key : {"false", "true"}) {json description;if (criteria.is_object() && criteria.contains(key)) {@@ -209,7 +261,7 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c}question.options.push_back({key, description});}- if (noul_true_first) {+ if (noul_true_first || (noul_criteria_order && criteria.is_object() && criteria.size() == 2 && criteria.begin().key() == "true")) {std::swap(question.options[0], question.options[1]);}} else {@@ -658,6 +710,11 @@ void server_decision_context::fill_task(mtmd_context * mctx,const mtmd_helper_init_opt & init_opt,server_task & task) const {+ // each question has its own prompt, with the spans that the head reads+ if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) {+ fill_task_joint(state, { question }, files, mctx, init_opt, task);+ return;+ }if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {fill_task_d1omni(state, questions, question, variant, files, mctx, init_opt, task);return;@@ -1056,6 +1113,32 @@ static double decision_confidence_score(const std::vector<double> & probs) {return std::max(0.0, 1.0 - dist / dist_uniform);}+// VLLM_DECISION2: normalized entropy, as in the reference implementation+static double decision_confidence_entropy(const std::vector<double> & probs) {+ if (probs.size() < 2) {+ return 1.0;+ }+ double entropy = 0.0;+ for (const double p : probs) {+ if (p > 0.0) {+ entropy -= p * std::log(p);+ }+ }+ return std::max(0.0, std::min(1.0, 1.0 - entropy / std::log((double) probs.size())));+}++double server_decision_context::get_decision_confidence(server_decision_question_type question_type, const std::vector<double> & probs) const {+ if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) {+ return decision_confidence_entropy(probs);+ }+ switch (question_type) {+ case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs);+ case SERVER_DECISION_QUESTION_SCORE: return decision_confidence_score(probs);+ case SERVER_DECISION_QUESTION_NOUL: break;+ }+ throw std::runtime_error("no confidence for this question type");+}+json server_decision_context::format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media) const {const size_t n = n_outputs(question);if (scores.size() != n_variants(question)) {@@ -1075,11 +1158,17 @@ json server_decision_context::format_answer(const server_decision_question & queif (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) {throw std::runtime_error("the model could not evaluate the decision");}- const float score_max = *std::max_element(s.begin(), s.end());+ std::vector<double> logits(s.begin(), s.end());+ if (question.type == SERVER_DECISION_QUESTION_SCORE && score_bias.count(n) > 0) {+ for (size_t i = 0; i < n; i++) {+ logits[i] += score_bias.at(n)[i];+ }+ }+ const double logit_max = *std::max_element(logits.begin(), logits.end());std::vector<double> p(n);double sum = 0.0;for (size_t i = 0; i < n; i++) {- p[i] = std::exp((double) (s[i] - score_max) / temperature);+ p[i] = std::exp((logits[i] - logit_max) / temperature);sum += p[i];}for (size_t i = 0; i < n; i++) {@@ -1116,18 +1205,20 @@ json server_decision_context::format_answer(const server_decision_question & queconst size_t best = std::max_element(probs.begin(), probs.end()) - probs.begin();answer["choice"] = question.options[best].key;answer["probabilities"] = probabilities;- answer["confidence"] = decision_confidence_choice(probs);+ answer["confidence"] = get_decision_confidence(question.type, probs);} else {double expected = 0.0;json legend = json::object();for (size_t i = 0; i < n; i++) {expected += i * probs[i];- legend[question.options[i].key] = question.options[i].description;+ const json & description = question.options[i].description;+ // vllm-decision2 returns structured descriptions as canonical JSON text+ legend[question.options[i].key] = strict_payload && !description.is_string() ? json(decision_sort_keys(description).dump()) : description;}answer["score"] = expected;answer["legend"] = legend;answer["probabilities"] = probabilities;- answer["confidence"] = decision_confidence_score(probs);+ answer["confidence"] = get_decision_confidence(question.type, probs);}return answer;}diff --git a/tools/server/server-decision.h b/tools/server/server-decision.hindex 5c7e44f37b..3e81098cb2 100644--- a/tools/server/server-decision.h+++ b/tools/server/server-decision.h@@ -43,6 +43,7 @@ struct server_decision_context {case COMMON_DECISION_TYPE_KEV:case COMMON_DECISION_TYPE_NIMBLE:case COMMON_DECISION_TYPE_PPLX_DECIDER:+ case COMMON_DECISION_TYPE_VLLM_DECISION2:case COMMON_DECISION_TYPE_LFM2_D1:return true;default:@@ -110,9 +111,12 @@ private:std::shared_ptr<const common_chat_template> tmpl; // the "systemone" templatestd::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"+ std::map<size_t, std::vector<float>> score_bias; // added to the scores of a score question, by number of levelssize_t n_options_max = 0;bool noul_true_first = false; // noul options are [true, false] instead of [false, true]+ bool noul_criteria_order = false; // noul options are in the order of the criteria keys, if givenbool choice_sorted = false; // choice options are in the order of their keys+ bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array// OPENJEV, LEV, NIMBLE, PPLX_DECIDERstd::vector<llama_token> labels;@@ -148,6 +152,7 @@ private:server_task & task) const;float get_temperature(const server_decision_question & question) const;+ double get_decision_confidence(server_decision_question_type type, const std::vector<double> & probs) const;};// group the tasks so that the common prefix of their prompts is evaluated only oncediff --git a/tools/server/server-task.h b/tools/server/server-task.hindex b8d6a3b670..50926ec3ca 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -191,6 +191,11 @@ struct server_task {for (const int32_t marker : markers) {pos = pos < 0 ? marker : std::min(pos, marker);}+ for (size_t i = 0; i < order.size(); i++) {+ if (order[i] != 0) {+ return pos < 0 ? (int32_t) i : std::min(pos, (int32_t) i);+ }+ }return pos;}