LLM inference in C/C++
C++ ggml
model : Granite switch decision 256 options (#30693)
Gabe Goodhart1268edac5791
diff --git a/conversion/granite.py b/conversion/granite.pyindex d2ef7c836c..4c6c5a0e4c 100644--- a/conversion/granite.py+++ b/conversion/granite.py@@ -296,19 +296,20 @@ class GraniteSwitchModel(GraniteMoeModel):def set_vocab(self):super().set_vocab()if self._has_decision_adapter():- self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])+ systemone_template = self._systemone_template()+ logger.info("System One Template: %s", systemone_template)+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}])def _systemone_template(self) -> str:# prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token# (it replaces the first token of "### DECISION"), the answer is the label token that comes nextdescription = jinja_str_or_json("o.description")option = (- "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"+ "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}""{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}""{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}")return (- "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}""<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n""<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")+ "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"diff --git a/tools/server/README.md b/tools/server/README.mdindex e4a0c418bf..c42336c7f2 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1723,7 +1723,7 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppoThe questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.-The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cppindex 3905c81b34..be9bc8ef49 100644--- a/tools/server/server-decision.cpp+++ b/tools/server/server-decision.cpp@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {bias[level] = std::strtof(val, nullptr);}- if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {+ if (model_type == COMMON_DECISION_TYPE_OPENJEV) {// one letter per option, each must be a single tokenconst std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";for (const char c : letters) {@@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) {}n_options_max = labels.size();noul_true_first = true;- } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {+ } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {// label codes are A..Z then AA..ZZ, only the ones that are a single token are usedstd::vector<std::string> codes;for (char a = 'A'; a <= 'Z'; a++) {@@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) {if (model_type == COMMON_DECISION_TYPE_VLLM_D3) {canonical_legend = true;}+ if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {+ noul_true_first = true;+ }} else if (model_type == COMMON_DECISION_TYPE_KEV) {// the hidden state of an option is read at the token that ends itconst auto toks = common_tokenize(vocab, "<|box_end|>", false, true);