Skip to content

LLM inference in C/C++

C++ ggml

model : Granite switch decision 256 options (#30693)

Gabe Goodhart1268edac5791
diff --git a/conversion/granite.py b/conversion/granite.py
index d2ef7c836c..4c6c5a0e4c 100644
--- a/conversion/granite.py
+++ b/conversion/granite.py
@@ -296,19 +296,20 @@ class GraniteSwitchModel(GraniteMoeModel):
def set_vocab(self):
super().set_vocab()
if self._has_decision_adapter():
- self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+ systemone_template = self._systemone_template()
+ logger.info("System One Template: %s", systemone_template)
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}])
def _systemone_template(self) -> str:
# prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token
# (it replaces the first token of "### DECISION"), the answer is the label token that comes next
description = jinja_str_or_json("o.description")
option = (
- "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"
+ "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}"
"{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"
"{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"
)
return (
- "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"
"<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")
+ "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"
diff --git a/tools/server/README.md b/tools/server/README.md
index e4a0c418bf..c42336c7f2 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1723,7 +1723,7 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index 3905c81b34..be9bc8ef49 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {
bias[level] = std::strtof(val, nullptr);
}
- if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
+ if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
// one letter per option, each must be a single token
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
for (const char c : letters) {
@@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) {
}
n_options_max = labels.size();
noul_true_first = true;
- } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
+ } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) {
// label codes are A..Z then AA..ZZ, only the ones that are a single token are used
std::vector<std::string> codes;
for (char a = 'A'; a <= 'Z'; a++) {
@@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) {
if (model_type == COMMON_DECISION_TYPE_VLLM_D3) {
canonical_legend = true;
}
+ if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {
+ noul_true_first = true;
+ }
} else if (model_type == COMMON_DECISION_TYPE_KEV) {
// the hidden state of an option is read at the token that ends it
const auto toks = common_tokenize(vocab, "<|box_end|>", false, true);