LLM inference in C/C++
C++ ggml
model : Granite Switch Decision (#30676)
Gabe Goodhart23f7aabc16c7
diff --git a/common/common.cpp b/common/common.cppindex c5db228329..b59e66e3bd 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1174,6 +1174,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA{ COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" },{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },+ { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"};diff --git a/common/common.h b/common/common.hindex 2cd6e2ec98..0c2d5e46f1 100644--- a/common/common.h+++ b/common/common.h@@ -970,6 +970,7 @@ enum common_decision_type {COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attentionCOMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question typeCOMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout+ COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switchCOMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported};diff --git a/conversion/granite.py b/conversion/granite.pyindex 4cbeb76178..d2ef7c836c 100644--- a/conversion/granite.py+++ b/conversion/granite.py@@ -8,7 +8,7 @@ import torchif TYPE_CHECKING:from torch import Tensor-from .base import MmprojModel, ModelBase, gguf, logger+from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, loggerfrom .llama import LlamaModelfrom .mamba import Mamba2Model@@ -284,10 +284,39 @@ class GraniteSwitchModel(GraniteMoeModel):self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank)self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"])self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"])+ if self._has_decision_adapter():+ self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH)router_gain = float(self.hparams.get("control_token_gain", 15.0))self.gguf_writer.add_adapter_router_gain(router_gain)logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain)+ def _has_decision_adapter(self) -> bool:+ return "decision" in self.hparams.get("adapter_names", [])++ def set_vocab(self):+ super().set_vocab()+ if self._has_decision_adapter():+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])++ def _systemone_template(self) -> str:+ # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token+ # (it replaces the first token of "### DECISION"), the answer is the label token that comes next+ description = jinja_str_or_json("o.description")+ option = (+ "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}"+ "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}"+ "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}"+ )+ return (+ "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}"+ "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n"+ "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions")+ + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}"+ "{{ '\n\nOptions:\n' }}"+ "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}"+ "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}"+ )+def _lora_a(self, data: Tensor) -> Tensor:# on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in]a = data.squeeze(1)diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.pyindex 00b51165c9..4261e76e3e 100644--- a/gguf-py/gguf/constants.py+++ b/gguf-py/gguf/constants.py@@ -6283,6 +6283,7 @@ class DecisionType:VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attentionLFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question typeLFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout+ GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switchclass VisionProjectorType:diff --git a/tools/server/README.md b/tools/server/README.mdindex 7d697633ed..e4a0c418bf 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1723,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppoThe questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.+The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.+For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.*Image input:*diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex c32c0d1a5a..5577c339db 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -161,6 +161,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type deccase COMMON_DECISION_TYPE_LFM2_D1:case COMMON_DECISION_TYPE_LFM2_D1_OMNI:return {"decisions"};+ case COMMON_DECISION_TYPE_GRANITE_SWITCH:+ return {"text", "decisions"};default:// fallback when there is no decision type or the metadata is badreturn {"text"};diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cppindex b3c19e1f03..3905c81b34 100644--- a/tools/server/server-decision.cpp+++ b/tools/server/server-decision.cpp@@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) {bias[level] = std::strtof(val, nullptr);}- if (model_type == COMMON_DECISION_TYPE_OPENJEV) {+ if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) {// one letter per option, each must be a single tokenconst std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";for (const char c : letters) {@@ -753,7 +753,7 @@ void server_decision_context::fill_task(const std::string prompt = render(state, questions, question, variant, files.size());- if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {+ if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {// lev reads the ratings of a noul question at its first labels, not at the digitstask.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));}diff --git a/tools/server/server-decision.h b/tools/server/server-decision.hindex bfe9fd0ee5..ed97244293 100644--- a/tools/server/server-decision.h+++ b/tools/server/server-decision.h@@ -39,6 +39,7 @@ struct server_decision_context {bool can_share_prompt() const {switch (type) {case COMMON_DECISION_TYPE_OPENJEV:+ case COMMON_DECISION_TYPE_GRANITE_SWITCH:case COMMON_DECISION_TYPE_LEV:case COMMON_DECISION_TYPE_KEV:case COMMON_DECISION_TYPE_NIMBLE: