Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
19.3 KB
Raw Blame
#!/usr/bin/env python3
"""Write gliner_decide_tiny/ref.json: what the `gliner2` package answers on the tiny fixture.

Maintainer-run, offline, once per change of the fixture: it needs torch,
transformers < 5 and the pinned `gliner2` package (pip install gliner2==2.0.0),
which CI does not. The answers come from fastino's own runtime,
AutoExtractor.from_pretrained(...).classify_text(), on the checkpoint
tools/make_gliner_decide_tiny.py writes; the SHA-256 of that checkpoint is
recorded so tests/test_gliner_decide_tiny.py refuses a regenerated fixture that
differs.

A System One request becomes one classify_text() call (decision_tasks below,
the mapping docs/gliner_decide.md states and gliner_decide.c implements): one
task per question, named by the question id, its instructions as the task's
prompt, its labels (a noul's "yes" then "no", a score's "0".."n-1"), and the
option texts as label descriptions. For every case it records what the engine
must reproduce:
  ids          the encoder's input_ids (processor._format_input_with_mapping)
  markers      the [L] positions the classifier reads (schema_special_indices)
  logits       the classifier's output per label, in the record's option order
  probs        the softmax _extract_classification_result applies, unrounded
and the package's own answer (classify_text with include_confidence), against
which the unrounded numbers are checked here before anything is written. A
long state is cut at MAX_LEN tokens the way the engine cuts it: the reference
is called with max_len set to the words that fit, its own truncation. A case
whose questions alone exceed MAX_LEN records the refusal the engine owes.
Every decision is checked for a near-tie, so a float-level difference can
never flip an argmax in the test.

    python tools/make_gliner_decide_tiny.py --output ./gliner_decide_tiny
    python tools/make_gliner_decide_ref.py --fixture ./gliner_decide_tiny
"""

import argparse
import hashlib
import json
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(HERE))
from openai_server import systemone_decision_record  # noqa: E402

PINNED_GLINER2 = "2.0.0"
MIN_GAP = 1e-3                  # smallest top-1 / top-2 probability gap accepted
MAX_LEN = 300                   # COLI_GLINER_MAX_LEN the test runs the engine with

LONG_BODY = (
    "Hello team, I am writing again about the invoice we received on the first of the month. "
    "The amount is twice what we agreed in the contract and the payment was already taken from "
    "our card. We asked for a refund last week and nobody answered. Our finance department needs "
    "the corrected invoice before Friday or we will have to escalate this to our lawyers and stop "
    "using the service. Please also check the other invoices of this year, because we think the "
    "same mistake happened in March and in April. Thank you, the accounts team.")

CONVERSATION = [
    {"role": "user", "content": "Hi, I cannot log in to my account since yesterday."},
    {"role": "assistant", "content": "Sorry about that. Did you try to reset the password?"},
    {"role": "user", "content": "Yes, twice, and the reset email never arrives."},
    {"role": "user", "content": "If it is not fixed today I will cancel the subscription."},
]

DEPARTMENT = {"type": "choice", "instructions": "Which department should handle this request?",
              "criteria": {"billing": "invoices, payments, refunds",
                           "technical": "bugs, outages, system errors",
                           "sales": "pricing, new contracts", "other": "everything else"}}

CASES = [
    ("ticket_three_types",
     "Hi, we were billed twice for March. Please refund the duplicate today or we will cancel our plan.",
     {"department": DEPARTMENT,
      "urgency": {"type": "score", "instructions": "How urgent is this request?",
                  "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},
      "churn_risk": {"type": "noul", "instructions": "Does the user threaten to cancel or leave?"},
      "refund_requested": {"type": "noul",
                           "instructions": "Does the user explicitly request a refund?"}}),
    ("two_options_and_noul_criteria",
     "The application crashes every time I open the settings page!",
     {"is_bug": {"type": "choice", "instructions": "Is this a bug report or a feature request?",
                 "criteria": {"bug": "something is broken", "feature": "something new is wanted"}},
      "blocking": {"type": "noul", "instructions": "Is the user blocked?",
                   "criteria": {"true": "the user cannot work at all",
                                "false": "the user can still work"}},
      "only_true": {"type": "noul", "instructions": "Does it mention settings?",
                    "criteria": {"true": "the settings page is named in the message"}}}),
    ("labels_without_and_with_descriptions",
     "We would like to schedule a demo of the product for our team next week",
     {"intent": {"type": "choice", "instructions": "What does the sender want?",
                 "criteria": {"demo": "a product demonstration", "refund": None, "cancel": "",
                              "pricing": {"desc": "price questions", "examples": ["cost", "plan"]},
                              "support": 3, "partnership": ["co-marketing", "reselling"],
                              "other": "anything else"}}}),
    ("twelve_labels_no_instructions",
     "I love this product, it works great and the support team was very helpful.",
     {"emotion": {"type": "choice",
                  "criteria": {name: None for name in (
                      "joy", "anger", "sadness", "fear", "surprise", "disgust", "trust",
                      "anticipation", "neutral", "confusion", "gratitude", "pride")}}}),
    ("scores",
     "This is the worst service I have ever used, nothing works and nobody answers?",
     {"sentiment": {"type": "score", "instructions": "What is the sender's tone?",
                    "criteria": ["angry", "negative", "neutral", "positive", "delighted"]},
      "six_levels": {"type": "score", "instructions": "How severe is the impact?",
                     "criteria": ["none", "minor", "moderate", "major", "critical", "catastrophic"]},
      "two_levels": {"type": "score", "instructions": "Is any action needed?",
                     "criteria": ["no action", "action needed"]},
      "numeric_levels": {"type": "score", "instructions": "Rate the urgency.",
                         "criteria": [1, 2, 3]}}),
    ("long_state_cut",
     {"subject": "Invoice problem again", "body": LONG_BODY},
     {"department": DEPARTMENT,
      "legal_threat": {"type": "noul", "instructions": "Does the sender threaten legal action?"}}),
    ("conversation", CONVERSATION,
     {"churn": {"type": "noul", "instructions": "Does the user threaten to cancel?"},
      "topic": {"type": "choice", "instructions": "What is the conversation about?",
                "criteria": {"login": "access to the account", "billing": "payments",
                             "other": "anything else"}}}),
    ("json_state_and_instructions",
     {"ticket": {"id": 4411, "amount": 129.5, "currency": "EUR", "paid": True,
                 "refund": None, "tags": ["billing", "urgent"], "nested": {"a": [1, 2.0, {"b": "c"}]}}},
     {"action": {"type": "choice",
                 "instructions": {"task": "pick the next action", "context": "support desk"},
                 "criteria": {"refund": "give the money back", "escalate": "a human decides",
                              "close": "nothing to do"}},
      "paid": {"type": "noul", "instructions": ["Was", "the", "ticket", "paid?"]}}),
    ("special_tokens_and_whitespace",
     "Order [P] arrived  broken,   see\tphoto.\r\n\r\nThe cafe [SEP_TEXT] said [L] is blocked "
     "[UNK] and support said 'we're sorry'",
     {"damaged": {"type": "noul", "instructions": "Was the item [DESCRIPTION] damaged?"},
      "channel": {"type": "choice", "instructions": "Which channel  should   answer?",
                  "criteria": {"email": "write an email [L] back",
                               "phone": "call\tthe customer", "none": None}}}),
    ("urls_mail_handles",
     "Visit https://example.com/help?x=1 or WWW.example.org, write to [email protected] or "
     "ping @support_team about state-of-the-art_tools and 2026-10-02 at 3.14 pm",
     {"channel": {"type": "choice", "instructions": "Where should the user go?",
                  "criteria": {"website": None, "email": None, "social": None}}}),
    ("unicode",
     "Buongiorno, la fattura è sbagliata. Grüße aus München, ευχαριστώ ΟΔΟΣ ΣΟΦΙΑΣ. "
     "İstanbul 我们的账户被锁了 \U0001F642\U0001F44D\U0001F3FD Å Ω café",
     {"language": {"type": "choice", "instructions": "Which language comes first?",
                   "criteria": {"italian": "italiano", "german": "deutsch", "chinese": "中文"}},
      "polite": {"type": "noul", "instructions": "Is the message polite?"}}),
    ("many_options",
     "Security incident: a phishing email asked employees to reset their passwords.",
     {"category": {"type": "choice", "instructions": "Classify the security incident.",
                   "criteria": {f"cat{i:02d}": None for i in range(40)}}}),
    ("empty_state", "",
     {"anything": {"type": "noul", "instructions": "Is there any request?"}}),
    ("questions_over_max_len",
     "Too many options for the token budget.",
     {"bucket": {"type": "choice", "instructions": "Pick one.",
                 "criteria": {f"option{i}": f"the description of option {i}" for i in range(30)}}}),
]

TOKENIZER_STRINGS = [
    "", " ", "  ", "a", " a", "a ", "Hello world", "a  b\t\tc\n\nd", "tab\tand\r\nnewline\n",
    "   leading spaces", "trailing spaces   ", "x[P]y", "a [UNK] b", "[SEP_TEXT][SEP_STRUCT]",
    "[sep] [CLS] [MASK]", "<0x41>", "▁▁a", "foo▁bar", "(", ")", "[L]",
    "intent: What does the customer want? [DESCRIPTION] refund: give the money back",
    "café café Å Ω", "日本語のテキスト", "\U0001F642\U0001F44D\U0001F3FD", "a b a  b",
    "a
b", "mixed ,.;:!? punctuation!!! ... ???", "über straße Ελληνικά ΟΔΟΣ русский",
    "state-of-the-art_tools 2026-10-02 3.14",
]

SPLIT_STRINGS = [
    "Hello, World!", "state-of-the-art_tools -x- a--b _x_", "https://Example.com/a?b=1 http:// www.x",
    "HTTPS://EXAMPLE.COM wwW.Example.org www. ftp://x.y", "mail [email protected] [email protected] [email protected]",
    "@user @_x @ a@", "ΟΔΟΣ ΣΟΦΙΑΣ Σ ΑΣ ΣΑ", "İstanbul ıi ſ K KELVIN", "日本語 テキスト 中文",
    "café é ́", "\U0001F642 \U0001F44D\U0001F3FD", "tab\tnew\nline\x1c\x85 x",
    "2026-10-02 3.14 1,000 $5 50%", "", "   ",
]


def decision_tasks(record):
    """The classify_text() tasks for a DECIDE record: one per question, in order."""
    tasks = {}
    for question in record["questions"]:
        options = question["options"]
        if question["type"] == "noul":
            labels, texts = ["yes", "no"], [options[1]["text"], options[0]["text"]]
        else:
            labels, texts = [o["label"] for o in options], [o["text"] for o in options]
        task = {"labels": labels}
        if question["instructions"]:
            task["prompt"] = question["instructions"]
        descriptions = {label: text for label, text in zip(labels, texts) if text}
        if descriptions:
            task["label_descriptions"] = descriptions
        tasks[question["id"]] = task
    return tasks


def reference_text(state):
    """processor._collate_batch: a text that does not end in . ! or ? gets one."""
    if state and not state.endswith((".", "!", "?")):
        return state + "."
    return state or "."


class Recorder:
    """Hooks on the reference model: the encoder's input ids, the classifier's
    logits per task, the marker positions the processor routed."""

    def __init__(self, model):
        import torch
        self.ids, self.logits, self.markers = [], [], []
        model.encoder.register_forward_pre_hook(
            lambda module, args, kwargs: self.ids.append(kwargs["input_ids"][0].tolist()),
            with_kwargs=True)
        model.classifier.register_forward_hook(
            lambda module, args, out: self.logits.append(out.squeeze(-1).to(torch.float64).tolist()))
        processor = model.processor
        original = processor.extract_embeddings_from_batch

        def routed(token_embeddings, input_ids, batch):
            self.markers.append(batch.schema_special_indices[0])
            return original(token_embeddings, input_ids, batch)
        processor.extract_embeddings_from_batch = routed

    def clear(self):
        self.ids.clear(); self.logits.clear(); self.markers.clear()


def fit_words(model, record, tasks, max_len):
    """How many words of the state the engine keeps under max_len tokens, found
    with the reference's own splitter and tokenizer: (schema tokens, words,
    tokens per word)."""
    processor = model.processor
    schema = model._classification_schema(tasks).build()
    schema = {**schema, "classifications": [{**c, "true_label": ["N/A"]} for c in schema["classifications"]]}
    processor.change_mode(is_training=False)
    schema_tokens = len(processor.transform_and_format("", schema).input_ids)   # up to [SEP_TEXT]
    words = [w for w, _, _ in processor.word_splitter(reference_text(record["state"]), lower=True)]
    counts = [len(processor.tokenizer.tokenize(w)) for w in words]
    return schema_tokens, words, counts


def run_case(model, recorder, record, max_len, min_gap=MIN_GAP):
    """One record through the reference: what the engine must reproduce, or the
    refusal it owes. min_gap=None accepts near-ties (the comparison tool)."""
    import numpy as np
    tasks = decision_tasks(record)
    schema_tokens, words, counts = fit_words(model, record, tasks, max_len)
    if schema_tokens > max_len:
        return {"refused": f"questions: the questions and their options take {schema_tokens} tokens, "
                           f"more than max_len={max_len}"}
    used, total = 0, schema_tokens
    for count in counts:
        if total + count > max_len:
            break
        total += count
        used += 1
    recorder.clear()
    result = model.classify_text(record["state"], tasks, include_confidence=True,
                                 max_len=None if used == len(words) else used)
    if len(recorder.ids) != 1 or len(recorder.logits) != len(tasks) or len(recorder.markers) != 1:
        sys.exit("the reference ran more than one forward or skipped a task")
    markers = []
    for positions in recorder.markers[0]:
        markers.extend(positions[1:])
    answers = []
    for question, logits in zip(record["questions"], recorder.logits):
        z = np.array(logits, dtype=np.float64)
        p = np.exp(z - z.max())
        p = p / p.sum()
        order = np.sort(p)[::-1]
        if min_gap is not None and len(p) > 1 and order[0] - order[1] < min_gap:
            sys.exit(f"{question['id']}: top-2 gap {order[0] - order[1]:.2e} is a near-tie; "
                     "change the case or SEED")
        got = result[question["id"]]
        labels = tasks[question["id"]]["labels"]
        best = int(np.argmax(p))
        if got["label"] != labels[best] or abs(got["confidence"] - float(p[best])) > 1e-6:
            sys.exit(f"{question['id']}: unrounded probabilities disagree with classify_text "
                     f"({got} vs {labels[best]} {p[best]})")
        if question["type"] == "noul":                   # read as yes, no; recorded false, true
            z, p = z[::-1], p[::-1]
        answers.append({"id": question["id"], "logits": [float(x) for x in z],
                        "probs": [float(x) for x in p]})
    return {"ids": recorder.ids[0], "markers": markers, "words": len(words), "words_used": used,
            "state_tokens": sum(counts), "state_used": sum(counts[:used]),
            "answers": answers, "predict": result}


def compact_json(ref):
    """One key per line at the top, one case or string per line below it:
    diffable, without a line per token id."""
    def dump(value):
        return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
    lines = ["{"]
    keys = list(ref)
    for i, key in enumerate(keys):
        tail = "," if i + 1 < len(keys) else ""
        value = ref[key]
        if isinstance(value, list):
            lines.append(f" {dump(key)}:[")
            lines.extend(f"  {dump(item)}{',' if j + 1 < len(value) else ''}"
                         for j, item in enumerate(value))
            lines.append(f" ]{tail}")
        else:
            lines.append(f" {dump(key)}:{dump(value)}{tail}")
    lines.append("}")
    return "\n".join(lines) + "\n"


def sha256(path):
    digest = hashlib.sha256()
    with open(path, "rb") as handle:
        for block in iter(lambda: handle.read(1 << 20), b""):
            digest.update(block)
    return digest.hexdigest()


def load_reference(path, allow_version=False):
    import importlib.metadata
    import torch
    version = importlib.metadata.version("gliner2")
    if version != PINNED_GLINER2 and not allow_version:
        sys.exit(f"gliner2 {version} is installed; the reference is pinned to {PINNED_GLINER2}")
    from gliner2 import AutoExtractor
    torch.manual_seed(0)
    model = AutoExtractor.from_pretrained(str(path))
    model.eval()
    return model, version


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--fixture", default="./gliner_decide_tiny")
    parser.add_argument("--allow-version", action="store_true",
                        help=f"run with an installed gliner2 other than {PINNED_GLINER2}")
    args = parser.parse_args()

    import torch
    import transformers
    from gliner2.processing.word_splitter import WhitespaceTokenSplitter

    fixture = Path(args.fixture)
    model, version = load_reference(fixture, args.allow_version)
    recorder = Recorder(model)
    cases = []
    for name, state, questions in CASES:
        record = systemone_decision_record({"state": state, "questions": questions})
        case = {"name": name, "state": state, "questions": questions}
        case.update(run_case(model, recorder, record, MAX_LEN))
        cases.append(case)
        what = case.get("refused") or f"{len(case['answers'])} question(s), {len(case['ids'])} tokens"
        print(f"{name}: {what}", file=sys.stderr)

    tok = model.processor.tokenizer
    tokenizer = [{"text": text, "ids": tok.convert_tokens_to_ids(tok.tokenize(text))}
                 for text in TOKENIZER_STRINGS]
    splitter = WhitespaceTokenSplitter()
    split = [{"text": text, "words": [w for w, _, _ in splitter(text, lower=True)]}
             for text in SPLIT_STRINGS]
    ref = {
        "generator": "tools/make_gliner_decide_ref.py",
        "reference": {"gliner2": version, "torch": torch.__version__,
                      "transformers": transformers.__version__},
        "fixture_sha256": {name: sha256(fixture / name) for name in (
            "model.safetensors", "tokenizer.json", "config.json", "encoder_config/config.json")},
        "max_len": MAX_LEN,
        "tolerance": {"probs": 2e-5, "logits": 1e-4},
        "tokenizer": tokenizer,
        "split": split,
        "cases": cases,
    }
    (fixture / "ref.json").write_text(compact_json(ref), encoding="utf-8")
    print(f"wrote {fixture / 'ref.json'}: {len(cases)} cases, {len(tokenizer)} tokenizer strings, "
          f"{len(split)} split strings", file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())