Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

5.1 KB
Raw Blame
"""Example 16 -- many questions, still one forward pass.

Batching 13 independent questions into one `predict()` call costs one pass over the
state. The encoder sees the state once per question in the batch, and the answers all
come back together, so `usage["input_tokens"]` is the sum across the whole call.
"""
from _common import STATE_EN, banner, describe, device_line, load, timed

banner("16", "Thirteen questions, one call", """
    Real triage asks many things about one message. Here 13 questions -- 5 choice,
    3 score and 5 noul -- are answered in a single `predict()` call.

    Two details worth noticing:

      * `usage["input_tokens"]` is the batch total. Split the questions across two
        calls and the per-call token counts add up to exactly the same number; the
        batch simply folds them into one pass.
      * latency barely moves with the number of questions. The encoder is the cost, and
        it runs once for the whole batch, not once per question.
    """)

agent = load("english")
device_line(agent)

MANY = {
    "department": {"type": "choice", "instructions": "Which team should handle this?",
                   "criteria": {"billing": "invoices, payments, refunds",
                                "technical": "bugs, outages, system errors",
                                "sales": "pricing, new contracts",
                                "other": "none of the above"}},
    "issue_type": {"type": "choice", "instructions": "What is the underlying issue?",
                   "criteria": {"duplicate_charge": "the same charge appears more than once",
                                "late_delivery": "an order has not arrived on time",
                                "outage": "a service is down or erroring",
                                "login_problem": "the user cannot sign in",
                                "other": "none of the above"}},
    "language": {"type": "choice", "instructions": "What language is this written in?",
                 "criteria": {"english": "English", "german": "German",
                              "french": "French", "other": "any other language"}},
    "severity": {"type": "choice", "instructions": "How serious is the problem?",
                 "criteria": {"cosmetic": "annoying but no real impact",
                              "costly": "costs money or time",
                              "blocking": "stops work or business entirely"}},
    "account_action": {"type": "choice", "instructions": "What account change is wanted?",
                       "criteria": {"none": "no account change",
                                    "cancel_plan": "cancel or close the account",
                                    "downgrade": "move to a cheaper plan",
                                    "upgrade": "move to a bigger plan"}},
    "urgency": {"type": "score", "instructions": "How urgent is this request?",
                "criteria": ["not urgent", "soon", "critical deadline or blocking issue"]},
    "sentiment": {"type": "score", "instructions": "How negative is the tone?",
                  "criteria": ["positive", "neutral", "negative", "very negative"]},
    "time_pressure": {"type": "score", "instructions": "How much time pressure is stated?",
                      "criteria": ["none", "implied", "explicit deadline", "already overdue"]},
    "churn_risk": {"type": "noul",
                   "instructions": "Does the sender threaten to cancel or leave?"},
    "refund_requested": {"type": "noul", "instructions": "Does the sender ask for money back?"},
    "is_angry": {"type": "noul", "instructions": "Does the sender sound angry?"},
    "needs_manager": {"type": "noul",
                      "instructions": "Does this need a manager to get involved?"},
    "contains_pii": {"type": "noul",
                     "instructions": "Does the message contain personal or financial data?"},
}

result = agent.predict(STATE_EN, MANY)
print("   %d questions, one call -- %d answers returned:\n" % (len(MANY), len(result["answers"])))
describe(result["answers"], indent="   ")
print("\n   batch call usage : %d input tokens, %d output tokens"
      % (result["usage"]["input_tokens"], result["usage"]["output_tokens"]))

first_id = next(iter(MANY))
rest = {qid: question for qid, question in MANY.items() if qid != first_id}
first_tokens = agent.predict(STATE_EN, {first_id: MANY[first_id]})["usage"]["input_tokens"]
rest_tokens = agent.predict(STATE_EN, rest)["usage"]["input_tokens"]
print("   one question alone        : %d input tokens" % first_tokens)
print("   the other %2d in one call  : %d input tokens" % (len(rest), rest_tokens))
print("   sum                       : %d input tokens, exactly the batch total above"
      % (first_tokens + rest_tokens))
print("   -> input_tokens is the batch total: every question in the call contributes.")
_, single_ms = timed(lambda: agent.predict(STATE_EN, {"urgency": MANY["urgency"]}), repeat=2)
_, batch_ms = timed(lambda: agent.predict(STATE_EN, MANY), repeat=2)
print("\n   latency 1 question  : %.0f ms" % single_ms)
print("   latency 13 questions: %.0f ms  (%.1fx for 13x the questions)"
      % (batch_ms, batch_ms / max(1.0, single_ms)))