Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #615 from RagingSilence/perf/reuse-question-tokenization

Nandakishor008deb86a47c
diff --cc .github/workflows/ci.yml
index 7b022096,e3e78d3d..2fe56ecf
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@@ -116,15 -109,12 +116,16 @@@ jobs
python tests/test_decision_model.py
python tests/test_head_checkpointing.py
python tests/test_packaging.py
+ python tests/test_compose_env.py
+ python tests/test_audit_scope.py
python tests/test_doc_tables.py
+ python tests/test_env_docs.py
python tests/test_tokenizer_cache.py
python tests/test_tokenizer_concurrency.py
+ python tests/test_question_token_reuse.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
+ python tests/test_option_collapse.py
python tests/test_lang_guess.py
python tests/test_identifier_complexity.py
python tests/test_lang_stats.py
@@@ -316,10 -255,14 +317,14 @@@
python tests/test_hooks_api.py
shell: bash
+ - name: Question token reuse tests
+ run: python tests/test_question_token_reuse.py
+ shell: bash
+
- # Same five pytest suites as the Linux job. They are named explicitly rather than added to
+ # Same six pytest suites as the Linux job. They are named explicitly rather than added to
# the loop above, because the loop runs each file as a script and that is exactly what does
# not execute a pytest suite (#374).
- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation)
+ - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)
run: |
python -m pytest tests/test_serve.py \
tests/test_router_batch.py \
diff --cc laya/agent.py
index 71a20e74,cdfd4c16..c8383523
--- a/laya/agent.py
+++ b/laya/agent.py
@@@ -976,16 -875,9 +979,18 @@@ class Agent(HookRegistry)
raise TypeError(
"predict_batch expects a list of states; pass a single state to predict()/system_one()."
)
+ if not isinstance(questions, dict):
+ raise TypeError(
+ "questions must be a dict of question id -> definition, got %s"
+ % type(questions).__name__
+ )
states = list(states)
+ if any(state is None for state in states):
+ raise TypeError(
+ "state must not be None; pass a string, dict, or list"
+ )
+ if len(states) == 1:
+ _disable_question_token_reuse()
if not states:
ctx.results = []
else:
diff --cc laya/common.py
index 404d994c,3cf70571..e5339320
--- a/laya/common.py
+++ b/laya/common.py
@@@ -128,13 -162,11 +168,13 @@@ def build_sequence
add_special_tokens=False,
truncation=True,
max_length=48,
- )["input_ids"]
+ )
opt_ids.append([tok.mask_token_id] + opt_tokens)
opt_budget = head_max_len - sum(len(o) for o in opt_ids)
+ per_option = None
if opt_budget < 16:
per = max(4, (head_max_len - 16) // max(1, len(opt_ids)))
+ per_option = per
opt_ids = [o[:per] for o in opt_ids]
opt_budget = head_max_len - sum(len(o) for o in opt_ids)
head_ids = head_ids[: max(8, opt_budget)]