Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(research): point the benchmark scripts at paths that exist (#133)
Ludwig J. Marxc00c54beaa12
diff --git a/README.md b/README.mdindex 7ccf0aad..4d2ca9eb 100644--- a/README.md+++ b/README.md@@ -293,7 +293,7 @@ triage = agent.predict({"message": "My payment failed twice"}, laya.triage_questAll Laya numbers below are measured. Every model answered byte-identical questions(fixed seed) in the same run. Reproduce with-[`notebooks/laya_benchmark_colab.ipynb`](https://github.com/NandhaKishorM/laya) on a T4.+[`research/scripts/laya_benchmark_colab.ipynb`](research/scripts/laya_benchmark_colab.ipynb) on a T4.### Speed (Tesla T4, measured)diff --git a/research/scripts/bench_apps.py b/research/scripts/bench_apps.pyindex c0b255ba..1d83a90e 100644--- a/research/scripts/bench_apps.py+++ b/research/scripts/bench_apps.py@@ -12,7 +12,7 @@ Workflows (the demo Space's tabs), each on real labelled data:Jev-comparable tasks (AbdelStark/jev-benchmarks published Jev accuracy on these):ag_news 0.910 | banking77 0.870 | dair emotion 0.480 (Brier 0.846, NLL 5.588)- USE_TF=0 python3 notebooks/bench_apps.py+ USE_TF=0 python3 research/scripts/bench_apps.py"""import gcimport json@@ -30,8 +30,8 @@ import numpy as np # noqa: E402import torch # noqa: E402REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))-sys.path.insert(0, os.path.join(REPO, "laya"))-sys.path.insert(0, os.path.join(REPO, "notebooks"))+sys.path.insert(0, os.path.dirname(REPO))+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # bench_local lives hereimport laya # noqa: E402from bench_local import load, metrics, score_cases, softmax_t, temp_for # noqa: E402diff --git a/research/scripts/bench_latency.py b/research/scripts/bench_latency.pyindex e97ddbb4..43dec911 100644--- a/research/scripts/bench_latency.py+++ b/research/scripts/bench_latency.py@@ -7,7 +7,7 @@ Routing changes the latency picture in two ways:So there are two regimes -- hot (right model already loaded) and cold (swap) -- and the realisticnumber for a mixed-language workload sits between them, determined by `max_loaded`.- USE_TF=0 python3 notebooks/bench_latency.py+ USE_TF=0 python3 research/scripts/bench_latency.py"""import gcimport json@@ -24,7 +24,7 @@ import numpy as np # noqa: E402import torch # noqa: E402REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))-sys.path.insert(0, os.path.join(REPO, "laya"))+sys.path.insert(0, os.path.dirname(REPO))import laya # noqa: E402from laya.lang import analyse # noqa: E402diff --git a/research/scripts/bench_local.py b/research/scripts/bench_local.pyindex 1b94ab20..3a3958fe 100644--- a/research/scripts/bench_local.py+++ b/research/scripts/bench_local.py@@ -7,7 +7,7 @@ Part B typed-decisions (400 cases / 2,000 decisions) on all three checkpoints,Writes local_benchmark_results.json.- USE_TF=0 python3 notebooks/bench_local.py [--langs N] [--per-lang N] [--skip-a] [--skip-b]+ USE_TF=0 python3 research/scripts/bench_local.py [--langs N] [--per-lang N] [--skip-a] [--skip-b]"""import argparseimport gc@@ -27,7 +27,7 @@ import numpy as np # noqa: E402import torch # noqa: E402REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))-sys.path.insert(0, os.path.join(REPO, "laya"))+sys.path.insert(0, os.path.dirname(REPO))import laya # noqa: E402from laya.common import QTYPES, build_sequence, collate_items, render_options, temp_bucket # noqa: E402diff --git a/research/scripts/make_plots.py b/research/scripts/make_plots.pyindex 08a9e857..2a944e7f 100644--- a/research/scripts/make_plots.py+++ b/research/scripts/make_plots.py@@ -1,9 +1,9 @@"""Render the benchmark figure used in the GitHub and Hugging Face READMEs.- USE_TF=0 python3 notebooks/make_plots.py+ USE_TF=0 python3 research/scripts/make_plots.py-Reads local_benchmark_results.json (51-language sweep) and laya_benchmark_results.json-(T4 latency + English suites); writes assets/laya_benchmark.png.+Reads the published results in research/results/ (the 51-language sweep and the T4 run)+and writes assets/laya_benchmark.png."""import jsonimport os@@ -13,7 +13,8 @@ matplotlib.use("Agg")import matplotlib.pyplot as plt # noqa: E402import numpy as np # noqa: E402-REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+RESEARCH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+REPO = os.path.dirname(RESEARCH)ASSETS = os.path.join(REPO, "assets")os.makedirs(ASSETS, exist_ok=True)@@ -38,8 +39,8 @@ def strip(ax, keep=("bottom",)):def main():- loc = json.load(open(os.path.join(REPO, "local_benchmark_results.json")))- col = json.load(open(os.path.join(REPO, "laya_benchmark_results.json")))+ loc = json.load(open(os.path.join(RESEARCH, "results", "cpu_51_language_sweep.json")))+ col = json.load(open(os.path.join(RESEARCH, "results", "t4_colab_benchmark.json")))en = loc["part_a"]["by_model"]["english"]["per_language"]ml = loc["part_a"]["by_model"].get("multilingual", {}).get("per_language", {})