Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(research): point the benchmark scripts at paths that exist (#133)

Ludwig J. Marxc00c54beaa12
diff --git a/README.md b/README.md
index 7ccf0aad..4d2ca9eb 100644
--- a/README.md
+++ b/README.md
@@ -293,7 +293,7 @@ triage = agent.predict({"message": "My payment failed twice"}, laya.triage_quest
All Laya numbers below are measured. Every model answered byte-identical questions
(fixed seed) in the same run. Reproduce with
-[`notebooks/laya_benchmark_colab.ipynb`](https://github.com/NandhaKishorM/laya) on a T4.
+[`research/scripts/laya_benchmark_colab.ipynb`](research/scripts/laya_benchmark_colab.ipynb) on a T4.
### Speed (Tesla T4, measured)
diff --git a/research/scripts/bench_apps.py b/research/scripts/bench_apps.py
index c0b255ba..1d83a90e 100644
--- a/research/scripts/bench_apps.py
+++ b/research/scripts/bench_apps.py
@@ -12,7 +12,7 @@ Workflows (the demo Space's tabs), each on real labelled data:
Jev-comparable tasks (AbdelStark/jev-benchmarks published Jev accuracy on these):
ag_news 0.910 | banking77 0.870 | dair emotion 0.480 (Brier 0.846, NLL 5.588)
- USE_TF=0 python3 notebooks/bench_apps.py
+ USE_TF=0 python3 research/scripts/bench_apps.py
"""
import gc
import json
@@ -30,8 +30,8 @@ import numpy as np # noqa: E402
import torch # noqa: E402
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, os.path.join(REPO, "laya"))
-sys.path.insert(0, os.path.join(REPO, "notebooks"))
+sys.path.insert(0, os.path.dirname(REPO))
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # bench_local lives here
import laya # noqa: E402
from bench_local import load, metrics, score_cases, softmax_t, temp_for # noqa: E402
diff --git a/research/scripts/bench_latency.py b/research/scripts/bench_latency.py
index e97ddbb4..43dec911 100644
--- a/research/scripts/bench_latency.py
+++ b/research/scripts/bench_latency.py
@@ -7,7 +7,7 @@ Routing changes the latency picture in two ways:
So there are two regimes -- hot (right model already loaded) and cold (swap) -- and the realistic
number for a mixed-language workload sits between them, determined by `max_loaded`.
- USE_TF=0 python3 notebooks/bench_latency.py
+ USE_TF=0 python3 research/scripts/bench_latency.py
"""
import gc
import json
@@ -24,7 +24,7 @@ import numpy as np # noqa: E402
import torch # noqa: E402
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, os.path.join(REPO, "laya"))
+sys.path.insert(0, os.path.dirname(REPO))
import laya # noqa: E402
from laya.lang import analyse # noqa: E402
diff --git a/research/scripts/bench_local.py b/research/scripts/bench_local.py
index 1b94ab20..3a3958fe 100644
--- a/research/scripts/bench_local.py
+++ b/research/scripts/bench_local.py
@@ -7,7 +7,7 @@ Part B typed-decisions (400 cases / 2,000 decisions) on all three checkpoints,
Writes local_benchmark_results.json.
- USE_TF=0 python3 notebooks/bench_local.py [--langs N] [--per-lang N] [--skip-a] [--skip-b]
+ USE_TF=0 python3 research/scripts/bench_local.py [--langs N] [--per-lang N] [--skip-a] [--skip-b]
"""
import argparse
import gc
@@ -27,7 +27,7 @@ import numpy as np # noqa: E402
import torch # noqa: E402
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, os.path.join(REPO, "laya"))
+sys.path.insert(0, os.path.dirname(REPO))
import laya # noqa: E402
from laya.common import QTYPES, build_sequence, collate_items, render_options, temp_bucket # noqa: E402
diff --git a/research/scripts/make_plots.py b/research/scripts/make_plots.py
index 08a9e857..2a944e7f 100644
--- a/research/scripts/make_plots.py
+++ b/research/scripts/make_plots.py
@@ -1,9 +1,9 @@
"""Render the benchmark figure used in the GitHub and Hugging Face READMEs.
- USE_TF=0 python3 notebooks/make_plots.py
+ USE_TF=0 python3 research/scripts/make_plots.py
-Reads local_benchmark_results.json (51-language sweep) and laya_benchmark_results.json
-(T4 latency + English suites); writes assets/laya_benchmark.png.
+Reads the published results in research/results/ (the 51-language sweep and the T4 run)
+and writes assets/laya_benchmark.png.
"""
import json
import os
@@ -13,7 +13,8 @@ matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
import numpy as np # noqa: E402
-REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+RESEARCH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+REPO = os.path.dirname(RESEARCH)
ASSETS = os.path.join(REPO, "assets")
os.makedirs(ASSETS, exist_ok=True)
@@ -38,8 +39,8 @@ def strip(ax, keep=("bottom",)):
def main():
- loc = json.load(open(os.path.join(REPO, "local_benchmark_results.json")))
- col = json.load(open(os.path.join(REPO, "laya_benchmark_results.json")))
+ loc = json.load(open(os.path.join(RESEARCH, "results", "cpu_51_language_sweep.json")))
+ col = json.load(open(os.path.join(RESEARCH, "results", "t4_colab_benchmark.json")))
en = loc["part_a"]["by_model"]["english"]["per_language"]
ml = loc["part_a"]["by_model"].get("multilingual", {}).get("per_language", {})