Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(research): make the Chinese benchmark audit platform-independent (#216)

HankGrimm885ba788ff38
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..f491c0bf
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,8 @@
+# research/benchmarks/feishu_zh/SOURCE.json pins a sha256 of the raw bytes of the
+# frozen dataset and of every archived run, and audit.py verifies them with
+# read_bytes(). A checkout that rewrites the line endings (git's default
+# core.autocrlf=true on Windows) changes those bytes, so the archive must stay LF
+# on every platform.
+research/benchmarks/feishu_zh/SOURCE.json text eol=lf
+research/benchmarks/feishu_zh/data/** text eol=lf
+research/benchmarks/feishu_zh/results/** text eol=lf
diff --git a/research/benchmarks/feishu_zh/audit.py b/research/benchmarks/feishu_zh/audit.py
index e90ae81b..fc76e293 100644
--- a/research/benchmarks/feishu_zh/audit.py
+++ b/research/benchmarks/feishu_zh/audit.py
@@ -11,8 +11,8 @@ ROOT = Path(__file__).resolve().parent
def load_cases():
- cases = [json.loads(line) for line in (ROOT / 'data/cases.jsonl').read_text().splitlines()]
- manifest = json.loads((ROOT / 'data/manifest.json').read_text())
+ cases = [json.loads(line) for line in (ROOT / 'data/cases.jsonl').read_text(encoding='utf-8').splitlines()]
+ manifest = json.loads((ROOT / 'data/manifest.json').read_text(encoding='utf-8'))
if digest(cases) != manifest['cases_sha256']:
raise ValueError('Dataset hash mismatch')
if digest({c['id']: requests_for(c) for c in cases}) != manifest['requests_sha256']:
@@ -25,7 +25,7 @@ def load_cases():
def audit_run(folder):
cases, manifest = load_cases()
by_id = {c['id']: c for c in cases}
- metadata = json.loads((folder / 'metadata.json').read_text())
+ metadata = json.loads((folder / 'metadata.json').read_text(encoding='utf-8'))
for key in ['cases_sha256', 'requests_sha256']:
if metadata[key] != manifest[key]:
raise ValueError('Run uses a different frozen protocol')
@@ -36,7 +36,7 @@ def audit_run(folder):
or not set(ids) <= set(by_id) or not modes or len(set(modes)) != len(modes)
or not set(modes) <= {'choice', 'four_noul'}):
raise ValueError('Invalid run metadata')
- rows = [json.loads(line) for line in (folder / 'raw.jsonl').read_text().splitlines()]
+ rows = [json.loads(line) for line in (folder / 'raw.jsonl').read_text(encoding='utf-8').splitlines()]
expected = {(case_id, mode, repeat) for case_id in ids for mode in modes for repeat in range(repeats)}
seen = set()
for row in rows:
@@ -67,7 +67,7 @@ def audit_run(folder):
def verify_archive():
- source = json.loads((ROOT / 'SOURCE.json').read_text())
+ source = json.loads((ROOT / 'SOURCE.json').read_text(encoding='utf-8'))
for item in source['files']:
name, expected = item['path'], item['sha256']
relative = Path(name)
@@ -89,7 +89,8 @@ def main():
if args.write_summary:
if args.write_summary.exists():
raise SystemExit('Refusing to overwrite an existing summary')
- args.write_summary.write_text(json.dumps(results, ensure_ascii=False, indent=2) + '\n')
+ args.write_summary.write_text(json.dumps(results, ensure_ascii=False, indent=2) + '\n',
+ encoding='utf-8', newline='\n')
for name, modes in results.items():
for mode, result in modes.items():
print(f"{name} / {mode}: {result['correct']}/{result['n']}; "
diff --git a/research/benchmarks/feishu_zh/render_cards.py b/research/benchmarks/feishu_zh/render_cards.py
index e1037290..9e85beb5 100644
--- a/research/benchmarks/feishu_zh/render_cards.py
+++ b/research/benchmarks/feishu_zh/render_cards.py
@@ -12,7 +12,7 @@ DPI=160; W,H=1440,1920
FONT=next((p for p in [Path('/System/Library/Fonts/Hiragino Sans GB.ttc'),Path('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc')] if p.exists()),None)
LANG='zh'
NAVY='#142d40';MUTED='#61798a';BLUE='#2874c8';TEAL='#159b8e';BG='#f4f8fb'
-S=json.loads((ROOT/'results/v1/summary.json').read_text())
+S=json.loads((ROOT/'results/v1/summary.json').read_text(encoding='utf-8'))
ROWS=[S[b][m] for b,m in [('jev','choice'),('jev','four_noul'),('laya','choice'),('laya','four_noul')]]
NAMES=['Jev · 单选','Jev · 四问','Laya · 单选','Laya · 四问'];COLORS=[BLUE,BLUE,TEAL,TEAL]
def canvas():
diff --git a/research/benchmarks/feishu_zh/run.py b/research/benchmarks/feishu_zh/run.py
index 0cb8b1bc..7b574dcf 100644
--- a/research/benchmarks/feishu_zh/run.py
+++ b/research/benchmarks/feishu_zh/run.py
@@ -125,10 +125,11 @@ def main():
metadata['warmups'].append(dict(mode=mode, elapsed_ms=elapsed, usage=result.get('usage')))
except Exception as e:
metadata['warmups'].append(dict(mode=mode, error_type=type(e).__name__))
- (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2))
+ (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2),
+ encoding='utf-8', newline='\n')
jobs = [(repeat, case, mode) for repeat in range(args.repeats) for case in cases for mode in args.modes]
random.Random(20260921).shuffle(jobs)
- with (args.output / 'raw.jsonl').open('w') as stream:
+ with (args.output / 'raw.jsonl').open('w', encoding='utf-8', newline='\n') as stream:
for i, (repeat, case, mode) in enumerate(jobs):
req = frozen_request(case)[mode]
row = dict(id=case['id'], family=case['family'], expected=case['expected'], mode=mode,
@@ -146,7 +147,8 @@ def main():
if (i + 1) % 32 == 0:
print(f'{i + 1}/{len(jobs)} requests completed', flush=True)
metadata['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
- (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2))
+ (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2),
+ encoding='utf-8', newline='\n')
finally:
if client:
client.close()
diff --git a/research/benchmarks/feishu_zh/tests/test_audit.py b/research/benchmarks/feishu_zh/tests/test_audit.py
index 7cdd2e0d..98c163fa 100644
--- a/research/benchmarks/feishu_zh/tests/test_audit.py
+++ b/research/benchmarks/feishu_zh/tests/test_audit.py
@@ -22,15 +22,15 @@ class AuditTests(unittest.TestCase):
self.assertEqual(result[mode]['failed_requests'], 0)
def fixture(self, folder):
- metadata = json.loads((ROOT / 'results/v1/laya/metadata.json').read_text())
- rows = [json.loads(x) for x in (ROOT / 'results/v1/laya/raw.jsonl').read_text().splitlines()]
+ metadata = json.loads((ROOT / 'results/v1/laya/metadata.json').read_text(encoding='utf-8'))
+ rows = [json.loads(x) for x in (ROOT / 'results/v1/laya/raw.jsonl').read_text(encoding='utf-8').splitlines()]
rows = [r for r in rows if r['mode'] == 'choice' and r['repeat'] == 0]
metadata.update(repeats=1, modes=['choice'], evaluated_ids=[r['id'] for r in rows])
- (folder / 'metadata.json').write_text(json.dumps(metadata))
+ (folder / 'metadata.json').write_text(json.dumps(metadata), encoding='utf-8')
return rows
def write(self, folder, rows):
- (folder / 'raw.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in rows))
+ (folder / 'raw.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in rows), encoding='utf-8')
def test_reject_corrupted_records(self):
for corruption in ['missing', 'duplicate', 'gold', 'request', 'prediction', 'negative_latency']: