Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(research): make the Chinese benchmark audit platform-independent (#216)
HankGrimm885ba788ff38
diff --git a/.gitattributes b/.gitattributesnew file mode 100644index 00000000..f491c0bf--- /dev/null+++ b/.gitattributes@@ -0,0 +1,8 @@+# research/benchmarks/feishu_zh/SOURCE.json pins a sha256 of the raw bytes of the+# frozen dataset and of every archived run, and audit.py verifies them with+# read_bytes(). A checkout that rewrites the line endings (git's default+# core.autocrlf=true on Windows) changes those bytes, so the archive must stay LF+# on every platform.+research/benchmarks/feishu_zh/SOURCE.json text eol=lf+research/benchmarks/feishu_zh/data/** text eol=lf+research/benchmarks/feishu_zh/results/** text eol=lfdiff --git a/research/benchmarks/feishu_zh/audit.py b/research/benchmarks/feishu_zh/audit.pyindex e90ae81b..fc76e293 100644--- a/research/benchmarks/feishu_zh/audit.py+++ b/research/benchmarks/feishu_zh/audit.py@@ -11,8 +11,8 @@ ROOT = Path(__file__).resolve().parentdef load_cases():- cases = [json.loads(line) for line in (ROOT / 'data/cases.jsonl').read_text().splitlines()]- manifest = json.loads((ROOT / 'data/manifest.json').read_text())+ cases = [json.loads(line) for line in (ROOT / 'data/cases.jsonl').read_text(encoding='utf-8').splitlines()]+ manifest = json.loads((ROOT / 'data/manifest.json').read_text(encoding='utf-8'))if digest(cases) != manifest['cases_sha256']:raise ValueError('Dataset hash mismatch')if digest({c['id']: requests_for(c) for c in cases}) != manifest['requests_sha256']:@@ -25,7 +25,7 @@ def load_cases():def audit_run(folder):cases, manifest = load_cases()by_id = {c['id']: c for c in cases}- metadata = json.loads((folder / 'metadata.json').read_text())+ metadata = json.loads((folder / 'metadata.json').read_text(encoding='utf-8'))for key in ['cases_sha256', 'requests_sha256']:if metadata[key] != manifest[key]:raise ValueError('Run uses a different frozen protocol')@@ -36,7 +36,7 @@ def audit_run(folder):or not set(ids) <= set(by_id) or not modes or len(set(modes)) != len(modes)or not set(modes) <= {'choice', 'four_noul'}):raise ValueError('Invalid run metadata')- rows = [json.loads(line) for line in (folder / 'raw.jsonl').read_text().splitlines()]+ rows = [json.loads(line) for line in (folder / 'raw.jsonl').read_text(encoding='utf-8').splitlines()]expected = {(case_id, mode, repeat) for case_id in ids for mode in modes for repeat in range(repeats)}seen = set()for row in rows:@@ -67,7 +67,7 @@ def audit_run(folder):def verify_archive():- source = json.loads((ROOT / 'SOURCE.json').read_text())+ source = json.loads((ROOT / 'SOURCE.json').read_text(encoding='utf-8'))for item in source['files']:name, expected = item['path'], item['sha256']relative = Path(name)@@ -89,7 +89,8 @@ def main():if args.write_summary:if args.write_summary.exists():raise SystemExit('Refusing to overwrite an existing summary')- args.write_summary.write_text(json.dumps(results, ensure_ascii=False, indent=2) + '\n')+ args.write_summary.write_text(json.dumps(results, ensure_ascii=False, indent=2) + '\n',+ encoding='utf-8', newline='\n')for name, modes in results.items():for mode, result in modes.items():print(f"{name} / {mode}: {result['correct']}/{result['n']}; "diff --git a/research/benchmarks/feishu_zh/render_cards.py b/research/benchmarks/feishu_zh/render_cards.pyindex e1037290..9e85beb5 100644--- a/research/benchmarks/feishu_zh/render_cards.py+++ b/research/benchmarks/feishu_zh/render_cards.py@@ -12,7 +12,7 @@ DPI=160; W,H=1440,1920FONT=next((p for p in [Path('/System/Library/Fonts/Hiragino Sans GB.ttc'),Path('/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc')] if p.exists()),None)LANG='zh'NAVY='#142d40';MUTED='#61798a';BLUE='#2874c8';TEAL='#159b8e';BG='#f4f8fb'-S=json.loads((ROOT/'results/v1/summary.json').read_text())+S=json.loads((ROOT/'results/v1/summary.json').read_text(encoding='utf-8'))ROWS=[S[b][m] for b,m in [('jev','choice'),('jev','four_noul'),('laya','choice'),('laya','four_noul')]]NAMES=['Jev · 单选','Jev · 四问','Laya · 单选','Laya · 四问'];COLORS=[BLUE,BLUE,TEAL,TEAL]def canvas():diff --git a/research/benchmarks/feishu_zh/run.py b/research/benchmarks/feishu_zh/run.pyindex 0cb8b1bc..7b574dcf 100644--- a/research/benchmarks/feishu_zh/run.py+++ b/research/benchmarks/feishu_zh/run.py@@ -125,10 +125,11 @@ def main():metadata['warmups'].append(dict(mode=mode, elapsed_ms=elapsed, usage=result.get('usage')))except Exception as e:metadata['warmups'].append(dict(mode=mode, error_type=type(e).__name__))- (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2))+ (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2),+ encoding='utf-8', newline='\n')jobs = [(repeat, case, mode) for repeat in range(args.repeats) for case in cases for mode in args.modes]random.Random(20260921).shuffle(jobs)- with (args.output / 'raw.jsonl').open('w') as stream:+ with (args.output / 'raw.jsonl').open('w', encoding='utf-8', newline='\n') as stream:for i, (repeat, case, mode) in enumerate(jobs):req = frozen_request(case)[mode]row = dict(id=case['id'], family=case['family'], expected=case['expected'], mode=mode,@@ -146,7 +147,8 @@ def main():if (i + 1) % 32 == 0:print(f'{i + 1}/{len(jobs)} requests completed', flush=True)metadata['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()- (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2))+ (args.output / 'metadata.json').write_text(json.dumps(metadata, ensure_ascii=False, indent=2),+ encoding='utf-8', newline='\n')finally:if client:client.close()diff --git a/research/benchmarks/feishu_zh/tests/test_audit.py b/research/benchmarks/feishu_zh/tests/test_audit.pyindex 7cdd2e0d..98c163fa 100644--- a/research/benchmarks/feishu_zh/tests/test_audit.py+++ b/research/benchmarks/feishu_zh/tests/test_audit.py@@ -22,15 +22,15 @@ class AuditTests(unittest.TestCase):self.assertEqual(result[mode]['failed_requests'], 0)def fixture(self, folder):- metadata = json.loads((ROOT / 'results/v1/laya/metadata.json').read_text())- rows = [json.loads(x) for x in (ROOT / 'results/v1/laya/raw.jsonl').read_text().splitlines()]+ metadata = json.loads((ROOT / 'results/v1/laya/metadata.json').read_text(encoding='utf-8'))+ rows = [json.loads(x) for x in (ROOT / 'results/v1/laya/raw.jsonl').read_text(encoding='utf-8').splitlines()]rows = [r for r in rows if r['mode'] == 'choice' and r['repeat'] == 0]metadata.update(repeats=1, modes=['choice'], evaluated_ids=[r['id'] for r in rows])- (folder / 'metadata.json').write_text(json.dumps(metadata))+ (folder / 'metadata.json').write_text(json.dumps(metadata), encoding='utf-8')return rowsdef write(self, folder, rows):- (folder / 'raw.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in rows))+ (folder / 'raw.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in rows), encoding='utf-8')def test_reject_corrupted_records(self):for corruption in ['missing', 'duplicate', 'gold', 'request', 'prediction', 'negative_latency']: