Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
research(step4): single-T4 Kaggle Gate D kernel for the remaining 5 cells
Bruce-Yii70f17ba3b2a6
diff --git a/research/scripts/step4_benchmark/kaggle/build_gate_d.py b/research/scripts/step4_benchmark/kaggle/build_gate_d.pynew file mode 100644index 00000000..7850b451--- /dev/null+++ b/research/scripts/step4_benchmark/kaggle/build_gate_d.py@@ -0,0 +1,222 @@+"""Generate the Gate D Kaggle kernel: the remaining 5 cells of the 3-seed x 2-loss panel.++One single-T4 session runs the five remaining trainings sequentially under the pinned Gate-R+contract, fail-closed after each: checkpoint exists, `training.laya_train` records the intended+loss/seed/config, counts match (5600/400/skipped {}), metrics finite, and the base sanity row stays+within tolerance of the Gate-R reference. A hard mismatch writes the partial results and aborts.++RLCD seed 0 is reused from the Gate R kernel (different session, same pinned SHA/data/command), as+the Coordinator explicitly allowed; it is embedded as the sixth cell.++ python research/scripts/step4_benchmark/kaggle/build_gate_d.py+ KAGGLE_API_TOKEN=$(kaggle auth print-access-token) kaggle kernels push -p <this dir>/gate_d+"""+from __future__ import annotations++import base64+import json+from pathlib import Path++HERE = Path(__file__).resolve().parent+HARNESS = HERE.parent+OUT = HERE / "gate_d"+EMBED = ("prepare_data.py", "evaluate.py", "metrics.py")+PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"+KERNEL_ID = "yuyi722333/laya-step4-gate-d"++TEMPLATE = '''"""#887 Step 4 Gate D, generated by build_gate_d.py -- do not edit here."""+import base64+import json+import math+import os+import subprocess+import sys+import time+from pathlib import Path++PINNED_SHA = "__PINNED_SHA__"+WORK = Path("/kaggle/working")+HARNESS = WORK / "step4_benchmark"+HARNESS.mkdir(parents=True, exist_ok=True)++_EMBEDDED = __EMBEDDED__+for _n, _b in _EMBEDDED.items():+ (HARNESS / _n).write_bytes(base64.b64decode(_b))+sys.path.insert(0, str(HARNESS))++os.environ["CUDA_VISIBLE_DEVICES"] = "0"+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")+import torch # noqa: E402++n_gpu = torch.cuda.device_count()+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu+print("GPU", torch.cuda.get_device_name(0), flush=True)++RESULTS_PATH = WORK / "step4_gate_d_results.json"+++def flush_results():+ RESULTS_PATH.write_text(json.dumps(RESULTS, indent=2), encoding="utf-8")+++def run(cmd):+ print("\\n$ " + " ".join(cmd), flush=True)+ p = subprocess.run(cmd, check=True, text=True, capture_output=True)+ sys.stdout.write(p.stdout)+ if p.stderr:+ sys.stderr.write(p.stderr)+ sys.stdout.flush()+ return p.stdout+++run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):+ try:+ __import__(_mod)+ except Exception:+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])++from huggingface_hub import snapshot_download # noqa: E402++base_dir = snapshot_download("convaiinnovations/laya")+base_cfg = json.loads((Path(base_dir) / "rl_agent_config.json").read_text(encoding="utf-8"))++run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])+test_meta = json.loads((WORK / "data" / "test_meta.json").read_text(encoding="utf-8"))++from evaluate import evaluate_checkpoint # noqa: E402++RESULTS = {+ "meta": {+ "pinned_sha": PINNED_SHA,+ "torch": torch.__version__,+ "gpu": torch.cuda.get_device_name(0),+ "visible_gpus": n_gpu,+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},+ "n_train_cases": 1200,+ "n_test_cases": len(test_meta),+ "reused": {"rlcd_seed0": "Gate R kernel (separate session, same SHA/data/command)"},+ },+ "rlcd_seed0": {+ "checkpoint": "Gate R kernel run_rlcd_seed0 (reused)",+ "temperatures": "shipped", "n_cases": 400, "n_decisions": 2000,+ "accuracy": 0.775, "ece": 0.1504, "brier": 0.0519, "soft_accuracy": 0.5654,+ "per_type_accuracy": {"choice": 0.7417, "noul": 0.8517, "score": 0.7425},+ "loss": "rlcd", "seed": 0, "wall_clock_s": 2691.3,+ },+}++# --- base sanity row, must match the Gate-R reference -----------------------+base_row = evaluate_checkpoint("convaiinnovations/laya", test_meta, device="cuda",+ temperatures="shipped")+RESULTS["base"] = base_row+base_ok = (abs(base_row["accuracy"] - 0.362) <= 0.01 and abs(base_row["ece"] - 0.1741) <= 0.01+ and abs(base_row["brier"] - 0.3287) <= 0.01)+print("BASE_SANITY", "OK" if base_ok else "FAIL", json.dumps(base_row), flush=True)+flush_results()+if not base_ok:+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: base sanity drift")++# --- the five remaining cells ----------------------------------------------+CELLS = [("rlcd", 1), ("soft-ce", 0), ("soft-ce", 1), ("rlcd", 2), ("soft-ce", 2)]+++def train(loss, seed):+ out = WORK / ("run_" + loss.replace("-", "_") + "_seed%d" % seed)+ cmd = [sys.executable, "-m", "laya.train_cli",+ "--data", str(WORK / "data" / "train.jsonl"),+ "--out", str(out),+ "--base", "convaiinnovations/laya",+ "--loss", loss, "--seed", str(seed),+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",+ "--max-len", "1024", "--head-max-len", "256",+ "--device", "cuda"]+ t0 = time.time()+ stdout = run(cmd)+ dt = round(time.time() - t0, 1)+ cfg = json.loads((out / "rl_agent_config.json").read_text(encoding="utf-8"))+ lt = (cfg.get("training") or {}).get("laya_train", {})+ split_line = next((ln for ln in stdout.splitlines() if ln.startswith("train items")), "")+ checks = {+ "loss": lt.get("loss") == loss,+ "seed": lt.get("seed") == seed,+ "epochs": lt.get("epochs") == 4,+ "micro_batch": lt.get("micro_batch") == 8,+ "grad_accum": lt.get("grad_accum") == 8,+ "encoder_lr": abs(float(lt.get("encoder_lr", 0)) - 2.5e-5) < 1e-12,+ "head_lr": abs(float(lt.get("head_lr", 0)) - 1e-4) < 1e-12,+ "max_len": lt.get("max_len") == 1024,+ "head_max_len": lt.get("head_max_len") == 256,+ "calib_seed": lt.get("calib_seed") == 20260922,+ "no_shuffle": list(lt.get("shuffle_options") or []) == [],+ "counts": "train items 5600, calibration items 400, skipped {}" in split_line,+ }+ return out, dt, cfg, checks, split_line+++for loss, seed in CELLS:+ out, dt, cfg, checks, split_line = train(loss, seed)+ print("CELL %s seed%d checks %s" % (loss, seed, json.dumps(checks)), flush=True)+ bad = sorted(k for k, v in checks.items() if not v)+ if bad:+ RESULTS["%s_seed%d" % (loss, seed)] = {"error": "protocol mismatch: " + ",".join(bad),+ "split_line": split_line}+ flush_results()+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: %s seed%d -> %s" % (loss, seed, bad))++ res = evaluate_checkpoint(str(out), test_meta, device="cuda", temperatures="shipped")+ finite = all(isinstance(res.get(k), (int, float)) and math.isfinite(res[k])+ for k in ("accuracy", "ece", "brier"))+ res.update({+ "loss": loss, "seed": seed, "wall_clock_s": dt,+ "temperature": cfg.get("temperature"),+ "calibration": (cfg.get("training") or {}).get("laya_train_calibration"),+ "split_line": split_line, "finite": finite,+ })+ RESULTS["%s_seed%d" % (loss, seed)] = res+ flush_results()+ print("RESULT %s seed%d %s" % (loss, seed, json.dumps(res)), flush=True)+ if not finite:+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: non-finite metric %s seed%d" % (loss, seed))++print("ALL_CELLS_DONE", flush=True)+print("RESULTS " + json.dumps(RESULTS, indent=2), flush=True)+'''+++def main() -> int:+ OUT.mkdir(parents=True, exist_ok=True)+ embedded = {name: base64.b64encode((HARNESS / name).read_bytes()).decode("ascii")+ for name in EMBED}+ script = (TEMPLATE.replace("__PINNED_SHA__", PINNED_SHA)+ .replace("__EMBEDDED__", json.dumps(embedded, indent=1)))+ (OUT / "step4_gate_d.py").write_text(script, encoding="utf-8", newline="\n")++ metadata = {+ "id": KERNEL_ID,+ "title": "Laya Step 4 Gate D",+ "code_file": "step4_gate_d.py",+ "language": "python",+ "kernel_type": "script",+ "is_private": True,+ "enable_gpu": True,+ "enable_tpu": False,+ "enable_internet": True,+ "machine_shape": "NvidiaTeslaT4",+ "dataset_sources": [],+ "competition_sources": [],+ "kernel_sources": [],+ "model_sources": [],+ }+ (OUT / "kernel-metadata.json").write_text(json.dumps(metadata, indent=2) + "\n",+ encoding="utf-8", newline="\n")+ print("wrote", OUT / "step4_gate_d.py", "and", OUT / "kernel-metadata.json")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.json b/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.jsonnew file mode 100644index 00000000..56fb0b84--- /dev/null+++ b/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.json@@ -0,0 +1,16 @@+{+ "id": "yuyi722333/laya-step4-gate-d",+ "title": "Laya Step 4 Gate D",+ "code_file": "step4_gate_d.py",+ "language": "python",+ "kernel_type": "script",+ "is_private": true,+ "enable_gpu": true,+ "enable_tpu": false,+ "enable_internet": true,+ "machine_shape": "NvidiaTeslaT4",+ "dataset_sources": [],+ "competition_sources": [],+ "kernel_sources": [],+ "model_sources": []+}diff --git a/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.py b/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.pynew file mode 100644index 00000000..814184b2--- /dev/null+++ b/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.py@@ -0,0 +1,165 @@+"""#887 Step 4 Gate D, generated by build_gate_d.py -- do not edit here."""+import base64+import json+import math+import os+import subprocess+import sys+import time+from pathlib import Path++PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"+WORK = Path("/kaggle/working")+HARNESS = WORK / "step4_benchmark"+HARNESS.mkdir(parents=True, exist_ok=True)++_EMBEDDED = {+ "prepare_data.py": "IiIiUHJlcGFyZSBMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyBmb3IgYGxheWEtdHJhaW5gIGFuZCBmb3IgZXZhbHVhdGlvbi4KClRoZSBiZW5jaG1hcmsncyBvd24gcm93cyBhcmUgYHtzdGF0ZSwgcXVlc3Rpb25zLCBnb2xkfWAgd2l0aCB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gcGVyCnF1ZXN0aW9uLiBUaGF0IGlzIGV4YWN0bHkgdGhlIHNjaGVtYSBgbGF5YS50cmFpbmAgcmVhZHMsIHNvIHRoZSBjb252ZXJzaW9uIGlzIGEgcGFyc2UsIG5vdCBhCnJlc2hhcGU6IHRoZSB0d28gSlNPTi1zdHJpbmcgY29sdW1ucyBhcmUgZGVjb2RlZCBhbmQgdGhlIHJvdyBpcyB3cml0dGVuIGJhY2sgb3V0IGFzIEpTT05MLgoKU3BsaXRzOgoKKiBgYHRyYWluLmpzb25sYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0cmFpbmBgIHNwbGl0ICgxLDIwMCBjYXNlcyAvIDYsMDAwIGRlY2lzaW9ucyksIHRoZSBpbnB1dCB0bwogIGBgbGF5YS10cmFpbmBgLiBUaGUgY2FsaWJyYXRpb24gc2xpY2UgaXMgaGVsZCBvdXQgKmluc2lkZSogYGBsYXlhLXRyYWluYGAgYXQgYSBmaXhlZCBzZWVkCiAgKGBgY2FsaWJfZnJhY2BgIDAuMSwgY2FwcGVkIGF0IDQwMCwgc2VlZCAyMDI2MDkyMiksIG1hdGNoaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCiogYGB0ZXN0Lmpzb25sYGAgLyBgYHRlc3RfbWV0YS5qc29uYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0ZXN0YGAgc3BsaXQgKDQwMCBjYXNlcyAvIDIsMDAwCiAgZGVjaXNpb25zKSwgdXNlZCBieSBgYGV2YWx1YXRlLnB5YGAuIGBgdGVzdF9tZXRhLmpzb25gYCBrZWVwcyBgYGlkYGAgYW5kIGBgd29ya2Zsb3dgYCBzbwogIHBlci13b3JrZmxvdyBhY2N1cmFjeSBjYW4gYmUgcmVwb3J0ZWQgYWxvbmdzaWRlIHRoZSBwb29sZWQgbnVtYmVyLgoKVXNhZ2U6CiAgICBweXRob24gcHJlcGFyZV9kYXRhLnB5IC0tb3V0LWRpciBkYXRhL3R5cGVkX2RlY2lzaW9ucwoiIiIKZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKaW1wb3J0IGFyZ3BhcnNlCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgc3lzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QKCgpkZWYgX3BhcnNlKHZhbHVlOiBBbnkpIC0+IEFueToKICAgIGlmIGlzaW5zdGFuY2UodmFsdWUsIHN0cik6CiAgICAgICAgcmV0dXJuIGpzb24ubG9hZHModmFsdWUpCiAgICByZXR1cm4gdmFsdWUKCgpkZWYgY2FzZV90b19yZWNvcmQocm93OiBEaWN0W3N0ciwgQW55XSkgLT4gRGljdFtzdHIsIEFueV06CiAgICAiIiJPbmUgZGF0YXNldCByb3cgYXMgYSBgbGF5YS10cmFpbmAgcmVjb3JkIChge3N0YXRlLCBxdWVzdGlvbnMsIGdvbGR9YCkuIiIiCiAgICByZXR1cm4gewogICAgICAgICJzdGF0ZSI6IF9wYXJzZShyb3dbInN0YXRlIl0pLAogICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2Uocm93WyJxdWVzdGlvbnMiXSksCiAgICAgICAgImdvbGQiOiBfcGFyc2Uocm93WyJnb2xkIl0pLAogICAgfQoKCmRlZiBsb2FkX3NwbGl0KHNwbGl0OiBzdHIsIGRhdGFzZXQ6IHN0ciA9ICJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpIC0+IExpc3RbRGljdFtzdHIsIEFueV1dOgogICAgdHJ5OgogICAgICAgIGZyb20gZGF0YXNldHMgaW1wb3J0IGxvYWRfZGF0YXNldAogICAgZXhjZXB0IEltcG9ydEVycm9yOgogICAgICAgIHByaW50KCJFUlJPUjogdGhlIGRhdGFzZXRzIHBhY2thZ2UgaXMgcmVxdWlyZWQgKHBpcCBpbnN0YWxsIGRhdGFzZXRzKSIsIGZpbGU9c3lzLnN0ZGVycikKICAgICAgICBzeXMuZXhpdCgxKQogICAgZHMgPSBsb2FkX2RhdGFzZXQoZGF0YXNldCwgImFsbCIsIHNwbGl0PXNwbGl0KQogICAgcmV0dXJuIGxpc3QoZHMpCgoKZGVmIHdyaXRlX3NwbGl0KHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIsIHNwbGl0OiBzdHIpIC0+IGludDoKICAgIHJlY29yZHMgPSBbY2FzZV90b19yZWNvcmQocikgZm9yIHIgaW4gcm93c10KICAgIHBhdGggPSBvcy5wYXRoLmpvaW4ob3V0X2RpciwgZiJ7c3BsaXR9Lmpzb25sIikKICAgIHdpdGggb3BlbihwYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAgZm9yIHJlYyBpbiByZWNvcmRzOgogICAgICAgICAgICBmLndyaXRlKGpzb24uZHVtcHMocmVjLCBlbnN1cmVfYXNjaWk9RmFsc2UpICsgIlxuIikKICAgIHJldHVybiBsZW4ocmVjb3JkcykKCgpkZWYgd3JpdGVfdGVzdF9tZXRhKHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIpIC0+IGludDoKICAgIG1ldGEgPSBbXQogICAgZm9yIHIgaW4gcm93czoKICAgICAgICBtZXRhLmFwcGVuZCh7CiAgICAgICAgICAgICJpZCI6IHIuZ2V0KCJpZCIsICIiKSwKICAgICAgICAgICAgIndvcmtmbG93Ijogci5nZXQoIndvcmtmbG93IiwgIiIpLAogICAgICAgICAgICAic3RhdGUiOiBfcGFyc2Uoclsic3RhdGUiXSksCiAgICAgICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2UoclsicXVlc3Rpb25zIl0pLAogICAgICAgICAgICAiZ29sZCI6IF9wYXJzZShyWyJnb2xkIl0pLAogICAgICAgIH0pCiAgICBwYXRoID0gb3MucGF0aC5qb2luKG91dF9kaXIsICJ0ZXN0X21ldGEuanNvbiIpCiAgICB3aXRoIG9wZW4ocGF0aCwgInciLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIGpzb24uZHVtcChtZXRhLCBmLCBpbmRlbnQ9MikKICAgIHJldHVybiBsZW4obWV0YSkKCgpkZWYgbWFpbigpIC0+IGludDoKICAgIHBhcnNlciA9IGFyZ3BhcnNlLkFyZ3VtZW50UGFyc2VyKGRlc2NyaXB0aW9uPV9fZG9jX18pCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dC1kaXIiLCBkZWZhdWx0PSJkYXRhL3R5cGVkX2RlY2lzaW9ucyIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRhdGFzZXQiLCBkZWZhdWx0PSJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpCiAgICBhcmdzID0gcGFyc2VyLnBhcnNlX2FyZ3MoKQoKICAgIG9zLm1ha2VkaXJzKGFyZ3Mub3V0X2RpciwgZXhpc3Rfb2s9VHJ1ZSkKICAgIGZvciBzcGxpdCBpbiAoInRyYWluIiwgInRlc3QiKToKICAgICAgICByb3dzID0gbG9hZF9zcGxpdChzcGxpdCwgYXJncy5kYXRhc2V0KQogICAgICAgIG4gPSB3cml0ZV9zcGxpdChyb3dzLCBhcmdzLm91dF9kaXIsIHNwbGl0KQogICAgICAgIHByaW50KGYie3NwbGl0fToge259IGNhc2VzIC0+IHtvcy5wYXRoLmpvaW4oYXJncy5vdXRfZGlyLCBzcGxpdCArICcuanNvbmwnKX0iKQogICAgICAgIGlmIHNwbGl0ID09ICJ0ZXN0IjoKICAgICAgICAgICAgbSA9IHdyaXRlX3Rlc3RfbWV0YShyb3dzLCBhcmdzLm91dF9kaXIpCiAgICAgICAgICAgIHByaW50KGYidGVzdCBtZXRhOiB7bX0gY2FzZXMgLT4ge29zLnBhdGguam9pbihhcmdzLm91dF9kaXIsICd0ZXN0X21ldGEuanNvbicpfSIpCiAgICByZXR1cm4gMAoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBzeXMuZXhpdChtYWluKCkpCg==",+ "evaluate.py": "IiIiRXZhbHVhdGUgYSBmaW5lLXR1bmVkIGNoZWNrcG9pbnQgb24gdGhlIHR5cGVkLWRlY2lzaW9ucyB0ZXN0IHNwbGl0LgoKTWlycm9ycyBjZWxsIDE0IG9mIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2ssIHJlc3RyaWN0ZWQgdG8gdGhlIHRocmVlIG1ldHJpY3MgdGhlIFN0ZXAtNCBjb250cmFjdApuYW1lczogKiphY2N1cmFjeSoqLCAqKkVDRSoqIGFuZCAqKkJyaWVyKiouIFRoZSAyMC1vcHRpb24gb3JkZXItZmxpcCBzdXJmYWNlIGlzIGRlbGliZXJhdGVseSBub3QKaGVyZTogaXQgYmVsb25ncyB0byBgbWFzc2l2ZV9pbnRlbnQuZW5gLCBub3QgdHlwZWQtZGVjaXNpb25zIChzZWUgUkVBRE1FKS4KCkNhbGlicmF0aW9uIGlzIG5vdCBieXRlLWNvbXBhcmFibGUgYWNyb3NzIGNvbnRyYWN0cy4gYGxheWEudHJhaW5gIGZpdHMgdGVtcGVyYXR1cmVzIHdpdGgKYGZpdF90ZW1wZXJhdHVyZV9tYXBgIChwZXIgdHlwZSwgYW5kIHBlciBvcHRpb24tY291bnQgYnVja2V0IGFib3ZlIGEgZmxvb3IpIHVuZGVyIHRoZSBydW50aW1lCmNsYW1wIGBbMC41LCA1LjBdYDsgdGhlIG9sZCBub3RlYm9vayBmaXR0ZWQgb25lIExCRkdTIHRlbXBlcmF0dXJlIHBlciB0eXBlIHVuZGVyIGBbMC4xLCAxMF1gLgpTbyBFQ0UgYW5kIEJyaWVyIGFyZSByZXBvcnRlZCB1bmRlciAqKmJvdGgqKiBwYXRoczoKCi0gYGAtLXRlbXBlcmF0dXJlcyBzaGlwcGVkYGAgKGRlZmF1bHQpIOKAlCB0aGUgY2hlY2twb2ludCdzIG93biBmaXR0ZWQgYHRlbXBlcmF0dXJlYCBhbmQKICBgdGVtcGVyYXR1cmVfYnlfb3B0aW9uc2AsIHdoaWNoIGlzIHdoYXQgdGhlIG5vdGVib29rJ3MgbnVtYmVyIGFsc28gdXNlZDsKLSBgYC0tdGVtcGVyYXR1cmVzIHJhd2BgIOKAlCB0ZW1wZXJhdHVyZSAxLjAgZXZlcnl3aGVyZSwgaXNvbGF0aW5nIHRoZSB0cmFpbmluZyBmcm9tIHRoZQogIGNhbGlicmF0aW9uIGNvbnRyYWN0LgoKQWNjdXJhY3kgaXMgdW5hZmZlY3RlZCBieSB0ZW1wZXJhdHVyZSAoaXQgb25seSByZXNjYWxlcyB0aGUgbG9naXRzKSwgc28gaXQgaXMgdGhlIGNsZWFuCnJlcHJvZHVjdGlvbiBzaWduYWwuCgpVc2FnZToKICAgIHB5dGhvbiBldmFsdWF0ZS5weSAtLWNoZWNrcG9pbnQgcnVucy9zdGVwNC9ybGNkLXNlZWQwIFwKICAgICAgICAtLXRlc3QtbWV0YSBkYXRhL3R5cGVkX2RlY2lzaW9ucy90ZXN0X21ldGEuanNvbiBcCiAgICAgICAgLS1vdXQgcnVucy9zdGVwNC9ybGNkLXNlZWQwL2V2YWxfc2hpcHBlZC5qc29uCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgppbXBvcnQgYXJncGFyc2UKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCBzeXMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdAoKc3lzLnBhdGguaW5zZXJ0KDAsIG9zLnBhdGguZGlybmFtZShvcy5wYXRoLmFic3BhdGgoX19maWxlX18pKSkKZnJvbSBtZXRyaWNzIGltcG9ydCAoICAjIG5vcWE6IEU0MDIKICAgIGFjY3VyYWN5LAogICAgYnJpZXJfc2NvcmUsCiAgICBleHBlY3RlZF9jYWxpYnJhdGlvbl9lcnJvciwKICAgIHNvZnRfYWNjdXJhY3ksCikKCgpkZWYgX25vcm1hbGlzZSh2YWx1ZXM6IExpc3RbZmxvYXRdKSAtPiBMaXN0W2Zsb2F0XToKICAgIHRvdGFsID0gc3VtKHZhbHVlcykKICAgIGlmIHRvdGFsID4gMDoKICAgICAgICByZXR1cm4gW3YgLyB0b3RhbCBmb3IgdiBpbiB2YWx1ZXNdCiAgICByZXR1cm4gdmFsdWVzCgoKZGVmIGV2YWx1YXRlX2NoZWNrcG9pbnQoY2hlY2twb2ludF9kaXI6IHN0ciwgdGVzdF9tZXRhOiBMaXN0W0RpY3Rbc3RyLCBBbnldXSwKICAgICAgICAgICAgICAgICAgICAgICAgZGV2aWNlOiBzdHIgPSAiYXV0byIsIHRlbXBlcmF0dXJlczogc3RyID0gInNoaXBwZWQiKSAtPiBEaWN0W3N0ciwgQW55XToKICAgIGltcG9ydCBsYXlhCgogICAgYWdlbnQgPSBsYXlhLmxvYWQoY2hlY2twb2ludF9kaXIsIGRldmljZT1kZXZpY2UpCiAgICBpZiB0ZW1wZXJhdHVyZXMgPT0gInJhdyI6CiAgICAgICAgYWdlbnQudGVtcGVyYXR1cmUgPSBbMS4wLCAxLjAsIDEuMF0KICAgICAgICBhZ2VudC50ZW1wZXJhdHVyZV9ieV9vcHRpb25zID0ge30KICAgIGVsaWYgdGVtcGVyYXR1cmVzICE9ICJzaGlwcGVkIjoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJ0ZW1wZXJhdHVyZXMgbXVzdCBiZSAnc2hpcHBlZCcgb3IgJ3JhdycsIGdvdCAlciIgJSAodGVtcGVyYXR1cmVzLCkpCgogICAgY29ycmVjdHM6IExpc3RbZmxvYXRdID0gW10KICAgIGNvbmZpZGVuY2VzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBicmllcnM6IExpc3RbZmxvYXRdID0gW10KICAgIHNvZnRzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBwZXJfdHlwZTogRGljdFtzdHIsIExpc3RbZmxvYXRdXSA9IHsiY2hvaWNlIjogW10sICJub3VsIjogW10sICJzY29yZSI6IFtdfQogICAgc2NvcmVfbWFlczogTGlzdFtmbG9hdF0gPSBbXQogICAgd2l0aGluX29uZTogTGlzdFtmbG9hdF0gPSBbXQoKICAgIGZvciBpLCBjYXNlIGluIGVudW1lcmF0ZSh0ZXN0X21ldGEpOgogICAgICAgIHN0YXRlID0gY2FzZVsic3RhdGUiXQogICAgICAgIHF1ZXN0aW9ucyA9IGNhc2VbInF1ZXN0aW9ucyJdCiAgICAgICAgZ29sZCA9IGNhc2VbImdvbGQiXQogICAgICAgIGFuc3dlcnMgPSBhZ2VudC5wcmVkaWN0KHN0YXRlLCBxdWVzdGlvbnMpWyJhbnN3ZXJzIl0KCiAgICAgICAgZm9yIHFpZCwgcWRlZiBpbiBxdWVzdGlvbnMuaXRlbXMoKToKICAgICAgICAgICAgaWYgcWlkIG5vdCBpbiBnb2xkOgogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgZyA9IGdvbGRbcWlkXQogICAgICAgICAgICBxdHlwZSA9IHFkZWZbInR5cGUiXQogICAgICAgICAgICBwX2FucyA9IGFuc3dlcnNbcWlkXQoKICAgICAgICAgICAgaWYgcXR5cGUgPT0gImNob2ljZSI6CiAgICAgICAgICAgICAgICBrZXlzID0gbGlzdChxZGVmWyJjcml0ZXJpYSJdLmtleXMoKSkKICAgICAgICAgICAgICAgIHByZWQgPSBwX2Fuc1siY2hvaWNlIl0KICAgICAgICAgICAgICAgIGdvbGRfbGFiZWwgPSBzdHIoZ1sibGFiZWwiXSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwcmVkID09IGdvbGRfbGFiZWwpCgogICAgICAgICAgICAgICAgcF9wcm9icyA9IF9ub3JtYWxpc2UoW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KGssIDFlLTYpIGZvciBrIGluIGtleXNdKQogICAgICAgICAgICAgICAgZ19wcm9icyA9IF9ub3JtYWxpc2UoW2dbInByb2JhYmlsaXRpZXMiXS5nZXQoaywgMWUtNikgZm9yIGsgaW4ga2V5c10pCgogICAgICAgICAgICAgICAgY29ycmVjdHMuYXBwZW5kKGlzX2NvcnIpCiAgICAgICAgICAgICAgICBjb25maWRlbmNlcy5hcHBlbmQobWF4KHBfcHJvYnMpKQogICAgICAgICAgICAgICAgYnJpZXJzLmFwcGVuZChicmllcl9zY29yZShwX3Byb2JzLCBnX3Byb2JzKSkKICAgICAgICAgICAgICAgIHNvZnRzLmFwcGVuZChzb2Z0X2FjY3VyYWN5KHBfcHJvYnMsIGdfcHJvYnMpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbImNob2ljZSJdLmFwcGVuZChpc19jb3JyKQoKICAgICAgICAgICAgZWxpZiBxdHlwZSA9PSAibm91bCI6CiAgICAgICAgICAgICAgICBwX3ZhbCA9IGZsb2F0KHBfYW5zWyJub3VsIl0pCiAgICAgICAgICAgICAgICBnX3ZhbCA9IGZsb2F0KGcuZ2V0KCJub3VsIiwgZy5nZXQoInByb2JhYmlsaXRpZXMiLCB7fSkuZ2V0KCJ0cnVlIiwgMC41KSkpCiAgICAgICAgICAgICAgICBnb2xkX2xhYmVsID0gc3RyKGdbImxhYmVsIl0pLmxvd2VyKCkKICAgICAgICAgICAgICAgIHByZWRfbGFiZWwgPSAidHJ1ZSIgaWYgcF92YWwgPj0gMC41IGVsc2UgImZhbHNlIgogICAgICAgICAgICAgICAgaXNfY29yciA9IGZsb2F0KHByZWRfbGFiZWwgPT0gZ29sZF9sYWJlbCkKCiAgICAgICAgICAgICAgICBwX2Rpc3QgPSBbMS4wIC0gcF92YWwsIHBfdmFsXQogICAgICAgICAgICAgICAgZ19kaXN0ID0gWzEuMCAtIGdfdmFsLCBnX3ZhbF0KCiAgICAgICAgICAgICAgICBjb3JyZWN0cy5hcHBlbmQoaXNfY29ycikKICAgICAgICAgICAgICAgIGNvbmZpZGVuY2VzLmFwcGVuZChtYXgocF92YWwsIDEuMCAtIHBfdmFsKSkKICAgICAgICAgICAgICAgIGJyaWVycy5hcHBlbmQoYnJpZXJfc2NvcmUocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgc29mdHMuYXBwZW5kKHNvZnRfYWNjdXJhY3kocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbIm5vdWwiXS5hcHBlbmQoaXNfY29ycikKCiAgICAgICAgICAgIGVsaWYgcXR5cGUgPT0gInNjb3JlIjoKICAgICAgICAgICAgICAgIHBfc2NvcmUgPSBmbG9hdChwX2Fuc1sic2NvcmUiXSkKICAgICAgICAgICAgICAgIGdfc2NvcmUgPSBmbG9hdChnLmdldCgic2NvcmUiLCAwLjApKQogICAgICAgICAgICAgICAgc2NvcmVfbWFlcy5hcHBlbmQoYWJzKHBfc2NvcmUgLSBnX3Njb3JlKSkKICAgICAgICAgICAgICAgIHdpdGhpbl9vbmUuYXBwZW5kKGZsb2F0KGFicyhwX3Njb3JlIC0gZ19zY29yZSkgPD0gMS4wKSkKCiAgICAgICAgICAgICAgICBuX2xldmVscyA9IGxlbihxZGVmLmdldCgiY3JpdGVyaWEiLCBbXSkpCiAgICAgICAgICAgICAgICBwX3Byb2JzID0gW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KHN0cihpKSwgMC4wKSBmb3IgaSBpbiByYW5nZShuX2xldmVscyldCiAgICAgICAgICAgICAgICBpZiBzdW0ocF9wcm9icykgPiAwOgogICAgICAgICAgICAgICAgICAgIHBfcHJvYnMgPSBfbm9ybWFsaXNlKHBfcHJvYnMpCiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQobWF4KHJhbmdlKGxlbihwX3Byb2JzKSksIGtleT1sYW1iZGEgajogcF9wcm9ic1tqXSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKG1heChwX3Byb2JzKSkKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQocm91bmQocF9zY29yZSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKDAuNSkKCiAgICAgICAgICAgICAgICBnX2x2bCA9IGludChnLmdldCgibGFiZWwiLCBpbnQocm91bmQoZ19zY29yZSkpKSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwX2x2bCA9PSBnX2x2bCkKICAgICAgICAgICAgICAgIGNvcnJlY3RzLmFwcGVuZChpc19jb3JyKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbInNjb3JlIl0uYXBwZW5kKGlzX2NvcnIpCgogICAgICAgIGlmIChpICsgMSkgJSAxMDAgPT0gMDoKICAgICAgICAgICAgcHJpbnQoZiIgIHtpICsgMX0ve2xlbih0ZXN0X21ldGEpfSBjYXNlcyIsIGZsdXNoPVRydWUpCgogICAgcmVzdWx0ID0gewogICAgICAgICJjaGVja3BvaW50IjogY2hlY2twb2ludF9kaXIsCiAgICAgICAgInRlbXBlcmF0dXJlcyI6IHRlbXBlcmF0dXJlcywKICAgICAgICAibl9jYXNlcyI6IGxlbih0ZXN0X21ldGEpLAogICAgICAgICJuX2RlY2lzaW9ucyI6IGxlbihjb3JyZWN0cyksCiAgICAgICAgImFjY3VyYWN5Ijogcm91bmQoYWNjdXJhY3koY29ycmVjdHMpLCA0KSwKICAgICAgICAiZWNlIjogcm91bmQoZXhwZWN0ZWRfY2FsaWJyYXRpb25fZXJyb3IoY29uZmlkZW5jZXMsIGNvcnJlY3RzKSwgNCksCiAgICAgICAgImJyaWVyIjogcm91bmQoZmxvYXQoc3VtKGJyaWVycykgLyBsZW4oYnJpZXJzKSkgaWYgYnJpZXJzIGVsc2UgZmxvYXQoIm5hbiIpLCA0KSwKICAgICAgICAic29mdF9hY2N1cmFjeSI6IHJvdW5kKGZsb2F0KHN1bShzb2Z0cykgLyBsZW4oc29mdHMpKSBpZiBzb2Z0cyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgInNjb3JlX21hZSI6IHJvdW5kKGZsb2F0KHN1bShzY29yZV9tYWVzKSAvIGxlbihzY29yZV9tYWVzKSkgaWYgc2NvcmVfbWFlcyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgIndpdGhpbl9vbmVfbGV2ZWwiOiByb3VuZChmbG9hdChzdW0od2l0aGluX29uZSkgLyBsZW4od2l0aGluX29uZSkpIGlmIHdpdGhpbl9vbmUgZWxzZSBmbG9hdCgibmFuIiksIDQpLAogICAgICAgICJwZXJfdHlwZV9hY2N1cmFjeSI6IHtrOiByb3VuZChhY2N1cmFjeSh2KSwgNCkgZm9yIGssIHYgaW4gcGVyX3R5cGUuaXRlbXMoKSBpZiB2fSwKICAgIH0KICAgICMgYGJyaWVyYC9gc29mdF9hY2N1cmFjeWAvYGVjZWAgbWl4IHF1ZXN0aW9uIHR5cGVzOyByZWNvcmQgdGhlIGNvdW50cyB0aGV5IHJlc3Qgb24uCiAgICByZXN1bHRbIm5fYnJpZXJfZGVjaXNpb25zIl0gPSBsZW4oYnJpZXJzKQogICAgcmV0dXJuIHJlc3VsdAoKCmRlZiBtYWluKCkgLT4gaW50OgogICAgcGFyc2VyID0gYXJncGFyc2UuQXJndW1lbnRQYXJzZXIoZGVzY3JpcHRpb249X19kb2NfXykKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tY2hlY2twb2ludCIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLXRlc3QtbWV0YSIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRldmljZSIsIGRlZmF1bHQ9ImF1dG8iKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS10ZW1wZXJhdHVyZXMiLCBjaG9pY2VzPVsic2hpcHBlZCIsICJyYXciXSwgZGVmYXVsdD0ic2hpcHBlZCIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dCIsIGRlZmF1bHQ9Tm9uZSwKICAgICAgICAgICAgICAgICAgICAgICAgaGVscD0iT3V0cHV0IEpTT04gKGRlZmF1bHQ6IDxjaGVja3BvaW50Pi9ldmFsXzx0ZW1wZXJhdHVyZXM+Lmpzb24pIikKICAgIGFyZ3MgPSBwYXJzZXIucGFyc2VfYXJncygpCgogICAgd2l0aCBvcGVuKGFyZ3MudGVzdF9tZXRhLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIHRlc3RfbWV0YSA9IGpzb24ubG9hZChmKQogICAgcHJpbnQoZiJsb2FkZWQge2xlbih0ZXN0X21ldGEpfSB0ZXN0IGNhc2VzIiwgZmx1c2g9VHJ1ZSkKCiAgICByZXN1bHQgPSBldmFsdWF0ZV9jaGVja3BvaW50KGFyZ3MuY2hlY2twb2ludCwgdGVzdF9tZXRhLCBkZXZpY2U9YXJncy5kZXZpY2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRlbXBlcmF0dXJlcz1hcmdzLnRlbXBlcmF0dXJlcykKCiAgICBwcmludCgiXG4iICsgIj0iICogNjApCiAgICBwcmludChmImV2YWx1YXRpb246IHthcmdzLmNoZWNrcG9pbnR9ICh7YXJncy50ZW1wZXJhdHVyZXN9IHRlbXBlcmF0dXJlcykiKQogICAgcHJpbnQoIj0iICogNjApCiAgICBwcmludChmIiAgYWNjdXJhY3k6ICAgICB7cmVzdWx0WydhY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIEVDRTogICAgICAgICAge3Jlc3VsdFsnZWNlJ106LjRmfSIpCiAgICBwcmludChmIiAgQnJpZXI6ICAgICAgICB7cmVzdWx0WydicmllciddOi40Zn0iKQogICAgcHJpbnQoZiIgIHNvZnQgYWNjOiAgICAge3Jlc3VsdFsnc29mdF9hY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIHBlciB0eXBlOiAgICAge3Jlc3VsdFsncGVyX3R5cGVfYWNjdXJhY3knXX0iKQogICAgcHJpbnQoIj0iICogNjApCgogICAgb3V0X3BhdGggPSBhcmdzLm91dCBvciBvcy5wYXRoLmpvaW4oYXJncy5jaGVja3BvaW50LCBmImV2YWxfe2FyZ3MudGVtcGVyYXR1cmVzfS5qc29uIikKICAgIG9zLm1ha2VkaXJzKG9zLnBhdGguZGlybmFtZShvdXRfcGF0aCkgb3IgIi4iLCBleGlzdF9vaz1UcnVlKQogICAgd2l0aCBvcGVuKG91dF9wYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAganNvbi5kdW1wKHJlc3VsdCwgZiwgaW5kZW50PTIpCiAgICBwcmludChmIndyb3RlIHtvdXRfcGF0aH0iKQogICAgcmV0dXJuIDAKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgc3lzLmV4aXQobWFpbigpKQo=",+ "metrics.py": "IiIiTWV0cmljIGRlZmluaXRpb25zIGZvciB0aGUgU3RlcC00IGJlbmNobWFyaywgbWlycm9yaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCgpSZWZlcmVuY2U6IGNlbGwgMTQgb2YgYG5vdGVib29rcy9sYXlhX2ZpbmV0dW5lX3R5cGVkX2RlY2lzaW9uc18yeFQ0X2thZ2dsZS5pcHluYmAuIFRoZSBwdWJsaXNoZWQKMC43NjYgY2hlY2twb2ludCBhbmQgdGhlIG1haW50YWluZXIncyByZWZlcmVuY2UgdGFibGUgYXJlIHNjb3JlZCB3aXRoIHRoZXNlIGRlZmluaXRpb25zLCBzbyBhCnJlcHJvZHVjdGlvbiBoYXMgdG8gdXNlIHRoZW0gdG9vLCBub3QgYSBkaWZmZXJlbnRseS1kZWZpbmVkIGxpYnJhcnkgbWV0cmljOgoKLSAqKmFjY3VyYWN5Kiog4oCUIHRvcCByZXBvcnRlZCBvcHRpb24gYWdhaW5zdCB0aGUgZ29sZCBsYWJlbCwgb3ZlciBldmVyeSBkZWNpc2lvbiAoY2hvaWNlLCBzY29yZQogIGFuZCBub3VsKS4KLSAqKkVDRSoqIOKAlCBgbGF5YS5jb21tb24uZWNlX3Njb3JlYCBvZiB0aGUgdG9wIHByb2JhYmlsaXR5IGFnYWluc3QgY29ycmVjdG5lc3MsIDE1IGVxdWFsLXdpZHRoCiAgYmlucywgb3ZlciBldmVyeSBkZWNpc2lvbi4KLSAqKmJyaWVyKiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIGRlY2lzaW9ucyBvZiBgc3VtKChwIC0gZ29sZCkqKjIpYC4gVGhlIG5vdGVib29rIGRvZXMKICBub3QgYXBwZW5kIHNjb3JlIGRlY2lzaW9ucyB0byB0aGlzIG1lYW4sIHNvIGEgc2NvcmUgY2hhbmdlIG5ldmVyIG1vdmVzIEJyaWVyOyBrZXB0IGFzLWlzIHNvIHRoZQogIG51bWJlciBjb21wYXJlcyB3aXRoIHRoZSByZWZlcmVuY2UuCi0gKipzb2Z0X2FjY3VyYWN5Kiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIG9mIGBzdW0ocCAqIGdvbGQpYC4KCmBnb2xkYCBpcyB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gKGBnb2xkLnByb2JhYmlsaXRpZXNgKSwgbm90IGEgb25lLWhvdCBsYWJlbCwgc28gQnJpZXIgYW5kIHNvZnQKYWNjdXJhY3kgbWVhc3VyZSB0aGUgZGlzdHJpYnV0aW9uIHRoZSBydW4gd2FzIHRyYWluZWQgdG8gbWF0Y2guCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgpmcm9tIHR5cGluZyBpbXBvcnQgU2VxdWVuY2UKCmltcG9ydCBudW1weSBhcyBucAoKZnJvbSBsYXlhLmNvbW1vbiBpbXBvcnQgZWNlX3Njb3JlCgoKZGVmIGFjY3VyYWN5KGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICByZXR1cm4gZmxvYXQobnAubWVhbihjb3JyZWN0KSkgaWYgbGVuKGNvcnJlY3QpIGVsc2UgZmxvYXQoIm5hbiIpCgoKZGVmIGV4cGVjdGVkX2NhbGlicmF0aW9uX2Vycm9yKGNvbmZpZGVuY2VzOiBTZXF1ZW5jZVtmbG9hdF0sIGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJpbnM6IGludCA9IDE1KSAtPiBmbG9hdDoKICAgICIiIlRoaW4gd3JhcHBlciBvdmVyIHRoZSByZXBvc2l0b3J5J3MgYGNvbW1vbi5lY2Vfc2NvcmVgLCBhcyB0aGUgbm90ZWJvb2sgY2FsbHMgaXQuIiIiCiAgICBpZiBub3QgbGVuKGNvbmZpZGVuY2VzKToKICAgICAgICByZXR1cm4gZmxvYXQoIm5hbiIpCiAgICByZXR1cm4gZmxvYXQoZWNlX3Njb3JlKG5wLmFzYXJyYXkoY29uZmlkZW5jZXMsIGR0eXBlPWZsb2F0KSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbnAuYXNhcnJheShjb3JyZWN0LCBkdHlwZT1mbG9hdCksIGJpbnM9YmlucykpCgoKZGVmIGJyaWVyX3Njb3JlKHByb2JzOiBTZXF1ZW5jZVtmbG9hdF0sIGdvbGQ6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICBwID0gbnAuYXNhcnJheShwcm9icywgZHR5cGU9ZmxvYXQpCiAgICBnID0gbnAuYXNhcnJheShnb2xkLCBkdHlwZT1mbG9hdCkKICAgIHJldHVybiBmbG9hdCgoKHAgLSBnKSAqKiAyKS5zdW0oKSkKCgpkZWYgc29mdF9hY2N1cmFjeShwcm9iczogU2VxdWVuY2VbZmxvYXRdLCBnb2xkOiBTZXF1ZW5jZVtmbG9hdF0pIC0+IGZsb2F0OgogICAgcCA9IG5wLmFzYXJyYXkocHJvYnMsIGR0eXBlPWZsb2F0KQogICAgZyA9IG5wLmFzYXJyYXkoZ29sZCwgZHR5cGU9ZmxvYXQpCiAgICByZXR1cm4gZmxvYXQoKHAgKiBnKS5zdW0oKSkK"+}+for _n, _b in _EMBEDDED.items():+ (HARNESS / _n).write_bytes(base64.b64decode(_b))+sys.path.insert(0, str(HARNESS))++os.environ["CUDA_VISIBLE_DEVICES"] = "0"+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")+import torch # noqa: E402++n_gpu = torch.cuda.device_count()+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu+print("GPU", torch.cuda.get_device_name(0), flush=True)++RESULTS_PATH = WORK / "step4_gate_d_results.json"+++def flush_results():+ RESULTS_PATH.write_text(json.dumps(RESULTS, indent=2), encoding="utf-8")+++def run(cmd):+ print("\n$ " + " ".join(cmd), flush=True)+ p = subprocess.run(cmd, check=True, text=True, capture_output=True)+ sys.stdout.write(p.stdout)+ if p.stderr:+ sys.stderr.write(p.stderr)+ sys.stdout.flush()+ return p.stdout+++run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):+ try:+ __import__(_mod)+ except Exception:+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])++from huggingface_hub import snapshot_download # noqa: E402++base_dir = snapshot_download("convaiinnovations/laya")+base_cfg = json.loads((Path(base_dir) / "rl_agent_config.json").read_text(encoding="utf-8"))++run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])+test_meta = json.loads((WORK / "data" / "test_meta.json").read_text(encoding="utf-8"))++from evaluate import evaluate_checkpoint # noqa: E402++RESULTS = {+ "meta": {+ "pinned_sha": PINNED_SHA,+ "torch": torch.__version__,+ "gpu": torch.cuda.get_device_name(0),+ "visible_gpus": n_gpu,+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},+ "n_train_cases": 1200,+ "n_test_cases": len(test_meta),+ "reused": {"rlcd_seed0": "Gate R kernel (separate session, same SHA/data/command)"},+ },+ "rlcd_seed0": {+ "checkpoint": "Gate R kernel run_rlcd_seed0 (reused)",+ "temperatures": "shipped", "n_cases": 400, "n_decisions": 2000,+ "accuracy": 0.775, "ece": 0.1504, "brier": 0.0519, "soft_accuracy": 0.5654,+ "per_type_accuracy": {"choice": 0.7417, "noul": 0.8517, "score": 0.7425},+ "loss": "rlcd", "seed": 0, "wall_clock_s": 2691.3,+ },+}++# --- base sanity row, must match the Gate-R reference -----------------------+base_row = evaluate_checkpoint("convaiinnovations/laya", test_meta, device="cuda",+ temperatures="shipped")+RESULTS["base"] = base_row+base_ok = (abs(base_row["accuracy"] - 0.362) <= 0.01 and abs(base_row["ece"] - 0.1741) <= 0.01+ and abs(base_row["brier"] - 0.3287) <= 0.01)+print("BASE_SANITY", "OK" if base_ok else "FAIL", json.dumps(base_row), flush=True)+flush_results()+if not base_ok:+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: base sanity drift")++# --- the five remaining cells ----------------------------------------------+CELLS = [("rlcd", 1), ("soft-ce", 0), ("soft-ce", 1), ("rlcd", 2), ("soft-ce", 2)]+++def train(loss, seed):+ out = WORK / ("run_" + loss.replace("-", "_") + "_seed%d" % seed)+ cmd = [sys.executable, "-m", "laya.train_cli",+ "--data", str(WORK / "data" / "train.jsonl"),+ "--out", str(out),+ "--base", "convaiinnovations/laya",+ "--loss", loss, "--seed", str(seed),+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",+ "--max-len", "1024", "--head-max-len", "256",+ "--device", "cuda"]+ t0 = time.time()+ stdout = run(cmd)+ dt = round(time.time() - t0, 1)+ cfg = json.loads((out / "rl_agent_config.json").read_text(encoding="utf-8"))+ lt = (cfg.get("training") or {}).get("laya_train", {})+ split_line = next((ln for ln in stdout.splitlines() if ln.startswith("train items")), "")+ checks = {+ "loss": lt.get("loss") == loss,+ "seed": lt.get("seed") == seed,+ "epochs": lt.get("epochs") == 4,+ "micro_batch": lt.get("micro_batch") == 8,+ "grad_accum": lt.get("grad_accum") == 8,+ "encoder_lr": abs(float(lt.get("encoder_lr", 0)) - 2.5e-5) < 1e-12,+ "head_lr": abs(float(lt.get("head_lr", 0)) - 1e-4) < 1e-12,+ "max_len": lt.get("max_len") == 1024,+ "head_max_len": lt.get("head_max_len") == 256,+ "calib_seed": lt.get("calib_seed") == 20260922,+ "no_shuffle": list(lt.get("shuffle_options") or []) == [],+ "counts": "train items 5600, calibration items 400, skipped {}" in split_line,+ }+ return out, dt, cfg, checks, split_line+++for loss, seed in CELLS:+ out, dt, cfg, checks, split_line = train(loss, seed)+ print("CELL %s seed%d checks %s" % (loss, seed, json.dumps(checks)), flush=True)+ bad = sorted(k for k, v in checks.items() if not v)+ if bad:+ RESULTS["%s_seed%d" % (loss, seed)] = {"error": "protocol mismatch: " + ",".join(bad),+ "split_line": split_line}+ flush_results()+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: %s seed%d -> %s" % (loss, seed, bad))++ res = evaluate_checkpoint(str(out), test_meta, device="cuda", temperatures="shipped")+ finite = all(isinstance(res.get(k), (int, float)) and math.isfinite(res[k])+ for k in ("accuracy", "ece", "brier"))+ res.update({+ "loss": loss, "seed": seed, "wall_clock_s": dt,+ "temperature": cfg.get("temperature"),+ "calibration": (cfg.get("training") or {}).get("laya_train_calibration"),+ "split_line": split_line, "finite": finite,+ })+ RESULTS["%s_seed%d" % (loss, seed)] = res+ flush_results()+ print("RESULT %s seed%d %s" % (loss, seed, json.dumps(res)), flush=True)+ if not finite:+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: non-finite metric %s seed%d" % (loss, seed))++print("ALL_CELLS_DONE", flush=True)+print("RESULTS " + json.dumps(RESULTS, indent=2), flush=True)