Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

research(step4): single-T4 Kaggle Gate D kernel for the remaining 5 cells

Bruce-Yii70f17ba3b2a6
diff --git a/research/scripts/step4_benchmark/kaggle/build_gate_d.py b/research/scripts/step4_benchmark/kaggle/build_gate_d.py
new file mode 100644
index 00000000..7850b451
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/build_gate_d.py
@@ -0,0 +1,222 @@
+"""Generate the Gate D Kaggle kernel: the remaining 5 cells of the 3-seed x 2-loss panel.
+
+One single-T4 session runs the five remaining trainings sequentially under the pinned Gate-R
+contract, fail-closed after each: checkpoint exists, `training.laya_train` records the intended
+loss/seed/config, counts match (5600/400/skipped {}), metrics finite, and the base sanity row stays
+within tolerance of the Gate-R reference. A hard mismatch writes the partial results and aborts.
+
+RLCD seed 0 is reused from the Gate R kernel (different session, same pinned SHA/data/command), as
+the Coordinator explicitly allowed; it is embedded as the sixth cell.
+
+ python research/scripts/step4_benchmark/kaggle/build_gate_d.py
+ KAGGLE_API_TOKEN=$(kaggle auth print-access-token) kaggle kernels push -p <this dir>/gate_d
+"""
+from __future__ import annotations
+
+import base64
+import json
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+HARNESS = HERE.parent
+OUT = HERE / "gate_d"
+EMBED = ("prepare_data.py", "evaluate.py", "metrics.py")
+PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
+KERNEL_ID = "yuyi722333/laya-step4-gate-d"
+
+TEMPLATE = '''"""#887 Step 4 Gate D, generated by build_gate_d.py -- do not edit here."""
+import base64
+import json
+import math
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+PINNED_SHA = "__PINNED_SHA__"
+WORK = Path("/kaggle/working")
+HARNESS = WORK / "step4_benchmark"
+HARNESS.mkdir(parents=True, exist_ok=True)
+
+_EMBEDDED = __EMBEDDED__
+for _n, _b in _EMBEDDED.items():
+ (HARNESS / _n).write_bytes(base64.b64decode(_b))
+sys.path.insert(0, str(HARNESS))
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+import torch # noqa: E402
+
+n_gpu = torch.cuda.device_count()
+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu
+print("GPU", torch.cuda.get_device_name(0), flush=True)
+
+RESULTS_PATH = WORK / "step4_gate_d_results.json"
+
+
+def flush_results():
+ RESULTS_PATH.write_text(json.dumps(RESULTS, indent=2), encoding="utf-8")
+
+
+def run(cmd):
+ print("\\n$ " + " ".join(cmd), flush=True)
+ p = subprocess.run(cmd, check=True, text=True, capture_output=True)
+ sys.stdout.write(p.stdout)
+ if p.stderr:
+ sys.stderr.write(p.stderr)
+ sys.stdout.flush()
+ return p.stdout
+
+
+run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
+ try:
+ __import__(_mod)
+ except Exception:
+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])
+
+from huggingface_hub import snapshot_download # noqa: E402
+
+base_dir = snapshot_download("convaiinnovations/laya")
+base_cfg = json.loads((Path(base_dir) / "rl_agent_config.json").read_text(encoding="utf-8"))
+
+run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
+test_meta = json.loads((WORK / "data" / "test_meta.json").read_text(encoding="utf-8"))
+
+from evaluate import evaluate_checkpoint # noqa: E402
+
+RESULTS = {
+ "meta": {
+ "pinned_sha": PINNED_SHA,
+ "torch": torch.__version__,
+ "gpu": torch.cuda.get_device_name(0),
+ "visible_gpus": n_gpu,
+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
+ "n_train_cases": 1200,
+ "n_test_cases": len(test_meta),
+ "reused": {"rlcd_seed0": "Gate R kernel (separate session, same SHA/data/command)"},
+ },
+ "rlcd_seed0": {
+ "checkpoint": "Gate R kernel run_rlcd_seed0 (reused)",
+ "temperatures": "shipped", "n_cases": 400, "n_decisions": 2000,
+ "accuracy": 0.775, "ece": 0.1504, "brier": 0.0519, "soft_accuracy": 0.5654,
+ "per_type_accuracy": {"choice": 0.7417, "noul": 0.8517, "score": 0.7425},
+ "loss": "rlcd", "seed": 0, "wall_clock_s": 2691.3,
+ },
+}
+
+# --- base sanity row, must match the Gate-R reference -----------------------
+base_row = evaluate_checkpoint("convaiinnovations/laya", test_meta, device="cuda",
+ temperatures="shipped")
+RESULTS["base"] = base_row
+base_ok = (abs(base_row["accuracy"] - 0.362) <= 0.01 and abs(base_row["ece"] - 0.1741) <= 0.01
+ and abs(base_row["brier"] - 0.3287) <= 0.01)
+print("BASE_SANITY", "OK" if base_ok else "FAIL", json.dumps(base_row), flush=True)
+flush_results()
+if not base_ok:
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: base sanity drift")
+
+# --- the five remaining cells ----------------------------------------------
+CELLS = [("rlcd", 1), ("soft-ce", 0), ("soft-ce", 1), ("rlcd", 2), ("soft-ce", 2)]
+
+
+def train(loss, seed):
+ out = WORK / ("run_" + loss.replace("-", "_") + "_seed%d" % seed)
+ cmd = [sys.executable, "-m", "laya.train_cli",
+ "--data", str(WORK / "data" / "train.jsonl"),
+ "--out", str(out),
+ "--base", "convaiinnovations/laya",
+ "--loss", loss, "--seed", str(seed),
+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
+ "--max-len", "1024", "--head-max-len", "256",
+ "--device", "cuda"]
+ t0 = time.time()
+ stdout = run(cmd)
+ dt = round(time.time() - t0, 1)
+ cfg = json.loads((out / "rl_agent_config.json").read_text(encoding="utf-8"))
+ lt = (cfg.get("training") or {}).get("laya_train", {})
+ split_line = next((ln for ln in stdout.splitlines() if ln.startswith("train items")), "")
+ checks = {
+ "loss": lt.get("loss") == loss,
+ "seed": lt.get("seed") == seed,
+ "epochs": lt.get("epochs") == 4,
+ "micro_batch": lt.get("micro_batch") == 8,
+ "grad_accum": lt.get("grad_accum") == 8,
+ "encoder_lr": abs(float(lt.get("encoder_lr", 0)) - 2.5e-5) < 1e-12,
+ "head_lr": abs(float(lt.get("head_lr", 0)) - 1e-4) < 1e-12,
+ "max_len": lt.get("max_len") == 1024,
+ "head_max_len": lt.get("head_max_len") == 256,
+ "calib_seed": lt.get("calib_seed") == 20260922,
+ "no_shuffle": list(lt.get("shuffle_options") or []) == [],
+ "counts": "train items 5600, calibration items 400, skipped {}" in split_line,
+ }
+ return out, dt, cfg, checks, split_line
+
+
+for loss, seed in CELLS:
+ out, dt, cfg, checks, split_line = train(loss, seed)
+ print("CELL %s seed%d checks %s" % (loss, seed, json.dumps(checks)), flush=True)
+ bad = sorted(k for k, v in checks.items() if not v)
+ if bad:
+ RESULTS["%s_seed%d" % (loss, seed)] = {"error": "protocol mismatch: " + ",".join(bad),
+ "split_line": split_line}
+ flush_results()
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: %s seed%d -> %s" % (loss, seed, bad))
+
+ res = evaluate_checkpoint(str(out), test_meta, device="cuda", temperatures="shipped")
+ finite = all(isinstance(res.get(k), (int, float)) and math.isfinite(res[k])
+ for k in ("accuracy", "ece", "brier"))
+ res.update({
+ "loss": loss, "seed": seed, "wall_clock_s": dt,
+ "temperature": cfg.get("temperature"),
+ "calibration": (cfg.get("training") or {}).get("laya_train_calibration"),
+ "split_line": split_line, "finite": finite,
+ })
+ RESULTS["%s_seed%d" % (loss, seed)] = res
+ flush_results()
+ print("RESULT %s seed%d %s" % (loss, seed, json.dumps(res)), flush=True)
+ if not finite:
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: non-finite metric %s seed%d" % (loss, seed))
+
+print("ALL_CELLS_DONE", flush=True)
+print("RESULTS " + json.dumps(RESULTS, indent=2), flush=True)
+'''
+
+
+def main() -> int:
+ OUT.mkdir(parents=True, exist_ok=True)
+ embedded = {name: base64.b64encode((HARNESS / name).read_bytes()).decode("ascii")
+ for name in EMBED}
+ script = (TEMPLATE.replace("__PINNED_SHA__", PINNED_SHA)
+ .replace("__EMBEDDED__", json.dumps(embedded, indent=1)))
+ (OUT / "step4_gate_d.py").write_text(script, encoding="utf-8", newline="\n")
+
+ metadata = {
+ "id": KERNEL_ID,
+ "title": "Laya Step 4 Gate D",
+ "code_file": "step4_gate_d.py",
+ "language": "python",
+ "kernel_type": "script",
+ "is_private": True,
+ "enable_gpu": True,
+ "enable_tpu": False,
+ "enable_internet": True,
+ "machine_shape": "NvidiaTeslaT4",
+ "dataset_sources": [],
+ "competition_sources": [],
+ "kernel_sources": [],
+ "model_sources": [],
+ }
+ (OUT / "kernel-metadata.json").write_text(json.dumps(metadata, indent=2) + "\n",
+ encoding="utf-8", newline="\n")
+ print("wrote", OUT / "step4_gate_d.py", "and", OUT / "kernel-metadata.json")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.json b/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.json
new file mode 100644
index 00000000..56fb0b84
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/gate_d/kernel-metadata.json
@@ -0,0 +1,16 @@
+{
+ "id": "yuyi722333/laya-step4-gate-d",
+ "title": "Laya Step 4 Gate D",
+ "code_file": "step4_gate_d.py",
+ "language": "python",
+ "kernel_type": "script",
+ "is_private": true,
+ "enable_gpu": true,
+ "enable_tpu": false,
+ "enable_internet": true,
+ "machine_shape": "NvidiaTeslaT4",
+ "dataset_sources": [],
+ "competition_sources": [],
+ "kernel_sources": [],
+ "model_sources": []
+}
diff --git a/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.py b/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.py
new file mode 100644
index 00000000..814184b2
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/gate_d/step4_gate_d.py
@@ -0,0 +1,165 @@
+"""#887 Step 4 Gate D, generated by build_gate_d.py -- do not edit here."""
+import base64
+import json
+import math
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
+WORK = Path("/kaggle/working")
+HARNESS = WORK / "step4_benchmark"
+HARNESS.mkdir(parents=True, exist_ok=True)
+
+_EMBEDDED = {
+ "prepare_data.py": "IiIiUHJlcGFyZSBMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyBmb3IgYGxheWEtdHJhaW5gIGFuZCBmb3IgZXZhbHVhdGlvbi4KClRoZSBiZW5jaG1hcmsncyBvd24gcm93cyBhcmUgYHtzdGF0ZSwgcXVlc3Rpb25zLCBnb2xkfWAgd2l0aCB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gcGVyCnF1ZXN0aW9uLiBUaGF0IGlzIGV4YWN0bHkgdGhlIHNjaGVtYSBgbGF5YS50cmFpbmAgcmVhZHMsIHNvIHRoZSBjb252ZXJzaW9uIGlzIGEgcGFyc2UsIG5vdCBhCnJlc2hhcGU6IHRoZSB0d28gSlNPTi1zdHJpbmcgY29sdW1ucyBhcmUgZGVjb2RlZCBhbmQgdGhlIHJvdyBpcyB3cml0dGVuIGJhY2sgb3V0IGFzIEpTT05MLgoKU3BsaXRzOgoKKiBgYHRyYWluLmpzb25sYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0cmFpbmBgIHNwbGl0ICgxLDIwMCBjYXNlcyAvIDYsMDAwIGRlY2lzaW9ucyksIHRoZSBpbnB1dCB0bwogIGBgbGF5YS10cmFpbmBgLiBUaGUgY2FsaWJyYXRpb24gc2xpY2UgaXMgaGVsZCBvdXQgKmluc2lkZSogYGBsYXlhLXRyYWluYGAgYXQgYSBmaXhlZCBzZWVkCiAgKGBgY2FsaWJfZnJhY2BgIDAuMSwgY2FwcGVkIGF0IDQwMCwgc2VlZCAyMDI2MDkyMiksIG1hdGNoaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCiogYGB0ZXN0Lmpzb25sYGAgLyBgYHRlc3RfbWV0YS5qc29uYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0ZXN0YGAgc3BsaXQgKDQwMCBjYXNlcyAvIDIsMDAwCiAgZGVjaXNpb25zKSwgdXNlZCBieSBgYGV2YWx1YXRlLnB5YGAuIGBgdGVzdF9tZXRhLmpzb25gYCBrZWVwcyBgYGlkYGAgYW5kIGBgd29ya2Zsb3dgYCBzbwogIHBlci13b3JrZmxvdyBhY2N1cmFjeSBjYW4gYmUgcmVwb3J0ZWQgYWxvbmdzaWRlIHRoZSBwb29sZWQgbnVtYmVyLgoKVXNhZ2U6CiAgICBweXRob24gcHJlcGFyZV9kYXRhLnB5IC0tb3V0LWRpciBkYXRhL3R5cGVkX2RlY2lzaW9ucwoiIiIKZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKaW1wb3J0IGFyZ3BhcnNlCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgc3lzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QKCgpkZWYgX3BhcnNlKHZhbHVlOiBBbnkpIC0+IEFueToKICAgIGlmIGlzaW5zdGFuY2UodmFsdWUsIHN0cik6CiAgICAgICAgcmV0dXJuIGpzb24ubG9hZHModmFsdWUpCiAgICByZXR1cm4gdmFsdWUKCgpkZWYgY2FzZV90b19yZWNvcmQocm93OiBEaWN0W3N0ciwgQW55XSkgLT4gRGljdFtzdHIsIEFueV06CiAgICAiIiJPbmUgZGF0YXNldCByb3cgYXMgYSBgbGF5YS10cmFpbmAgcmVjb3JkIChge3N0YXRlLCBxdWVzdGlvbnMsIGdvbGR9YCkuIiIiCiAgICByZXR1cm4gewogICAgICAgICJzdGF0ZSI6IF9wYXJzZShyb3dbInN0YXRlIl0pLAogICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2Uocm93WyJxdWVzdGlvbnMiXSksCiAgICAgICAgImdvbGQiOiBfcGFyc2Uocm93WyJnb2xkIl0pLAogICAgfQoKCmRlZiBsb2FkX3NwbGl0KHNwbGl0OiBzdHIsIGRhdGFzZXQ6IHN0ciA9ICJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpIC0+IExpc3RbRGljdFtzdHIsIEFueV1dOgogICAgdHJ5OgogICAgICAgIGZyb20gZGF0YXNldHMgaW1wb3J0IGxvYWRfZGF0YXNldAogICAgZXhjZXB0IEltcG9ydEVycm9yOgogICAgICAgIHByaW50KCJFUlJPUjogdGhlIGRhdGFzZXRzIHBhY2thZ2UgaXMgcmVxdWlyZWQgKHBpcCBpbnN0YWxsIGRhdGFzZXRzKSIsIGZpbGU9c3lzLnN0ZGVycikKICAgICAgICBzeXMuZXhpdCgxKQogICAgZHMgPSBsb2FkX2RhdGFzZXQoZGF0YXNldCwgImFsbCIsIHNwbGl0PXNwbGl0KQogICAgcmV0dXJuIGxpc3QoZHMpCgoKZGVmIHdyaXRlX3NwbGl0KHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIsIHNwbGl0OiBzdHIpIC0+IGludDoKICAgIHJlY29yZHMgPSBbY2FzZV90b19yZWNvcmQocikgZm9yIHIgaW4gcm93c10KICAgIHBhdGggPSBvcy5wYXRoLmpvaW4ob3V0X2RpciwgZiJ7c3BsaXR9Lmpzb25sIikKICAgIHdpdGggb3BlbihwYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAgZm9yIHJlYyBpbiByZWNvcmRzOgogICAgICAgICAgICBmLndyaXRlKGpzb24uZHVtcHMocmVjLCBlbnN1cmVfYXNjaWk9RmFsc2UpICsgIlxuIikKICAgIHJldHVybiBsZW4ocmVjb3JkcykKCgpkZWYgd3JpdGVfdGVzdF9tZXRhKHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIpIC0+IGludDoKICAgIG1ldGEgPSBbXQogICAgZm9yIHIgaW4gcm93czoKICAgICAgICBtZXRhLmFwcGVuZCh7CiAgICAgICAgICAgICJpZCI6IHIuZ2V0KCJpZCIsICIiKSwKICAgICAgICAgICAgIndvcmtmbG93Ijogci5nZXQoIndvcmtmbG93IiwgIiIpLAogICAgICAgICAgICAic3RhdGUiOiBfcGFyc2Uoclsic3RhdGUiXSksCiAgICAgICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2UoclsicXVlc3Rpb25zIl0pLAogICAgICAgICAgICAiZ29sZCI6IF9wYXJzZShyWyJnb2xkIl0pLAogICAgICAgIH0pCiAgICBwYXRoID0gb3MucGF0aC5qb2luKG91dF9kaXIsICJ0ZXN0X21ldGEuanNvbiIpCiAgICB3aXRoIG9wZW4ocGF0aCwgInciLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIGpzb24uZHVtcChtZXRhLCBmLCBpbmRlbnQ9MikKICAgIHJldHVybiBsZW4obWV0YSkKCgpkZWYgbWFpbigpIC0+IGludDoKICAgIHBhcnNlciA9IGFyZ3BhcnNlLkFyZ3VtZW50UGFyc2VyKGRlc2NyaXB0aW9uPV9fZG9jX18pCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dC1kaXIiLCBkZWZhdWx0PSJkYXRhL3R5cGVkX2RlY2lzaW9ucyIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRhdGFzZXQiLCBkZWZhdWx0PSJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpCiAgICBhcmdzID0gcGFyc2VyLnBhcnNlX2FyZ3MoKQoKICAgIG9zLm1ha2VkaXJzKGFyZ3Mub3V0X2RpciwgZXhpc3Rfb2s9VHJ1ZSkKICAgIGZvciBzcGxpdCBpbiAoInRyYWluIiwgInRlc3QiKToKICAgICAgICByb3dzID0gbG9hZF9zcGxpdChzcGxpdCwgYXJncy5kYXRhc2V0KQogICAgICAgIG4gPSB3cml0ZV9zcGxpdChyb3dzLCBhcmdzLm91dF9kaXIsIHNwbGl0KQogICAgICAgIHByaW50KGYie3NwbGl0fToge259IGNhc2VzIC0+IHtvcy5wYXRoLmpvaW4oYXJncy5vdXRfZGlyLCBzcGxpdCArICcuanNvbmwnKX0iKQogICAgICAgIGlmIHNwbGl0ID09ICJ0ZXN0IjoKICAgICAgICAgICAgbSA9IHdyaXRlX3Rlc3RfbWV0YShyb3dzLCBhcmdzLm91dF9kaXIpCiAgICAgICAgICAgIHByaW50KGYidGVzdCBtZXRhOiB7bX0gY2FzZXMgLT4ge29zLnBhdGguam9pbihhcmdzLm91dF9kaXIsICd0ZXN0X21ldGEuanNvbicpfSIpCiAgICByZXR1cm4gMAoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBzeXMuZXhpdChtYWluKCkpCg==",
+ "evaluate.py": "IiIiRXZhbHVhdGUgYSBmaW5lLXR1bmVkIGNoZWNrcG9pbnQgb24gdGhlIHR5cGVkLWRlY2lzaW9ucyB0ZXN0IHNwbGl0LgoKTWlycm9ycyBjZWxsIDE0IG9mIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2ssIHJlc3RyaWN0ZWQgdG8gdGhlIHRocmVlIG1ldHJpY3MgdGhlIFN0ZXAtNCBjb250cmFjdApuYW1lczogKiphY2N1cmFjeSoqLCAqKkVDRSoqIGFuZCAqKkJyaWVyKiouIFRoZSAyMC1vcHRpb24gb3JkZXItZmxpcCBzdXJmYWNlIGlzIGRlbGliZXJhdGVseSBub3QKaGVyZTogaXQgYmVsb25ncyB0byBgbWFzc2l2ZV9pbnRlbnQuZW5gLCBub3QgdHlwZWQtZGVjaXNpb25zIChzZWUgUkVBRE1FKS4KCkNhbGlicmF0aW9uIGlzIG5vdCBieXRlLWNvbXBhcmFibGUgYWNyb3NzIGNvbnRyYWN0cy4gYGxheWEudHJhaW5gIGZpdHMgdGVtcGVyYXR1cmVzIHdpdGgKYGZpdF90ZW1wZXJhdHVyZV9tYXBgIChwZXIgdHlwZSwgYW5kIHBlciBvcHRpb24tY291bnQgYnVja2V0IGFib3ZlIGEgZmxvb3IpIHVuZGVyIHRoZSBydW50aW1lCmNsYW1wIGBbMC41LCA1LjBdYDsgdGhlIG9sZCBub3RlYm9vayBmaXR0ZWQgb25lIExCRkdTIHRlbXBlcmF0dXJlIHBlciB0eXBlIHVuZGVyIGBbMC4xLCAxMF1gLgpTbyBFQ0UgYW5kIEJyaWVyIGFyZSByZXBvcnRlZCB1bmRlciAqKmJvdGgqKiBwYXRoczoKCi0gYGAtLXRlbXBlcmF0dXJlcyBzaGlwcGVkYGAgKGRlZmF1bHQpIOKAlCB0aGUgY2hlY2twb2ludCdzIG93biBmaXR0ZWQgYHRlbXBlcmF0dXJlYCBhbmQKICBgdGVtcGVyYXR1cmVfYnlfb3B0aW9uc2AsIHdoaWNoIGlzIHdoYXQgdGhlIG5vdGVib29rJ3MgbnVtYmVyIGFsc28gdXNlZDsKLSBgYC0tdGVtcGVyYXR1cmVzIHJhd2BgIOKAlCB0ZW1wZXJhdHVyZSAxLjAgZXZlcnl3aGVyZSwgaXNvbGF0aW5nIHRoZSB0cmFpbmluZyBmcm9tIHRoZQogIGNhbGlicmF0aW9uIGNvbnRyYWN0LgoKQWNjdXJhY3kgaXMgdW5hZmZlY3RlZCBieSB0ZW1wZXJhdHVyZSAoaXQgb25seSByZXNjYWxlcyB0aGUgbG9naXRzKSwgc28gaXQgaXMgdGhlIGNsZWFuCnJlcHJvZHVjdGlvbiBzaWduYWwuCgpVc2FnZToKICAgIHB5dGhvbiBldmFsdWF0ZS5weSAtLWNoZWNrcG9pbnQgcnVucy9zdGVwNC9ybGNkLXNlZWQwIFwKICAgICAgICAtLXRlc3QtbWV0YSBkYXRhL3R5cGVkX2RlY2lzaW9ucy90ZXN0X21ldGEuanNvbiBcCiAgICAgICAgLS1vdXQgcnVucy9zdGVwNC9ybGNkLXNlZWQwL2V2YWxfc2hpcHBlZC5qc29uCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgppbXBvcnQgYXJncGFyc2UKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCBzeXMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdAoKc3lzLnBhdGguaW5zZXJ0KDAsIG9zLnBhdGguZGlybmFtZShvcy5wYXRoLmFic3BhdGgoX19maWxlX18pKSkKZnJvbSBtZXRyaWNzIGltcG9ydCAoICAjIG5vcWE6IEU0MDIKICAgIGFjY3VyYWN5LAogICAgYnJpZXJfc2NvcmUsCiAgICBleHBlY3RlZF9jYWxpYnJhdGlvbl9lcnJvciwKICAgIHNvZnRfYWNjdXJhY3ksCikKCgpkZWYgX25vcm1hbGlzZSh2YWx1ZXM6IExpc3RbZmxvYXRdKSAtPiBMaXN0W2Zsb2F0XToKICAgIHRvdGFsID0gc3VtKHZhbHVlcykKICAgIGlmIHRvdGFsID4gMDoKICAgICAgICByZXR1cm4gW3YgLyB0b3RhbCBmb3IgdiBpbiB2YWx1ZXNdCiAgICByZXR1cm4gdmFsdWVzCgoKZGVmIGV2YWx1YXRlX2NoZWNrcG9pbnQoY2hlY2twb2ludF9kaXI6IHN0ciwgdGVzdF9tZXRhOiBMaXN0W0RpY3Rbc3RyLCBBbnldXSwKICAgICAgICAgICAgICAgICAgICAgICAgZGV2aWNlOiBzdHIgPSAiYXV0byIsIHRlbXBlcmF0dXJlczogc3RyID0gInNoaXBwZWQiKSAtPiBEaWN0W3N0ciwgQW55XToKICAgIGltcG9ydCBsYXlhCgogICAgYWdlbnQgPSBsYXlhLmxvYWQoY2hlY2twb2ludF9kaXIsIGRldmljZT1kZXZpY2UpCiAgICBpZiB0ZW1wZXJhdHVyZXMgPT0gInJhdyI6CiAgICAgICAgYWdlbnQudGVtcGVyYXR1cmUgPSBbMS4wLCAxLjAsIDEuMF0KICAgICAgICBhZ2VudC50ZW1wZXJhdHVyZV9ieV9vcHRpb25zID0ge30KICAgIGVsaWYgdGVtcGVyYXR1cmVzICE9ICJzaGlwcGVkIjoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJ0ZW1wZXJhdHVyZXMgbXVzdCBiZSAnc2hpcHBlZCcgb3IgJ3JhdycsIGdvdCAlciIgJSAodGVtcGVyYXR1cmVzLCkpCgogICAgY29ycmVjdHM6IExpc3RbZmxvYXRdID0gW10KICAgIGNvbmZpZGVuY2VzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBicmllcnM6IExpc3RbZmxvYXRdID0gW10KICAgIHNvZnRzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBwZXJfdHlwZTogRGljdFtzdHIsIExpc3RbZmxvYXRdXSA9IHsiY2hvaWNlIjogW10sICJub3VsIjogW10sICJzY29yZSI6IFtdfQogICAgc2NvcmVfbWFlczogTGlzdFtmbG9hdF0gPSBbXQogICAgd2l0aGluX29uZTogTGlzdFtmbG9hdF0gPSBbXQoKICAgIGZvciBpLCBjYXNlIGluIGVudW1lcmF0ZSh0ZXN0X21ldGEpOgogICAgICAgIHN0YXRlID0gY2FzZVsic3RhdGUiXQogICAgICAgIHF1ZXN0aW9ucyA9IGNhc2VbInF1ZXN0aW9ucyJdCiAgICAgICAgZ29sZCA9IGNhc2VbImdvbGQiXQogICAgICAgIGFuc3dlcnMgPSBhZ2VudC5wcmVkaWN0KHN0YXRlLCBxdWVzdGlvbnMpWyJhbnN3ZXJzIl0KCiAgICAgICAgZm9yIHFpZCwgcWRlZiBpbiBxdWVzdGlvbnMuaXRlbXMoKToKICAgICAgICAgICAgaWYgcWlkIG5vdCBpbiBnb2xkOgogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgZyA9IGdvbGRbcWlkXQogICAgICAgICAgICBxdHlwZSA9IHFkZWZbInR5cGUiXQogICAgICAgICAgICBwX2FucyA9IGFuc3dlcnNbcWlkXQoKICAgICAgICAgICAgaWYgcXR5cGUgPT0gImNob2ljZSI6CiAgICAgICAgICAgICAgICBrZXlzID0gbGlzdChxZGVmWyJjcml0ZXJpYSJdLmtleXMoKSkKICAgICAgICAgICAgICAgIHByZWQgPSBwX2Fuc1siY2hvaWNlIl0KICAgICAgICAgICAgICAgIGdvbGRfbGFiZWwgPSBzdHIoZ1sibGFiZWwiXSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwcmVkID09IGdvbGRfbGFiZWwpCgogICAgICAgICAgICAgICAgcF9wcm9icyA9IF9ub3JtYWxpc2UoW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KGssIDFlLTYpIGZvciBrIGluIGtleXNdKQogICAgICAgICAgICAgICAgZ19wcm9icyA9IF9ub3JtYWxpc2UoW2dbInByb2JhYmlsaXRpZXMiXS5nZXQoaywgMWUtNikgZm9yIGsgaW4ga2V5c10pCgogICAgICAgICAgICAgICAgY29ycmVjdHMuYXBwZW5kKGlzX2NvcnIpCiAgICAgICAgICAgICAgICBjb25maWRlbmNlcy5hcHBlbmQobWF4KHBfcHJvYnMpKQogICAgICAgICAgICAgICAgYnJpZXJzLmFwcGVuZChicmllcl9zY29yZShwX3Byb2JzLCBnX3Byb2JzKSkKICAgICAgICAgICAgICAgIHNvZnRzLmFwcGVuZChzb2Z0X2FjY3VyYWN5KHBfcHJvYnMsIGdfcHJvYnMpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbImNob2ljZSJdLmFwcGVuZChpc19jb3JyKQoKICAgICAgICAgICAgZWxpZiBxdHlwZSA9PSAibm91bCI6CiAgICAgICAgICAgICAgICBwX3ZhbCA9IGZsb2F0KHBfYW5zWyJub3VsIl0pCiAgICAgICAgICAgICAgICBnX3ZhbCA9IGZsb2F0KGcuZ2V0KCJub3VsIiwgZy5nZXQoInByb2JhYmlsaXRpZXMiLCB7fSkuZ2V0KCJ0cnVlIiwgMC41KSkpCiAgICAgICAgICAgICAgICBnb2xkX2xhYmVsID0gc3RyKGdbImxhYmVsIl0pLmxvd2VyKCkKICAgICAgICAgICAgICAgIHByZWRfbGFiZWwgPSAidHJ1ZSIgaWYgcF92YWwgPj0gMC41IGVsc2UgImZhbHNlIgogICAgICAgICAgICAgICAgaXNfY29yciA9IGZsb2F0KHByZWRfbGFiZWwgPT0gZ29sZF9sYWJlbCkKCiAgICAgICAgICAgICAgICBwX2Rpc3QgPSBbMS4wIC0gcF92YWwsIHBfdmFsXQogICAgICAgICAgICAgICAgZ19kaXN0ID0gWzEuMCAtIGdfdmFsLCBnX3ZhbF0KCiAgICAgICAgICAgICAgICBjb3JyZWN0cy5hcHBlbmQoaXNfY29ycikKICAgICAgICAgICAgICAgIGNvbmZpZGVuY2VzLmFwcGVuZChtYXgocF92YWwsIDEuMCAtIHBfdmFsKSkKICAgICAgICAgICAgICAgIGJyaWVycy5hcHBlbmQoYnJpZXJfc2NvcmUocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgc29mdHMuYXBwZW5kKHNvZnRfYWNjdXJhY3kocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbIm5vdWwiXS5hcHBlbmQoaXNfY29ycikKCiAgICAgICAgICAgIGVsaWYgcXR5cGUgPT0gInNjb3JlIjoKICAgICAgICAgICAgICAgIHBfc2NvcmUgPSBmbG9hdChwX2Fuc1sic2NvcmUiXSkKICAgICAgICAgICAgICAgIGdfc2NvcmUgPSBmbG9hdChnLmdldCgic2NvcmUiLCAwLjApKQogICAgICAgICAgICAgICAgc2NvcmVfbWFlcy5hcHBlbmQoYWJzKHBfc2NvcmUgLSBnX3Njb3JlKSkKICAgICAgICAgICAgICAgIHdpdGhpbl9vbmUuYXBwZW5kKGZsb2F0KGFicyhwX3Njb3JlIC0gZ19zY29yZSkgPD0gMS4wKSkKCiAgICAgICAgICAgICAgICBuX2xldmVscyA9IGxlbihxZGVmLmdldCgiY3JpdGVyaWEiLCBbXSkpCiAgICAgICAgICAgICAgICBwX3Byb2JzID0gW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KHN0cihpKSwgMC4wKSBmb3IgaSBpbiByYW5nZShuX2xldmVscyldCiAgICAgICAgICAgICAgICBpZiBzdW0ocF9wcm9icykgPiAwOgogICAgICAgICAgICAgICAgICAgIHBfcHJvYnMgPSBfbm9ybWFsaXNlKHBfcHJvYnMpCiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQobWF4KHJhbmdlKGxlbihwX3Byb2JzKSksIGtleT1sYW1iZGEgajogcF9wcm9ic1tqXSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKG1heChwX3Byb2JzKSkKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQocm91bmQocF9zY29yZSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKDAuNSkKCiAgICAgICAgICAgICAgICBnX2x2bCA9IGludChnLmdldCgibGFiZWwiLCBpbnQocm91bmQoZ19zY29yZSkpKSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwX2x2bCA9PSBnX2x2bCkKICAgICAgICAgICAgICAgIGNvcnJlY3RzLmFwcGVuZChpc19jb3JyKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbInNjb3JlIl0uYXBwZW5kKGlzX2NvcnIpCgogICAgICAgIGlmIChpICsgMSkgJSAxMDAgPT0gMDoKICAgICAgICAgICAgcHJpbnQoZiIgIHtpICsgMX0ve2xlbih0ZXN0X21ldGEpfSBjYXNlcyIsIGZsdXNoPVRydWUpCgogICAgcmVzdWx0ID0gewogICAgICAgICJjaGVja3BvaW50IjogY2hlY2twb2ludF9kaXIsCiAgICAgICAgInRlbXBlcmF0dXJlcyI6IHRlbXBlcmF0dXJlcywKICAgICAgICAibl9jYXNlcyI6IGxlbih0ZXN0X21ldGEpLAogICAgICAgICJuX2RlY2lzaW9ucyI6IGxlbihjb3JyZWN0cyksCiAgICAgICAgImFjY3VyYWN5Ijogcm91bmQoYWNjdXJhY3koY29ycmVjdHMpLCA0KSwKICAgICAgICAiZWNlIjogcm91bmQoZXhwZWN0ZWRfY2FsaWJyYXRpb25fZXJyb3IoY29uZmlkZW5jZXMsIGNvcnJlY3RzKSwgNCksCiAgICAgICAgImJyaWVyIjogcm91bmQoZmxvYXQoc3VtKGJyaWVycykgLyBsZW4oYnJpZXJzKSkgaWYgYnJpZXJzIGVsc2UgZmxvYXQoIm5hbiIpLCA0KSwKICAgICAgICAic29mdF9hY2N1cmFjeSI6IHJvdW5kKGZsb2F0KHN1bShzb2Z0cykgLyBsZW4oc29mdHMpKSBpZiBzb2Z0cyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgInNjb3JlX21hZSI6IHJvdW5kKGZsb2F0KHN1bShzY29yZV9tYWVzKSAvIGxlbihzY29yZV9tYWVzKSkgaWYgc2NvcmVfbWFlcyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgIndpdGhpbl9vbmVfbGV2ZWwiOiByb3VuZChmbG9hdChzdW0od2l0aGluX29uZSkgLyBsZW4od2l0aGluX29uZSkpIGlmIHdpdGhpbl9vbmUgZWxzZSBmbG9hdCgibmFuIiksIDQpLAogICAgICAgICJwZXJfdHlwZV9hY2N1cmFjeSI6IHtrOiByb3VuZChhY2N1cmFjeSh2KSwgNCkgZm9yIGssIHYgaW4gcGVyX3R5cGUuaXRlbXMoKSBpZiB2fSwKICAgIH0KICAgICMgYGJyaWVyYC9gc29mdF9hY2N1cmFjeWAvYGVjZWAgbWl4IHF1ZXN0aW9uIHR5cGVzOyByZWNvcmQgdGhlIGNvdW50cyB0aGV5IHJlc3Qgb24uCiAgICByZXN1bHRbIm5fYnJpZXJfZGVjaXNpb25zIl0gPSBsZW4oYnJpZXJzKQogICAgcmV0dXJuIHJlc3VsdAoKCmRlZiBtYWluKCkgLT4gaW50OgogICAgcGFyc2VyID0gYXJncGFyc2UuQXJndW1lbnRQYXJzZXIoZGVzY3JpcHRpb249X19kb2NfXykKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tY2hlY2twb2ludCIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLXRlc3QtbWV0YSIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRldmljZSIsIGRlZmF1bHQ9ImF1dG8iKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS10ZW1wZXJhdHVyZXMiLCBjaG9pY2VzPVsic2hpcHBlZCIsICJyYXciXSwgZGVmYXVsdD0ic2hpcHBlZCIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dCIsIGRlZmF1bHQ9Tm9uZSwKICAgICAgICAgICAgICAgICAgICAgICAgaGVscD0iT3V0cHV0IEpTT04gKGRlZmF1bHQ6IDxjaGVja3BvaW50Pi9ldmFsXzx0ZW1wZXJhdHVyZXM+Lmpzb24pIikKICAgIGFyZ3MgPSBwYXJzZXIucGFyc2VfYXJncygpCgogICAgd2l0aCBvcGVuKGFyZ3MudGVzdF9tZXRhLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIHRlc3RfbWV0YSA9IGpzb24ubG9hZChmKQogICAgcHJpbnQoZiJsb2FkZWQge2xlbih0ZXN0X21ldGEpfSB0ZXN0IGNhc2VzIiwgZmx1c2g9VHJ1ZSkKCiAgICByZXN1bHQgPSBldmFsdWF0ZV9jaGVja3BvaW50KGFyZ3MuY2hlY2twb2ludCwgdGVzdF9tZXRhLCBkZXZpY2U9YXJncy5kZXZpY2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRlbXBlcmF0dXJlcz1hcmdzLnRlbXBlcmF0dXJlcykKCiAgICBwcmludCgiXG4iICsgIj0iICogNjApCiAgICBwcmludChmImV2YWx1YXRpb246IHthcmdzLmNoZWNrcG9pbnR9ICh7YXJncy50ZW1wZXJhdHVyZXN9IHRlbXBlcmF0dXJlcykiKQogICAgcHJpbnQoIj0iICogNjApCiAgICBwcmludChmIiAgYWNjdXJhY3k6ICAgICB7cmVzdWx0WydhY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIEVDRTogICAgICAgICAge3Jlc3VsdFsnZWNlJ106LjRmfSIpCiAgICBwcmludChmIiAgQnJpZXI6ICAgICAgICB7cmVzdWx0WydicmllciddOi40Zn0iKQogICAgcHJpbnQoZiIgIHNvZnQgYWNjOiAgICAge3Jlc3VsdFsnc29mdF9hY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIHBlciB0eXBlOiAgICAge3Jlc3VsdFsncGVyX3R5cGVfYWNjdXJhY3knXX0iKQogICAgcHJpbnQoIj0iICogNjApCgogICAgb3V0X3BhdGggPSBhcmdzLm91dCBvciBvcy5wYXRoLmpvaW4oYXJncy5jaGVja3BvaW50LCBmImV2YWxfe2FyZ3MudGVtcGVyYXR1cmVzfS5qc29uIikKICAgIG9zLm1ha2VkaXJzKG9zLnBhdGguZGlybmFtZShvdXRfcGF0aCkgb3IgIi4iLCBleGlzdF9vaz1UcnVlKQogICAgd2l0aCBvcGVuKG91dF9wYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAganNvbi5kdW1wKHJlc3VsdCwgZiwgaW5kZW50PTIpCiAgICBwcmludChmIndyb3RlIHtvdXRfcGF0aH0iKQogICAgcmV0dXJuIDAKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgc3lzLmV4aXQobWFpbigpKQo=",
+ "metrics.py": "IiIiTWV0cmljIGRlZmluaXRpb25zIGZvciB0aGUgU3RlcC00IGJlbmNobWFyaywgbWlycm9yaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCgpSZWZlcmVuY2U6IGNlbGwgMTQgb2YgYG5vdGVib29rcy9sYXlhX2ZpbmV0dW5lX3R5cGVkX2RlY2lzaW9uc18yeFQ0X2thZ2dsZS5pcHluYmAuIFRoZSBwdWJsaXNoZWQKMC43NjYgY2hlY2twb2ludCBhbmQgdGhlIG1haW50YWluZXIncyByZWZlcmVuY2UgdGFibGUgYXJlIHNjb3JlZCB3aXRoIHRoZXNlIGRlZmluaXRpb25zLCBzbyBhCnJlcHJvZHVjdGlvbiBoYXMgdG8gdXNlIHRoZW0gdG9vLCBub3QgYSBkaWZmZXJlbnRseS1kZWZpbmVkIGxpYnJhcnkgbWV0cmljOgoKLSAqKmFjY3VyYWN5Kiog4oCUIHRvcCByZXBvcnRlZCBvcHRpb24gYWdhaW5zdCB0aGUgZ29sZCBsYWJlbCwgb3ZlciBldmVyeSBkZWNpc2lvbiAoY2hvaWNlLCBzY29yZQogIGFuZCBub3VsKS4KLSAqKkVDRSoqIOKAlCBgbGF5YS5jb21tb24uZWNlX3Njb3JlYCBvZiB0aGUgdG9wIHByb2JhYmlsaXR5IGFnYWluc3QgY29ycmVjdG5lc3MsIDE1IGVxdWFsLXdpZHRoCiAgYmlucywgb3ZlciBldmVyeSBkZWNpc2lvbi4KLSAqKmJyaWVyKiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIGRlY2lzaW9ucyBvZiBgc3VtKChwIC0gZ29sZCkqKjIpYC4gVGhlIG5vdGVib29rIGRvZXMKICBub3QgYXBwZW5kIHNjb3JlIGRlY2lzaW9ucyB0byB0aGlzIG1lYW4sIHNvIGEgc2NvcmUgY2hhbmdlIG5ldmVyIG1vdmVzIEJyaWVyOyBrZXB0IGFzLWlzIHNvIHRoZQogIG51bWJlciBjb21wYXJlcyB3aXRoIHRoZSByZWZlcmVuY2UuCi0gKipzb2Z0X2FjY3VyYWN5Kiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIG9mIGBzdW0ocCAqIGdvbGQpYC4KCmBnb2xkYCBpcyB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gKGBnb2xkLnByb2JhYmlsaXRpZXNgKSwgbm90IGEgb25lLWhvdCBsYWJlbCwgc28gQnJpZXIgYW5kIHNvZnQKYWNjdXJhY3kgbWVhc3VyZSB0aGUgZGlzdHJpYnV0aW9uIHRoZSBydW4gd2FzIHRyYWluZWQgdG8gbWF0Y2guCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgpmcm9tIHR5cGluZyBpbXBvcnQgU2VxdWVuY2UKCmltcG9ydCBudW1weSBhcyBucAoKZnJvbSBsYXlhLmNvbW1vbiBpbXBvcnQgZWNlX3Njb3JlCgoKZGVmIGFjY3VyYWN5KGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICByZXR1cm4gZmxvYXQobnAubWVhbihjb3JyZWN0KSkgaWYgbGVuKGNvcnJlY3QpIGVsc2UgZmxvYXQoIm5hbiIpCgoKZGVmIGV4cGVjdGVkX2NhbGlicmF0aW9uX2Vycm9yKGNvbmZpZGVuY2VzOiBTZXF1ZW5jZVtmbG9hdF0sIGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJpbnM6IGludCA9IDE1KSAtPiBmbG9hdDoKICAgICIiIlRoaW4gd3JhcHBlciBvdmVyIHRoZSByZXBvc2l0b3J5J3MgYGNvbW1vbi5lY2Vfc2NvcmVgLCBhcyB0aGUgbm90ZWJvb2sgY2FsbHMgaXQuIiIiCiAgICBpZiBub3QgbGVuKGNvbmZpZGVuY2VzKToKICAgICAgICByZXR1cm4gZmxvYXQoIm5hbiIpCiAgICByZXR1cm4gZmxvYXQoZWNlX3Njb3JlKG5wLmFzYXJyYXkoY29uZmlkZW5jZXMsIGR0eXBlPWZsb2F0KSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbnAuYXNhcnJheShjb3JyZWN0LCBkdHlwZT1mbG9hdCksIGJpbnM9YmlucykpCgoKZGVmIGJyaWVyX3Njb3JlKHByb2JzOiBTZXF1ZW5jZVtmbG9hdF0sIGdvbGQ6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICBwID0gbnAuYXNhcnJheShwcm9icywgZHR5cGU9ZmxvYXQpCiAgICBnID0gbnAuYXNhcnJheShnb2xkLCBkdHlwZT1mbG9hdCkKICAgIHJldHVybiBmbG9hdCgoKHAgLSBnKSAqKiAyKS5zdW0oKSkKCgpkZWYgc29mdF9hY2N1cmFjeShwcm9iczogU2VxdWVuY2VbZmxvYXRdLCBnb2xkOiBTZXF1ZW5jZVtmbG9hdF0pIC0+IGZsb2F0OgogICAgcCA9IG5wLmFzYXJyYXkocHJvYnMsIGR0eXBlPWZsb2F0KQogICAgZyA9IG5wLmFzYXJyYXkoZ29sZCwgZHR5cGU9ZmxvYXQpCiAgICByZXR1cm4gZmxvYXQoKHAgKiBnKS5zdW0oKSkK"
+}
+for _n, _b in _EMBEDDED.items():
+ (HARNESS / _n).write_bytes(base64.b64decode(_b))
+sys.path.insert(0, str(HARNESS))
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+import torch # noqa: E402
+
+n_gpu = torch.cuda.device_count()
+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu
+print("GPU", torch.cuda.get_device_name(0), flush=True)
+
+RESULTS_PATH = WORK / "step4_gate_d_results.json"
+
+
+def flush_results():
+ RESULTS_PATH.write_text(json.dumps(RESULTS, indent=2), encoding="utf-8")
+
+
+def run(cmd):
+ print("\n$ " + " ".join(cmd), flush=True)
+ p = subprocess.run(cmd, check=True, text=True, capture_output=True)
+ sys.stdout.write(p.stdout)
+ if p.stderr:
+ sys.stderr.write(p.stderr)
+ sys.stdout.flush()
+ return p.stdout
+
+
+run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
+ try:
+ __import__(_mod)
+ except Exception:
+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])
+
+from huggingface_hub import snapshot_download # noqa: E402
+
+base_dir = snapshot_download("convaiinnovations/laya")
+base_cfg = json.loads((Path(base_dir) / "rl_agent_config.json").read_text(encoding="utf-8"))
+
+run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
+test_meta = json.loads((WORK / "data" / "test_meta.json").read_text(encoding="utf-8"))
+
+from evaluate import evaluate_checkpoint # noqa: E402
+
+RESULTS = {
+ "meta": {
+ "pinned_sha": PINNED_SHA,
+ "torch": torch.__version__,
+ "gpu": torch.cuda.get_device_name(0),
+ "visible_gpus": n_gpu,
+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
+ "n_train_cases": 1200,
+ "n_test_cases": len(test_meta),
+ "reused": {"rlcd_seed0": "Gate R kernel (separate session, same SHA/data/command)"},
+ },
+ "rlcd_seed0": {
+ "checkpoint": "Gate R kernel run_rlcd_seed0 (reused)",
+ "temperatures": "shipped", "n_cases": 400, "n_decisions": 2000,
+ "accuracy": 0.775, "ece": 0.1504, "brier": 0.0519, "soft_accuracy": 0.5654,
+ "per_type_accuracy": {"choice": 0.7417, "noul": 0.8517, "score": 0.7425},
+ "loss": "rlcd", "seed": 0, "wall_clock_s": 2691.3,
+ },
+}
+
+# --- base sanity row, must match the Gate-R reference -----------------------
+base_row = evaluate_checkpoint("convaiinnovations/laya", test_meta, device="cuda",
+ temperatures="shipped")
+RESULTS["base"] = base_row
+base_ok = (abs(base_row["accuracy"] - 0.362) <= 0.01 and abs(base_row["ece"] - 0.1741) <= 0.01
+ and abs(base_row["brier"] - 0.3287) <= 0.01)
+print("BASE_SANITY", "OK" if base_ok else "FAIL", json.dumps(base_row), flush=True)
+flush_results()
+if not base_ok:
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: base sanity drift")
+
+# --- the five remaining cells ----------------------------------------------
+CELLS = [("rlcd", 1), ("soft-ce", 0), ("soft-ce", 1), ("rlcd", 2), ("soft-ce", 2)]
+
+
+def train(loss, seed):
+ out = WORK / ("run_" + loss.replace("-", "_") + "_seed%d" % seed)
+ cmd = [sys.executable, "-m", "laya.train_cli",
+ "--data", str(WORK / "data" / "train.jsonl"),
+ "--out", str(out),
+ "--base", "convaiinnovations/laya",
+ "--loss", loss, "--seed", str(seed),
+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
+ "--max-len", "1024", "--head-max-len", "256",
+ "--device", "cuda"]
+ t0 = time.time()
+ stdout = run(cmd)
+ dt = round(time.time() - t0, 1)
+ cfg = json.loads((out / "rl_agent_config.json").read_text(encoding="utf-8"))
+ lt = (cfg.get("training") or {}).get("laya_train", {})
+ split_line = next((ln for ln in stdout.splitlines() if ln.startswith("train items")), "")
+ checks = {
+ "loss": lt.get("loss") == loss,
+ "seed": lt.get("seed") == seed,
+ "epochs": lt.get("epochs") == 4,
+ "micro_batch": lt.get("micro_batch") == 8,
+ "grad_accum": lt.get("grad_accum") == 8,
+ "encoder_lr": abs(float(lt.get("encoder_lr", 0)) - 2.5e-5) < 1e-12,
+ "head_lr": abs(float(lt.get("head_lr", 0)) - 1e-4) < 1e-12,
+ "max_len": lt.get("max_len") == 1024,
+ "head_max_len": lt.get("head_max_len") == 256,
+ "calib_seed": lt.get("calib_seed") == 20260922,
+ "no_shuffle": list(lt.get("shuffle_options") or []) == [],
+ "counts": "train items 5600, calibration items 400, skipped {}" in split_line,
+ }
+ return out, dt, cfg, checks, split_line
+
+
+for loss, seed in CELLS:
+ out, dt, cfg, checks, split_line = train(loss, seed)
+ print("CELL %s seed%d checks %s" % (loss, seed, json.dumps(checks)), flush=True)
+ bad = sorted(k for k, v in checks.items() if not v)
+ if bad:
+ RESULTS["%s_seed%d" % (loss, seed)] = {"error": "protocol mismatch: " + ",".join(bad),
+ "split_line": split_line}
+ flush_results()
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: %s seed%d -> %s" % (loss, seed, bad))
+
+ res = evaluate_checkpoint(str(out), test_meta, device="cuda", temperatures="shipped")
+ finite = all(isinstance(res.get(k), (int, float)) and math.isfinite(res[k])
+ for k in ("accuracy", "ece", "brier"))
+ res.update({
+ "loss": loss, "seed": seed, "wall_clock_s": dt,
+ "temperature": cfg.get("temperature"),
+ "calibration": (cfg.get("training") or {}).get("laya_train_calibration"),
+ "split_line": split_line, "finite": finite,
+ })
+ RESULTS["%s_seed%d" % (loss, seed)] = res
+ flush_results()
+ print("RESULT %s seed%d %s" % (loss, seed, json.dumps(res)), flush=True)
+ if not finite:
+ raise SystemExit("GATE_D_PROTOCOL_PROBLEM: non-finite metric %s seed%d" % (loss, seed))
+
+print("ALL_CELLS_DONE", flush=True)
+print("RESULTS " + json.dumps(RESULTS, indent=2), flush=True)