Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

research(step4): single-T4 self-contained Kaggle Gate R package

Bruce-Yii710f806fc2f1
diff --git a/research/scripts/step4_benchmark/kaggle/build_kernel.py b/research/scripts/step4_benchmark/kaggle/build_kernel.py
new file mode 100644
index 00000000..58d74491
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/build_kernel.py
@@ -0,0 +1,187 @@
+"""Generate the self-contained Kaggle Gate R runner for #887 Step 4.
+
+The harness under this directory is not in the upstream repo yet (public mutation authority is
+NONE), so the kernel cannot clone it. This generator embeds the four harness files verbatim (as
+base64) into one script, plus a `kernel-metadata.json` pinned to a single T4.
+
+Single T4 on Kaggle means `machine_shape: "NvidiaTeslaT4"`; the kernel then pins
+`CUDA_VISIBLE_DEVICES=0` and asserts exactly one visible GPU, so the run is single-device with no
+torchrun and no 2-GPU assert — the shipped `laya-train` path.
+
+Run the API with `KAGGLE_API_TOKEN` exported from `kaggle auth print-access-token`; the stored
+credential is not picked up automatically by this CLI build.
+
+ python research/scripts/step4_benchmark/kaggle/build_kernel.py
+ KAGGLE_API_TOKEN=$(kaggle auth print-access-token) kaggle kernels push -p <dir>
+"""
+from __future__ import annotations
+
+import base64
+import json
+import os
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+HARNESS = HERE.parent
+EMBED = ("prepare_data.py", "run_benchmark.py", "evaluate.py", "metrics.py")
+PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
+KERNEL_ID = "yuyi722333/laya-step4-gate-r"
+
+TEMPLATE = '''"""#887 Step 4 Gate R, generated by build_kernel.py -- do not edit here.
+
+Single-device T4 reproduction of the typed-decisions fine-tune through the shipped `laya-train`,
+then the Step-4 evaluator. No torchrun, no 2-GPU assert, no early stopping, no option shuffling.
+"""
+import base64
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+PINNED_SHA = "__PINNED_SHA__"
+WORK = Path("/kaggle/working")
+HARNESS = WORK / "step4_benchmark"
+HARNESS.mkdir(parents=True, exist_ok=True)
+
+_EMBEDDED = __EMBEDDED__
+for _name, _b64 in _EMBEDDED.items():
+ (HARNESS / _name).write_bytes(base64.b64decode(_b64))
+sys.path.insert(0, str(HARNESS))
+
+
+def run(cmd):
+ print("\\n$ " + " ".join(cmd), flush=True)
+ return subprocess.run(cmd, check=True)
+
+
+# --- one visible T4 -------------------------------------------------------------------------
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+import torch # noqa: E402
+
+n_gpu = torch.cuda.device_count()
+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
+for _i in range(n_gpu):
+ _p = torch.cuda.get_device_properties(_i)
+ print(" GPU", _i, _p.name, round(_p.total_memory / 1e9, 1), "GB", flush=True)
+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu
+
+# --- laya at the pinned SHA, without disturbing Kaggle's torch ------------------------------
+run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
+ try:
+ __import__(_mod)
+ except Exception:
+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])
+
+# --- base checkpoint config readback --------------------------------------------------------
+from huggingface_hub import snapshot_download # noqa: E402
+
+base_dir = snapshot_download("convaiinnovations/laya")
+with open(os.path.join(base_dir, "rl_agent_config.json"), encoding="utf-8") as _f:
+ base_cfg = json.load(_f)
+base_keys = ("max_len", "head_max_len", "max_tokens_per_batch", "temperature",
+ "temperature_by_options", "option_layout")
+print("BASE_CONFIG " + json.dumps({k: base_cfg.get(k) for k in base_keys}), flush=True)
+
+# --- data -----------------------------------------------------------------------------------
+run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
+with open(WORK / "data" / "train.jsonl", encoding="utf-8") as _f:
+ n_train_cases = sum(1 for _l in _f if _l.strip())
+with open(WORK / "data" / "test_meta.json", encoding="utf-8") as _f:
+ test_meta = json.load(_f)
+print("SPLITS train_cases=%d test_cases=%d" % (n_train_cases, len(test_meta)), flush=True)
+
+# --- Gate R: rlcd, seed 0, notebook hyperparameters ----------------------------------------
+cmd = [sys.executable, "-m", "laya.train_cli",
+ "--data", str(WORK / "data" / "train.jsonl"),
+ "--out", str(WORK / "run_rlcd_seed0"),
+ "--base", "convaiinnovations/laya",
+ "--loss", "rlcd", "--seed", "0",
+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
+ "--max-len", "1024", "--head-max-len", "256",
+ "--device", "cuda"]
+_t0 = time.time()
+_proc = subprocess.run(cmd, check=True, capture_output=True, text=True)
+train_seconds = round(time.time() - _t0, 1)
+print(_proc.stdout)
+if _proc.stderr:
+ print(_proc.stderr, file=sys.stderr)
+(WORK / "step4_train.log").write_text(_proc.stdout + "\\n" + _proc.stderr, encoding="utf-8")
+
+train_split_line = next((ln for ln in _proc.stdout.splitlines() if ln.startswith("train items")), "")
+peak_gpu_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
+print("TRAIN_SECONDS", train_seconds, "PEAK_GPU_GB", peak_gpu_gb, flush=True)
+print("TRAIN_SPLIT", train_split_line, flush=True)
+
+# --- evaluate with the Step-4 evaluator -----------------------------------------------------
+from evaluate import evaluate_checkpoint # noqa: E402
+
+results = {
+ "meta": {
+ "pinned_sha": PINNED_SHA,
+ "torch": torch.__version__,
+ "gpu": torch.cuda.get_device_name(0),
+ "visible_gpus": n_gpu,
+ "train_seconds": train_seconds,
+ "peak_gpu_gb": peak_gpu_gb,
+ "train_split_line": train_split_line,
+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
+ "n_train_cases": n_train_cases,
+ "n_test_cases": len(test_meta),
+ }
+}
+for _label, _ckpt in (("rlcd_seed0", str(WORK / "run_rlcd_seed0")),
+ ("base", "convaiinnovations/laya"),
+ ("published", "convaiinnovations/laya-typed-decisions")):
+ try:
+ results[_label] = evaluate_checkpoint(_ckpt, test_meta, device="cuda", temperatures="shipped")
+ except Exception as _e: # noqa: BLE001 - a missing optional row must not kill the run
+ results[_label] = {"error": "%s: %s" % (type(_e).__name__, _e)}
+ print(_label, json.dumps(results[_label]), flush=True)
+
+with open(WORK / "step4_gate_r_results.json", "w", encoding="utf-8") as _f:
+ json.dump(results, _f, indent=2)
+print("\\nRESULTS " + json.dumps(results, indent=2), flush=True)
+'''
+
+
+def main() -> int:
+ embedded = {}
+ for name in EMBED:
+ embedded[name] = base64.b64encode((HARNESS / name).read_bytes()).decode("ascii")
+
+ script = (TEMPLATE
+ .replace("__PINNED_SHA__", PINNED_SHA)
+ .replace("__EMBEDDED__", json.dumps(embedded, indent=1)))
+ (HERE / "step4_gate_r.py").write_text(script, encoding="utf-8", newline="\n")
+
+ metadata = {
+ "id": KERNEL_ID,
+ "title": "Laya Step 4 Gate R",
+ "code_file": "step4_gate_r.py",
+ "language": "python",
+ "kernel_type": "script",
+ "is_private": True,
+ "enable_gpu": True,
+ "enable_tpu": False,
+ "enable_internet": True,
+ "machine_shape": "NvidiaTeslaT4",
+ "dataset_sources": [],
+ "competition_sources": [],
+ "kernel_sources": [],
+ "model_sources": [],
+ }
+ (HERE / "kernel-metadata.json").write_text(json.dumps(metadata, indent=2) + "\n",
+ encoding="utf-8", newline="\n")
+ print("wrote", HERE / "step4_gate_r.py", "and", HERE / "kernel-metadata.json")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/research/scripts/step4_benchmark/kaggle/kernel-metadata.json b/research/scripts/step4_benchmark/kaggle/kernel-metadata.json
new file mode 100644
index 00000000..7e3e95e2
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/kernel-metadata.json
@@ -0,0 +1,16 @@
+{
+ "id": "yuyi722333/laya-step4-gate-r",
+ "title": "Laya Step 4 Gate R",
+ "code_file": "step4_gate_r.py",
+ "language": "python",
+ "kernel_type": "script",
+ "is_private": true,
+ "enable_gpu": true,
+ "enable_tpu": false,
+ "enable_internet": true,
+ "machine_shape": "NvidiaTeslaT4",
+ "dataset_sources": [],
+ "competition_sources": [],
+ "kernel_sources": [],
+ "model_sources": []
+}
diff --git a/research/scripts/step4_benchmark/kaggle/step4_gate_r.py b/research/scripts/step4_benchmark/kaggle/step4_gate_r.py
new file mode 100644
index 00000000..4e29c5ca
--- /dev/null
+++ b/research/scripts/step4_benchmark/kaggle/step4_gate_r.py
@@ -0,0 +1,126 @@
+"""#887 Step 4 Gate R, generated by build_kernel.py -- do not edit here.
+
+Single-device T4 reproduction of the typed-decisions fine-tune through the shipped `laya-train`,
+then the Step-4 evaluator. No torchrun, no 2-GPU assert, no early stopping, no option shuffling.
+"""
+import base64
+import json
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
+WORK = Path("/kaggle/working")
+HARNESS = WORK / "step4_benchmark"
+HARNESS.mkdir(parents=True, exist_ok=True)
+
+_EMBEDDED = {
+ "prepare_data.py": "IiIiUHJlcGFyZSBMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyBmb3IgYGxheWEtdHJhaW5gIGFuZCBmb3IgZXZhbHVhdGlvbi4KClRoZSBiZW5jaG1hcmsncyBvd24gcm93cyBhcmUgYHtzdGF0ZSwgcXVlc3Rpb25zLCBnb2xkfWAgd2l0aCB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gcGVyCnF1ZXN0aW9uLiBUaGF0IGlzIGV4YWN0bHkgdGhlIHNjaGVtYSBgbGF5YS50cmFpbmAgcmVhZHMsIHNvIHRoZSBjb252ZXJzaW9uIGlzIGEgcGFyc2UsIG5vdCBhCnJlc2hhcGU6IHRoZSB0d28gSlNPTi1zdHJpbmcgY29sdW1ucyBhcmUgZGVjb2RlZCBhbmQgdGhlIHJvdyBpcyB3cml0dGVuIGJhY2sgb3V0IGFzIEpTT05MLgoKU3BsaXRzOgoKKiBgYHRyYWluLmpzb25sYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0cmFpbmBgIHNwbGl0ICgxLDIwMCBjYXNlcyAvIDYsMDAwIGRlY2lzaW9ucyksIHRoZSBpbnB1dCB0bwogIGBgbGF5YS10cmFpbmBgLiBUaGUgY2FsaWJyYXRpb24gc2xpY2UgaXMgaGVsZCBvdXQgKmluc2lkZSogYGBsYXlhLXRyYWluYGAgYXQgYSBmaXhlZCBzZWVkCiAgKGBgY2FsaWJfZnJhY2BgIDAuMSwgY2FwcGVkIGF0IDQwMCwgc2VlZCAyMDI2MDkyMiksIG1hdGNoaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCiogYGB0ZXN0Lmpzb25sYGAgLyBgYHRlc3RfbWV0YS5qc29uYGAg4oCUIHRoZSBkYXRhc2V0J3MgYGB0ZXN0YGAgc3BsaXQgKDQwMCBjYXNlcyAvIDIsMDAwCiAgZGVjaXNpb25zKSwgdXNlZCBieSBgYGV2YWx1YXRlLnB5YGAuIGBgdGVzdF9tZXRhLmpzb25gYCBrZWVwcyBgYGlkYGAgYW5kIGBgd29ya2Zsb3dgYCBzbwogIHBlci13b3JrZmxvdyBhY2N1cmFjeSBjYW4gYmUgcmVwb3J0ZWQgYWxvbmdzaWRlIHRoZSBwb29sZWQgbnVtYmVyLgoKVXNhZ2U6CiAgICBweXRob24gcHJlcGFyZV9kYXRhLnB5IC0tb3V0LWRpciBkYXRhL3R5cGVkX2RlY2lzaW9ucwoiIiIKZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKaW1wb3J0IGFyZ3BhcnNlCmltcG9ydCBqc29uCmltcG9ydCBvcwppbXBvcnQgc3lzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QKCgpkZWYgX3BhcnNlKHZhbHVlOiBBbnkpIC0+IEFueToKICAgIGlmIGlzaW5zdGFuY2UodmFsdWUsIHN0cik6CiAgICAgICAgcmV0dXJuIGpzb24ubG9hZHModmFsdWUpCiAgICByZXR1cm4gdmFsdWUKCgpkZWYgY2FzZV90b19yZWNvcmQocm93OiBEaWN0W3N0ciwgQW55XSkgLT4gRGljdFtzdHIsIEFueV06CiAgICAiIiJPbmUgZGF0YXNldCByb3cgYXMgYSBgbGF5YS10cmFpbmAgcmVjb3JkIChge3N0YXRlLCBxdWVzdGlvbnMsIGdvbGR9YCkuIiIiCiAgICByZXR1cm4gewogICAgICAgICJzdGF0ZSI6IF9wYXJzZShyb3dbInN0YXRlIl0pLAogICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2Uocm93WyJxdWVzdGlvbnMiXSksCiAgICAgICAgImdvbGQiOiBfcGFyc2Uocm93WyJnb2xkIl0pLAogICAgfQoKCmRlZiBsb2FkX3NwbGl0KHNwbGl0OiBzdHIsIGRhdGFzZXQ6IHN0ciA9ICJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpIC0+IExpc3RbRGljdFtzdHIsIEFueV1dOgogICAgdHJ5OgogICAgICAgIGZyb20gZGF0YXNldHMgaW1wb3J0IGxvYWRfZGF0YXNldAogICAgZXhjZXB0IEltcG9ydEVycm9yOgogICAgICAgIHByaW50KCJFUlJPUjogdGhlIGRhdGFzZXRzIHBhY2thZ2UgaXMgcmVxdWlyZWQgKHBpcCBpbnN0YWxsIGRhdGFzZXRzKSIsIGZpbGU9c3lzLnN0ZGVycikKICAgICAgICBzeXMuZXhpdCgxKQogICAgZHMgPSBsb2FkX2RhdGFzZXQoZGF0YXNldCwgImFsbCIsIHNwbGl0PXNwbGl0KQogICAgcmV0dXJuIGxpc3QoZHMpCgoKZGVmIHdyaXRlX3NwbGl0KHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIsIHNwbGl0OiBzdHIpIC0+IGludDoKICAgIHJlY29yZHMgPSBbY2FzZV90b19yZWNvcmQocikgZm9yIHIgaW4gcm93c10KICAgIHBhdGggPSBvcy5wYXRoLmpvaW4ob3V0X2RpciwgZiJ7c3BsaXR9Lmpzb25sIikKICAgIHdpdGggb3BlbihwYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAgZm9yIHJlYyBpbiByZWNvcmRzOgogICAgICAgICAgICBmLndyaXRlKGpzb24uZHVtcHMocmVjLCBlbnN1cmVfYXNjaWk9RmFsc2UpICsgIlxuIikKICAgIHJldHVybiBsZW4ocmVjb3JkcykKCgpkZWYgd3JpdGVfdGVzdF9tZXRhKHJvd3M6IExpc3RbRGljdFtzdHIsIEFueV1dLCBvdXRfZGlyOiBzdHIpIC0+IGludDoKICAgIG1ldGEgPSBbXQogICAgZm9yIHIgaW4gcm93czoKICAgICAgICBtZXRhLmFwcGVuZCh7CiAgICAgICAgICAgICJpZCI6IHIuZ2V0KCJpZCIsICIiKSwKICAgICAgICAgICAgIndvcmtmbG93Ijogci5nZXQoIndvcmtmbG93IiwgIiIpLAogICAgICAgICAgICAic3RhdGUiOiBfcGFyc2Uoclsic3RhdGUiXSksCiAgICAgICAgICAgICJxdWVzdGlvbnMiOiBfcGFyc2UoclsicXVlc3Rpb25zIl0pLAogICAgICAgICAgICAiZ29sZCI6IF9wYXJzZShyWyJnb2xkIl0pLAogICAgICAgIH0pCiAgICBwYXRoID0gb3MucGF0aC5qb2luKG91dF9kaXIsICJ0ZXN0X21ldGEuanNvbiIpCiAgICB3aXRoIG9wZW4ocGF0aCwgInciLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIGpzb24uZHVtcChtZXRhLCBmLCBpbmRlbnQ9MikKICAgIHJldHVybiBsZW4obWV0YSkKCgpkZWYgbWFpbigpIC0+IGludDoKICAgIHBhcnNlciA9IGFyZ3BhcnNlLkFyZ3VtZW50UGFyc2VyKGRlc2NyaXB0aW9uPV9fZG9jX18pCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dC1kaXIiLCBkZWZhdWx0PSJkYXRhL3R5cGVkX2RlY2lzaW9ucyIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRhdGFzZXQiLCBkZWZhdWx0PSJMb2NhbExMYU1BL3R5cGVkLWRlY2lzaW9ucyIpCiAgICBhcmdzID0gcGFyc2VyLnBhcnNlX2FyZ3MoKQoKICAgIG9zLm1ha2VkaXJzKGFyZ3Mub3V0X2RpciwgZXhpc3Rfb2s9VHJ1ZSkKICAgIGZvciBzcGxpdCBpbiAoInRyYWluIiwgInRlc3QiKToKICAgICAgICByb3dzID0gbG9hZF9zcGxpdChzcGxpdCwgYXJncy5kYXRhc2V0KQogICAgICAgIG4gPSB3cml0ZV9zcGxpdChyb3dzLCBhcmdzLm91dF9kaXIsIHNwbGl0KQogICAgICAgIHByaW50KGYie3NwbGl0fToge259IGNhc2VzIC0+IHtvcy5wYXRoLmpvaW4oYXJncy5vdXRfZGlyLCBzcGxpdCArICcuanNvbmwnKX0iKQogICAgICAgIGlmIHNwbGl0ID09ICJ0ZXN0IjoKICAgICAgICAgICAgbSA9IHdyaXRlX3Rlc3RfbWV0YShyb3dzLCBhcmdzLm91dF9kaXIpCiAgICAgICAgICAgIHByaW50KGYidGVzdCBtZXRhOiB7bX0gY2FzZXMgLT4ge29zLnBhdGguam9pbihhcmdzLm91dF9kaXIsICd0ZXN0X21ldGEuanNvbicpfSIpCiAgICByZXR1cm4gMAoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBzeXMuZXhpdChtYWluKCkpCg==",
+ "run_benchmark.py": "IiIiUnVuIHRoZSBTdGVwLTQgYmVuY2htYXJrIHRocm91Z2ggdGhlIGBsYXlhLXRyYWluYCBDTEkuCgpUd28gZ2F0ZXMsIGtlcHQgc2VwYXJhdGUgb24gcHVycG9zZToKCiogKipHYXRlIFIg4oCUIHJlcHJvZHVjdGlvbi4qKiBTZWVkIDAsIHRoZSBub3RlYm9vaydzIGh5cGVycGFyYW1ldGVycywgYC0tbG9zcyBybGNkYC4gSXQgaGFzIHRvIGxhbmQKICBvbiB0aGUgcHVibGlzaGVkIGNoZWNrcG9pbnQgLyB0aGUgbWFpbnRhaW5lcidzIHRhcmdldCB3aXRoaW4gbm9pc2U7IHRoYXQgdmVyaWZpZXMgYWNjZXB0YW5jZQogIGNyaXRlcmlvbiAxLiBOb3RoaW5nIGVsc2UgaXMgY2xhaW1lZCBmcm9tIG9uZSBzZWVkLgoqICoqR2F0ZSBEIOKAlCBkZWZhdWx0LWxvc3MgZGVjaXNpb24uKiogYHJsY2RgIHZzIGBzb2Z0LWNlYCBvdmVyIG1vcmUgdGhhbiBvbmUgc2VlZC4gVGhlIGV4aXN0aW5nCiAgc2luZ2xlLXNlZWQgc29mdC1jZSB3aW4gaXMgbm90IGVub3VnaCB0byByZWNvbW1lbmQgY2hhbmdpbmcgYSBzaGlwcGVkIGRlZmF1bHQsIHNvIEdhdGUgRCBpcwogIG9ubHkgZGVzaWduZWQgYW5kIHJ1biBhZnRlciBHYXRlIFIgcGFzc2VzLgoKVGhpcyBjYWxscyB0aGUgQ0xJIGFzIGEgc3VicHJvY2VzczogaXQgZG9lcyBub3QgaW1wb3J0IGBsYXlhLnRyYWluYCwgc28gaXQgY2Fubm90IGRlcGVuZCBvbiB0aGUKdHJhaW5pbmcgaW50ZXJuYWxzICM5NjcvIzk2OC8jOTgyIGFyZSBzdGlsbCBjaGFuZ2luZy4KClVzYWdlOgogICAgIyBHYXRlIFI6IG9uZSBydW4KICAgIHB5dGhvbiBydW5fYmVuY2htYXJrLnB5IC0tZ2F0ZSBSIC0tZGF0YSBkYXRhL3R5cGVkX2RlY2lzaW9ucy90cmFpbi5qc29ubCBcCiAgICAgICAgLS1vdXQtZGlyIHJ1bnMvc3RlcDQgLS1kZXZpY2UgY3VkYQoKICAgICMgR2F0ZSBEOiBybGNkICsgc29mdC1jZSBvdmVyIHNlZWRzIDAvMS8yCiAgICBweXRob24gcnVuX2JlbmNobWFyay5weSAtLWdhdGUgRCAtLWRhdGEgZGF0YS90eXBlZF9kZWNpc2lvbnMvdHJhaW4uanNvbmwgXAogICAgICAgIC0tb3V0LWRpciBydW5zL3N0ZXA0IC0tZGV2aWNlIGN1ZGEKIiIiCmZyb20gX19mdXR1cmVfXyBpbXBvcnQgYW5ub3RhdGlvbnMKCmltcG9ydCBhcmdwYXJzZQppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHNobGV4CmltcG9ydCBzdWJwcm9jZXNzCmltcG9ydCBzeXMKaW1wb3J0IHRpbWUKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgVHVwbGUKCiMgVGhlIG5vdGVib29rJ3MgcmVjaXBlLCBzaW5nbGUtZGV2aWNlIGZvcm0uIDggbWljcm8tYmF0Y2ggeCAyIEdQVXMgeCA0IGFjY3VtdWxhdGlvbiA9IDY0CiMgZWZmZWN0aXZlOyBvbiBvbmUgZGV2aWNlIHRoYXQgaXMgOCB4IDguCk5PVEVCT09LX0hZUEVSUEFSQU1TID0gewogICAgImVwb2NocyI6IDQsCiAgICAibWljcm9fYmF0Y2giOiA4LAogICAgImdyYWRfYWNjdW0iOiA4LAogICAgImVuY29kZXJfbHIiOiAyLjVlLTUsCiAgICAiaGVhZF9sciI6IDFlLTQsCiAgICAibWF4X2xlbiI6IDEwMjQsCiAgICAiaGVhZF9tYXhfbGVuIjogMjU2LAp9CgpHQVRFUzogRGljdFtzdHIsIExpc3RbVHVwbGVbc3RyLCBpbnRdXV0gPSB7CiAgICAiUiI6IFsoInJsY2QiLCAwKV0sCiAgICAiRCI6IFsoInJsY2QiLCAwKSwgKCJybGNkIiwgMSksICgicmxjZCIsIDIpLAogICAgICAgICAgKCJzb2Z0LWNlIiwgMCksICgic29mdC1jZSIsIDEpLCAoInNvZnQtY2UiLCAyKV0sCn0KCgpkZWYgYnVpbGRfY29tbWFuZChkYXRhOiBzdHIsIG91dF9kaXI6IHN0ciwgbG9zczogc3RyLCBzZWVkOiBpbnQsIGJhc2U6IHN0ciwgZGV2aWNlOiBzdHIsCiAgICAgICAgICAgICAgICAgIGh5cGVyOiBEaWN0W3N0ciwgQW55XSkgLT4gTGlzdFtzdHJdOgogICAgcmV0dXJuIFsKICAgICAgICBzeXMuZXhlY3V0YWJsZSwgIi1tIiwgImxheWEudHJhaW5fY2xpIiwKICAgICAgICAiLS1kYXRhIiwgZGF0YSwKICAgICAgICAiLS1vdXQiLCBvdXRfZGlyLAogICAgICAgICItLWJhc2UiLCBiYXNlLAogICAgICAgICItLWxvc3MiLCBsb3NzLAogICAgICAgICItLXNlZWQiLCBzdHIoc2VlZCksCiAgICAgICAgIi0tZXBvY2hzIiwgc3RyKGh5cGVyWyJlcG9jaHMiXSksCiAgICAgICAgIi0tbWljcm8tYmF0Y2giLCBzdHIoaHlwZXJbIm1pY3JvX2JhdGNoIl0pLAogICAgICAgICItLWdyYWQtYWNjdW0iLCBzdHIoaHlwZXJbImdyYWRfYWNjdW0iXSksCiAgICAgICAgIi0tZW5jb2Rlci1sciIsIHN0cihoeXBlclsiZW5jb2Rlcl9sciJdKSwKICAgICAgICAiLS1oZWFkLWxyIiwgc3RyKGh5cGVyWyJoZWFkX2xyIl0pLAogICAgICAgICItLW1heC1sZW4iLCBzdHIoaHlwZXJbIm1heF9sZW4iXSksCiAgICAgICAgIi0taGVhZC1tYXgtbGVuIiwgc3RyKGh5cGVyWyJoZWFkX21heF9sZW4iXSksCiAgICAgICAgIi0tZGV2aWNlIiwgZGV2aWNlLAogICAgXQoKCmRlZiBydW5fb25lKGRhdGE6IHN0ciwgb3V0X3Jvb3Q6IHN0ciwgbG9zczogc3RyLCBzZWVkOiBpbnQsIGJhc2U6IHN0ciwgZGV2aWNlOiBzdHIsCiAgICAgICAgICAgIGh5cGVyOiBEaWN0W3N0ciwgQW55XSkgLT4gRGljdFtzdHIsIEFueV06CiAgICBydW5fZGlyID0gb3MucGF0aC5qb2luKG91dF9yb290LCBmIntsb3NzLnJlcGxhY2UoJy0nLCAnXycpfS1zZWVke3NlZWR9IikKICAgIG9zLm1ha2VkaXJzKHJ1bl9kaXIsIGV4aXN0X29rPVRydWUpCiAgICBjbWQgPSBidWlsZF9jb21tYW5kKGRhdGEsIHJ1bl9kaXIsIGxvc3MsIHNlZWQsIGJhc2UsIGRldmljZSwgaHlwZXIpCiAgICBwcmludCgiXG4iICsgIj0iICogNzIpCiAgICBwcmludCgiICIuam9pbihzaGxleC5xdW90ZShjKSBmb3IgYyBpbiBjbWQpKQogICAgcHJpbnQoIj0iICogNzIsIGZsdXNoPVRydWUpCgogICAgdDAgPSB0aW1lLnRpbWUoKQogICAgcmVzdWx0ID0gc3VicHJvY2Vzcy5ydW4oY21kLCB0ZXh0PVRydWUpCiAgICBlbGFwc2VkID0gcm91bmQodGltZS50aW1lKCkgLSB0MCwgMSkKICAgIGlmIHJlc3VsdC5yZXR1cm5jb2RlICE9IDA6CiAgICAgICAgcmFpc2UgU3lzdGVtRXhpdChmImxheWEtdHJhaW4gZmFpbGVkIGZvciBsb3NzPXtsb3NzfSBzZWVkPXtzZWVkfSAiCiAgICAgICAgICAgICAgICAgICAgICAgICBmIihleGl0IHtyZXN1bHQucmV0dXJuY29kZX0pIikKCiAgICBjZmdfcGF0aCA9IG9zLnBhdGguam9pbihydW5fZGlyLCAicmxfYWdlbnRfY29uZmlnLmpzb24iKQogICAgY2ZnOiBEaWN0W3N0ciwgQW55XSA9IHt9CiAgICBpZiBvcy5wYXRoLmV4aXN0cyhjZmdfcGF0aCk6CiAgICAgICAgd2l0aCBvcGVuKGNmZ19wYXRoLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgICAgICBjZmcgPSBqc29uLmxvYWQoZikKCiAgICByZXR1cm4gewogICAgICAgICJsb3NzIjogbG9zcywKICAgICAgICAic2VlZCI6IHNlZWQsCiAgICAgICAgInJ1bl9kaXIiOiBydW5fZGlyLAogICAgICAgICJlbGFwc2VkX3NlY29uZHMiOiBlbGFwc2VkLAogICAgICAgICJ0ZW1wZXJhdHVyZSI6IGNmZy5nZXQoInRlbXBlcmF0dXJlIiksCiAgICAgICAgInRlbXBlcmF0dXJlX2J5X29wdGlvbnMiOiBjZmcuZ2V0KCJ0ZW1wZXJhdHVyZV9ieV9vcHRpb25zIiwge30pLAogICAgICAgICJjYWxpYnJhdGlvbiI6IChjZmcuZ2V0KCJ0cmFpbmluZyIpIG9yIHt9KS5nZXQoImxheWFfdHJhaW5fY2FsaWJyYXRpb24iKSwKICAgICAgICAibWF4X2xlbiI6IGNmZy5nZXQoIm1heF9sZW4iKSwKICAgICAgICAiaGVhZF9tYXhfbGVuIjogY2ZnLmdldCgiaGVhZF9tYXhfbGVuIiksCiAgICB9CgoKZGVmIG1haW4oKSAtPiBpbnQ6CiAgICBwYXJzZXIgPSBhcmdwYXJzZS5Bcmd1bWVudFBhcnNlcihkZXNjcmlwdGlvbj1fX2RvY19fKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS1nYXRlIiwgY2hvaWNlcz1zb3J0ZWQoR0FURVMpLCByZXF1aXJlZD1UcnVlKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS1kYXRhIiwgcmVxdWlyZWQ9VHJ1ZSkKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tb3V0LWRpciIsIGRlZmF1bHQ9InJ1bnMvc3RlcDQiKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS1iYXNlIiwgZGVmYXVsdD0iY29udmFpaW5ub3ZhdGlvbnMvbGF5YSIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRldmljZSIsIGRlZmF1bHQ9ImF1dG8iKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS1sb3NzZXMiLCBuYXJncz0iKyIsIGRlZmF1bHQ9Tm9uZSwKICAgICAgICAgICAgICAgICAgICAgICAgaGVscD0iT3ZlcnJpZGUgdGhlIGdhdGUncyBsb3NzZXMuIikKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tc2VlZHMiLCBuYXJncz0iKyIsIHR5cGU9aW50LCBkZWZhdWx0PU5vbmUsCiAgICAgICAgICAgICAgICAgICAgICAgIGhlbHA9Ik92ZXJyaWRlIHRoZSBnYXRlJ3Mgc2VlZHMuIikKICAgIGFyZ3MgPSBwYXJzZXIucGFyc2VfYXJncygpCgogICAgbWF0cml4ID0gR0FURVNbYXJncy5nYXRlXQogICAgaWYgYXJncy5sb3NzZXM6CiAgICAgICAgbWF0cml4ID0gWyhsb3NzLCBzZWVkKSBmb3IgbG9zcywgc2VlZCBpbiBtYXRyaXggaWYgbG9zcyBpbiBhcmdzLmxvc3Nlc10KICAgIGlmIGFyZ3Muc2VlZHM6CiAgICAgICAgbWF0cml4ID0gWyhsb3NzLCBzZWVkKSBmb3IgbG9zcywgc2VlZCBpbiBtYXRyaXggaWYgc2VlZCBpbiBhcmdzLnNlZWRzXQogICAgaWYgbm90IG1hdHJpeDoKICAgICAgICBwYXJzZXIuZXJyb3IoInRoZSBsb3NzL3NlZWQgZmlsdGVycyBsZWZ0IHRoZSBnYXRlIGVtcHR5IikKCiAgICBvcy5tYWtlZGlycyhhcmdzLm91dF9kaXIsIGV4aXN0X29rPVRydWUpCiAgICBydW5zID0gW3J1bl9vbmUoYXJncy5kYXRhLCBhcmdzLm91dF9kaXIsIGxvc3MsIHNlZWQsIGFyZ3MuYmFzZSwgYXJncy5kZXZpY2UsCiAgICAgICAgICAgICAgICAgICAgTk9URUJPT0tfSFlQRVJQQVJBTVMpCiAgICAgICAgICAgIGZvciBsb3NzLCBzZWVkIGluIG1hdHJpeF0KCiAgICBtYW5pZmVzdCA9IHsKICAgICAgICAiZ2F0ZSI6IGFyZ3MuZ2F0ZSwKICAgICAgICAiYmFzZSI6IGFyZ3MuYmFzZSwKICAgICAgICAiZGF0YSI6IGFyZ3MuZGF0YSwKICAgICAgICAiaHlwZXJwYXJhbWV0ZXJzIjogTk9URUJPT0tfSFlQRVJQQVJBTVMsCiAgICAgICAgInJ1bnMiOiBydW5zLAogICAgfQogICAgbWFuaWZlc3RfcGF0aCA9IG9zLnBhdGguam9pbihhcmdzLm91dF9kaXIsIGYibWFuaWZlc3Rfe2FyZ3MuZ2F0ZX0uanNvbiIpCiAgICB3aXRoIG9wZW4obWFuaWZlc3RfcGF0aCwgInciLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIGpzb24uZHVtcChtYW5pZmVzdCwgZiwgaW5kZW50PTIpCiAgICBwcmludChmIlxud3JvdGUge21hbmlmZXN0X3BhdGh9IikKICAgIHJldHVybiAwCgoKaWYgX19uYW1lX18gPT0gIl9fbWFpbl9fIjoKICAgIHN5cy5leGl0KG1haW4oKSkK",
+ "evaluate.py": "IiIiRXZhbHVhdGUgYSBmaW5lLXR1bmVkIGNoZWNrcG9pbnQgb24gdGhlIHR5cGVkLWRlY2lzaW9ucyB0ZXN0IHNwbGl0LgoKTWlycm9ycyBjZWxsIDE0IG9mIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2ssIHJlc3RyaWN0ZWQgdG8gdGhlIHRocmVlIG1ldHJpY3MgdGhlIFN0ZXAtNCBjb250cmFjdApuYW1lczogKiphY2N1cmFjeSoqLCAqKkVDRSoqIGFuZCAqKkJyaWVyKiouIFRoZSAyMC1vcHRpb24gb3JkZXItZmxpcCBzdXJmYWNlIGlzIGRlbGliZXJhdGVseSBub3QKaGVyZTogaXQgYmVsb25ncyB0byBgbWFzc2l2ZV9pbnRlbnQuZW5gLCBub3QgdHlwZWQtZGVjaXNpb25zIChzZWUgUkVBRE1FKS4KCkNhbGlicmF0aW9uIGlzIG5vdCBieXRlLWNvbXBhcmFibGUgYWNyb3NzIGNvbnRyYWN0cy4gYGxheWEudHJhaW5gIGZpdHMgdGVtcGVyYXR1cmVzIHdpdGgKYGZpdF90ZW1wZXJhdHVyZV9tYXBgIChwZXIgdHlwZSwgYW5kIHBlciBvcHRpb24tY291bnQgYnVja2V0IGFib3ZlIGEgZmxvb3IpIHVuZGVyIHRoZSBydW50aW1lCmNsYW1wIGBbMC41LCA1LjBdYDsgdGhlIG9sZCBub3RlYm9vayBmaXR0ZWQgb25lIExCRkdTIHRlbXBlcmF0dXJlIHBlciB0eXBlIHVuZGVyIGBbMC4xLCAxMF1gLgpTbyBFQ0UgYW5kIEJyaWVyIGFyZSByZXBvcnRlZCB1bmRlciAqKmJvdGgqKiBwYXRoczoKCi0gYGAtLXRlbXBlcmF0dXJlcyBzaGlwcGVkYGAgKGRlZmF1bHQpIOKAlCB0aGUgY2hlY2twb2ludCdzIG93biBmaXR0ZWQgYHRlbXBlcmF0dXJlYCBhbmQKICBgdGVtcGVyYXR1cmVfYnlfb3B0aW9uc2AsIHdoaWNoIGlzIHdoYXQgdGhlIG5vdGVib29rJ3MgbnVtYmVyIGFsc28gdXNlZDsKLSBgYC0tdGVtcGVyYXR1cmVzIHJhd2BgIOKAlCB0ZW1wZXJhdHVyZSAxLjAgZXZlcnl3aGVyZSwgaXNvbGF0aW5nIHRoZSB0cmFpbmluZyBmcm9tIHRoZQogIGNhbGlicmF0aW9uIGNvbnRyYWN0LgoKQWNjdXJhY3kgaXMgdW5hZmZlY3RlZCBieSB0ZW1wZXJhdHVyZSAoaXQgb25seSByZXNjYWxlcyB0aGUgbG9naXRzKSwgc28gaXQgaXMgdGhlIGNsZWFuCnJlcHJvZHVjdGlvbiBzaWduYWwuCgpVc2FnZToKICAgIHB5dGhvbiBldmFsdWF0ZS5weSAtLWNoZWNrcG9pbnQgcnVucy9zdGVwNC9ybGNkLXNlZWQwIFwKICAgICAgICAtLXRlc3QtbWV0YSBkYXRhL3R5cGVkX2RlY2lzaW9ucy90ZXN0X21ldGEuanNvbiBcCiAgICAgICAgLS1vdXQgcnVucy9zdGVwNC9ybGNkLXNlZWQwL2V2YWxfc2hpcHBlZC5qc29uCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgppbXBvcnQgYXJncGFyc2UKaW1wb3J0IGpzb24KaW1wb3J0IG9zCmltcG9ydCBzeXMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdAoKc3lzLnBhdGguaW5zZXJ0KDAsIG9zLnBhdGguZGlybmFtZShvcy5wYXRoLmFic3BhdGgoX19maWxlX18pKSkKZnJvbSBtZXRyaWNzIGltcG9ydCAoICAjIG5vcWE6IEU0MDIKICAgIGFjY3VyYWN5LAogICAgYnJpZXJfc2NvcmUsCiAgICBleHBlY3RlZF9jYWxpYnJhdGlvbl9lcnJvciwKICAgIHNvZnRfYWNjdXJhY3ksCikKCgpkZWYgX25vcm1hbGlzZSh2YWx1ZXM6IExpc3RbZmxvYXRdKSAtPiBMaXN0W2Zsb2F0XToKICAgIHRvdGFsID0gc3VtKHZhbHVlcykKICAgIGlmIHRvdGFsID4gMDoKICAgICAgICByZXR1cm4gW3YgLyB0b3RhbCBmb3IgdiBpbiB2YWx1ZXNdCiAgICByZXR1cm4gdmFsdWVzCgoKZGVmIGV2YWx1YXRlX2NoZWNrcG9pbnQoY2hlY2twb2ludF9kaXI6IHN0ciwgdGVzdF9tZXRhOiBMaXN0W0RpY3Rbc3RyLCBBbnldXSwKICAgICAgICAgICAgICAgICAgICAgICAgZGV2aWNlOiBzdHIgPSAiYXV0byIsIHRlbXBlcmF0dXJlczogc3RyID0gInNoaXBwZWQiKSAtPiBEaWN0W3N0ciwgQW55XToKICAgIGltcG9ydCBsYXlhCgogICAgYWdlbnQgPSBsYXlhLmxvYWQoY2hlY2twb2ludF9kaXIsIGRldmljZT1kZXZpY2UpCiAgICBpZiB0ZW1wZXJhdHVyZXMgPT0gInJhdyI6CiAgICAgICAgYWdlbnQudGVtcGVyYXR1cmUgPSBbMS4wLCAxLjAsIDEuMF0KICAgICAgICBhZ2VudC50ZW1wZXJhdHVyZV9ieV9vcHRpb25zID0ge30KICAgIGVsaWYgdGVtcGVyYXR1cmVzICE9ICJzaGlwcGVkIjoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJ0ZW1wZXJhdHVyZXMgbXVzdCBiZSAnc2hpcHBlZCcgb3IgJ3JhdycsIGdvdCAlciIgJSAodGVtcGVyYXR1cmVzLCkpCgogICAgY29ycmVjdHM6IExpc3RbZmxvYXRdID0gW10KICAgIGNvbmZpZGVuY2VzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBicmllcnM6IExpc3RbZmxvYXRdID0gW10KICAgIHNvZnRzOiBMaXN0W2Zsb2F0XSA9IFtdCiAgICBwZXJfdHlwZTogRGljdFtzdHIsIExpc3RbZmxvYXRdXSA9IHsiY2hvaWNlIjogW10sICJub3VsIjogW10sICJzY29yZSI6IFtdfQogICAgc2NvcmVfbWFlczogTGlzdFtmbG9hdF0gPSBbXQogICAgd2l0aGluX29uZTogTGlzdFtmbG9hdF0gPSBbXQoKICAgIGZvciBpLCBjYXNlIGluIGVudW1lcmF0ZSh0ZXN0X21ldGEpOgogICAgICAgIHN0YXRlID0gY2FzZVsic3RhdGUiXQogICAgICAgIHF1ZXN0aW9ucyA9IGNhc2VbInF1ZXN0aW9ucyJdCiAgICAgICAgZ29sZCA9IGNhc2VbImdvbGQiXQogICAgICAgIGFuc3dlcnMgPSBhZ2VudC5wcmVkaWN0KHN0YXRlLCBxdWVzdGlvbnMpWyJhbnN3ZXJzIl0KCiAgICAgICAgZm9yIHFpZCwgcWRlZiBpbiBxdWVzdGlvbnMuaXRlbXMoKToKICAgICAgICAgICAgaWYgcWlkIG5vdCBpbiBnb2xkOgogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgZyA9IGdvbGRbcWlkXQogICAgICAgICAgICBxdHlwZSA9IHFkZWZbInR5cGUiXQogICAgICAgICAgICBwX2FucyA9IGFuc3dlcnNbcWlkXQoKICAgICAgICAgICAgaWYgcXR5cGUgPT0gImNob2ljZSI6CiAgICAgICAgICAgICAgICBrZXlzID0gbGlzdChxZGVmWyJjcml0ZXJpYSJdLmtleXMoKSkKICAgICAgICAgICAgICAgIHByZWQgPSBwX2Fuc1siY2hvaWNlIl0KICAgICAgICAgICAgICAgIGdvbGRfbGFiZWwgPSBzdHIoZ1sibGFiZWwiXSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwcmVkID09IGdvbGRfbGFiZWwpCgogICAgICAgICAgICAgICAgcF9wcm9icyA9IF9ub3JtYWxpc2UoW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KGssIDFlLTYpIGZvciBrIGluIGtleXNdKQogICAgICAgICAgICAgICAgZ19wcm9icyA9IF9ub3JtYWxpc2UoW2dbInByb2JhYmlsaXRpZXMiXS5nZXQoaywgMWUtNikgZm9yIGsgaW4ga2V5c10pCgogICAgICAgICAgICAgICAgY29ycmVjdHMuYXBwZW5kKGlzX2NvcnIpCiAgICAgICAgICAgICAgICBjb25maWRlbmNlcy5hcHBlbmQobWF4KHBfcHJvYnMpKQogICAgICAgICAgICAgICAgYnJpZXJzLmFwcGVuZChicmllcl9zY29yZShwX3Byb2JzLCBnX3Byb2JzKSkKICAgICAgICAgICAgICAgIHNvZnRzLmFwcGVuZChzb2Z0X2FjY3VyYWN5KHBfcHJvYnMsIGdfcHJvYnMpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbImNob2ljZSJdLmFwcGVuZChpc19jb3JyKQoKICAgICAgICAgICAgZWxpZiBxdHlwZSA9PSAibm91bCI6CiAgICAgICAgICAgICAgICBwX3ZhbCA9IGZsb2F0KHBfYW5zWyJub3VsIl0pCiAgICAgICAgICAgICAgICBnX3ZhbCA9IGZsb2F0KGcuZ2V0KCJub3VsIiwgZy5nZXQoInByb2JhYmlsaXRpZXMiLCB7fSkuZ2V0KCJ0cnVlIiwgMC41KSkpCiAgICAgICAgICAgICAgICBnb2xkX2xhYmVsID0gc3RyKGdbImxhYmVsIl0pLmxvd2VyKCkKICAgICAgICAgICAgICAgIHByZWRfbGFiZWwgPSAidHJ1ZSIgaWYgcF92YWwgPj0gMC41IGVsc2UgImZhbHNlIgogICAgICAgICAgICAgICAgaXNfY29yciA9IGZsb2F0KHByZWRfbGFiZWwgPT0gZ29sZF9sYWJlbCkKCiAgICAgICAgICAgICAgICBwX2Rpc3QgPSBbMS4wIC0gcF92YWwsIHBfdmFsXQogICAgICAgICAgICAgICAgZ19kaXN0ID0gWzEuMCAtIGdfdmFsLCBnX3ZhbF0KCiAgICAgICAgICAgICAgICBjb3JyZWN0cy5hcHBlbmQoaXNfY29ycikKICAgICAgICAgICAgICAgIGNvbmZpZGVuY2VzLmFwcGVuZChtYXgocF92YWwsIDEuMCAtIHBfdmFsKSkKICAgICAgICAgICAgICAgIGJyaWVycy5hcHBlbmQoYnJpZXJfc2NvcmUocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgc29mdHMuYXBwZW5kKHNvZnRfYWNjdXJhY3kocF9kaXN0LCBnX2Rpc3QpKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbIm5vdWwiXS5hcHBlbmQoaXNfY29ycikKCiAgICAgICAgICAgIGVsaWYgcXR5cGUgPT0gInNjb3JlIjoKICAgICAgICAgICAgICAgIHBfc2NvcmUgPSBmbG9hdChwX2Fuc1sic2NvcmUiXSkKICAgICAgICAgICAgICAgIGdfc2NvcmUgPSBmbG9hdChnLmdldCgic2NvcmUiLCAwLjApKQogICAgICAgICAgICAgICAgc2NvcmVfbWFlcy5hcHBlbmQoYWJzKHBfc2NvcmUgLSBnX3Njb3JlKSkKICAgICAgICAgICAgICAgIHdpdGhpbl9vbmUuYXBwZW5kKGZsb2F0KGFicyhwX3Njb3JlIC0gZ19zY29yZSkgPD0gMS4wKSkKCiAgICAgICAgICAgICAgICBuX2xldmVscyA9IGxlbihxZGVmLmdldCgiY3JpdGVyaWEiLCBbXSkpCiAgICAgICAgICAgICAgICBwX3Byb2JzID0gW3BfYW5zWyJwcm9iYWJpbGl0aWVzIl0uZ2V0KHN0cihpKSwgMC4wKSBmb3IgaSBpbiByYW5nZShuX2xldmVscyldCiAgICAgICAgICAgICAgICBpZiBzdW0ocF9wcm9icykgPiAwOgogICAgICAgICAgICAgICAgICAgIHBfcHJvYnMgPSBfbm9ybWFsaXNlKHBfcHJvYnMpCiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQobWF4KHJhbmdlKGxlbihwX3Byb2JzKSksIGtleT1sYW1iZGEgajogcF9wcm9ic1tqXSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKG1heChwX3Byb2JzKSkKICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgcF9sdmwgPSBpbnQocm91bmQocF9zY29yZSkpCiAgICAgICAgICAgICAgICAgICAgY29uZmlkZW5jZXMuYXBwZW5kKDAuNSkKCiAgICAgICAgICAgICAgICBnX2x2bCA9IGludChnLmdldCgibGFiZWwiLCBpbnQocm91bmQoZ19zY29yZSkpKSkKICAgICAgICAgICAgICAgIGlzX2NvcnIgPSBmbG9hdChwX2x2bCA9PSBnX2x2bCkKICAgICAgICAgICAgICAgIGNvcnJlY3RzLmFwcGVuZChpc19jb3JyKQogICAgICAgICAgICAgICAgcGVyX3R5cGVbInNjb3JlIl0uYXBwZW5kKGlzX2NvcnIpCgogICAgICAgIGlmIChpICsgMSkgJSAxMDAgPT0gMDoKICAgICAgICAgICAgcHJpbnQoZiIgIHtpICsgMX0ve2xlbih0ZXN0X21ldGEpfSBjYXNlcyIsIGZsdXNoPVRydWUpCgogICAgcmVzdWx0ID0gewogICAgICAgICJjaGVja3BvaW50IjogY2hlY2twb2ludF9kaXIsCiAgICAgICAgInRlbXBlcmF0dXJlcyI6IHRlbXBlcmF0dXJlcywKICAgICAgICAibl9jYXNlcyI6IGxlbih0ZXN0X21ldGEpLAogICAgICAgICJuX2RlY2lzaW9ucyI6IGxlbihjb3JyZWN0cyksCiAgICAgICAgImFjY3VyYWN5Ijogcm91bmQoYWNjdXJhY3koY29ycmVjdHMpLCA0KSwKICAgICAgICAiZWNlIjogcm91bmQoZXhwZWN0ZWRfY2FsaWJyYXRpb25fZXJyb3IoY29uZmlkZW5jZXMsIGNvcnJlY3RzKSwgNCksCiAgICAgICAgImJyaWVyIjogcm91bmQoZmxvYXQoc3VtKGJyaWVycykgLyBsZW4oYnJpZXJzKSkgaWYgYnJpZXJzIGVsc2UgZmxvYXQoIm5hbiIpLCA0KSwKICAgICAgICAic29mdF9hY2N1cmFjeSI6IHJvdW5kKGZsb2F0KHN1bShzb2Z0cykgLyBsZW4oc29mdHMpKSBpZiBzb2Z0cyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgInNjb3JlX21hZSI6IHJvdW5kKGZsb2F0KHN1bShzY29yZV9tYWVzKSAvIGxlbihzY29yZV9tYWVzKSkgaWYgc2NvcmVfbWFlcyBlbHNlIGZsb2F0KCJuYW4iKSwgNCksCiAgICAgICAgIndpdGhpbl9vbmVfbGV2ZWwiOiByb3VuZChmbG9hdChzdW0od2l0aGluX29uZSkgLyBsZW4od2l0aGluX29uZSkpIGlmIHdpdGhpbl9vbmUgZWxzZSBmbG9hdCgibmFuIiksIDQpLAogICAgICAgICJwZXJfdHlwZV9hY2N1cmFjeSI6IHtrOiByb3VuZChhY2N1cmFjeSh2KSwgNCkgZm9yIGssIHYgaW4gcGVyX3R5cGUuaXRlbXMoKSBpZiB2fSwKICAgIH0KICAgICMgYGJyaWVyYC9gc29mdF9hY2N1cmFjeWAvYGVjZWAgbWl4IHF1ZXN0aW9uIHR5cGVzOyByZWNvcmQgdGhlIGNvdW50cyB0aGV5IHJlc3Qgb24uCiAgICByZXN1bHRbIm5fYnJpZXJfZGVjaXNpb25zIl0gPSBsZW4oYnJpZXJzKQogICAgcmV0dXJuIHJlc3VsdAoKCmRlZiBtYWluKCkgLT4gaW50OgogICAgcGFyc2VyID0gYXJncGFyc2UuQXJndW1lbnRQYXJzZXIoZGVzY3JpcHRpb249X19kb2NfXykKICAgIHBhcnNlci5hZGRfYXJndW1lbnQoIi0tY2hlY2twb2ludCIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLXRlc3QtbWV0YSIsIHJlcXVpcmVkPVRydWUpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLWRldmljZSIsIGRlZmF1bHQ9ImF1dG8iKQogICAgcGFyc2VyLmFkZF9hcmd1bWVudCgiLS10ZW1wZXJhdHVyZXMiLCBjaG9pY2VzPVsic2hpcHBlZCIsICJyYXciXSwgZGVmYXVsdD0ic2hpcHBlZCIpCiAgICBwYXJzZXIuYWRkX2FyZ3VtZW50KCItLW91dCIsIGRlZmF1bHQ9Tm9uZSwKICAgICAgICAgICAgICAgICAgICAgICAgaGVscD0iT3V0cHV0IEpTT04gKGRlZmF1bHQ6IDxjaGVja3BvaW50Pi9ldmFsXzx0ZW1wZXJhdHVyZXM+Lmpzb24pIikKICAgIGFyZ3MgPSBwYXJzZXIucGFyc2VfYXJncygpCgogICAgd2l0aCBvcGVuKGFyZ3MudGVzdF9tZXRhLCBlbmNvZGluZz0idXRmLTgiKSBhcyBmOgogICAgICAgIHRlc3RfbWV0YSA9IGpzb24ubG9hZChmKQogICAgcHJpbnQoZiJsb2FkZWQge2xlbih0ZXN0X21ldGEpfSB0ZXN0IGNhc2VzIiwgZmx1c2g9VHJ1ZSkKCiAgICByZXN1bHQgPSBldmFsdWF0ZV9jaGVja3BvaW50KGFyZ3MuY2hlY2twb2ludCwgdGVzdF9tZXRhLCBkZXZpY2U9YXJncy5kZXZpY2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRlbXBlcmF0dXJlcz1hcmdzLnRlbXBlcmF0dXJlcykKCiAgICBwcmludCgiXG4iICsgIj0iICogNjApCiAgICBwcmludChmImV2YWx1YXRpb246IHthcmdzLmNoZWNrcG9pbnR9ICh7YXJncy50ZW1wZXJhdHVyZXN9IHRlbXBlcmF0dXJlcykiKQogICAgcHJpbnQoIj0iICogNjApCiAgICBwcmludChmIiAgYWNjdXJhY3k6ICAgICB7cmVzdWx0WydhY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIEVDRTogICAgICAgICAge3Jlc3VsdFsnZWNlJ106LjRmfSIpCiAgICBwcmludChmIiAgQnJpZXI6ICAgICAgICB7cmVzdWx0WydicmllciddOi40Zn0iKQogICAgcHJpbnQoZiIgIHNvZnQgYWNjOiAgICAge3Jlc3VsdFsnc29mdF9hY2N1cmFjeSddOi40Zn0iKQogICAgcHJpbnQoZiIgIHBlciB0eXBlOiAgICAge3Jlc3VsdFsncGVyX3R5cGVfYWNjdXJhY3knXX0iKQogICAgcHJpbnQoIj0iICogNjApCgogICAgb3V0X3BhdGggPSBhcmdzLm91dCBvciBvcy5wYXRoLmpvaW4oYXJncy5jaGVja3BvaW50LCBmImV2YWxfe2FyZ3MudGVtcGVyYXR1cmVzfS5qc29uIikKICAgIG9zLm1ha2VkaXJzKG9zLnBhdGguZGlybmFtZShvdXRfcGF0aCkgb3IgIi4iLCBleGlzdF9vaz1UcnVlKQogICAgd2l0aCBvcGVuKG91dF9wYXRoLCAidyIsIGVuY29kaW5nPSJ1dGYtOCIpIGFzIGY6CiAgICAgICAganNvbi5kdW1wKHJlc3VsdCwgZiwgaW5kZW50PTIpCiAgICBwcmludChmIndyb3RlIHtvdXRfcGF0aH0iKQogICAgcmV0dXJuIDAKCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgc3lzLmV4aXQobWFpbigpKQo=",
+ "metrics.py": "IiIiTWV0cmljIGRlZmluaXRpb25zIGZvciB0aGUgU3RlcC00IGJlbmNobWFyaywgbWlycm9yaW5nIHRoZSBwdWJsaXNoZWQgbm90ZWJvb2suCgpSZWZlcmVuY2U6IGNlbGwgMTQgb2YgYG5vdGVib29rcy9sYXlhX2ZpbmV0dW5lX3R5cGVkX2RlY2lzaW9uc18yeFQ0X2thZ2dsZS5pcHluYmAuIFRoZSBwdWJsaXNoZWQKMC43NjYgY2hlY2twb2ludCBhbmQgdGhlIG1haW50YWluZXIncyByZWZlcmVuY2UgdGFibGUgYXJlIHNjb3JlZCB3aXRoIHRoZXNlIGRlZmluaXRpb25zLCBzbyBhCnJlcHJvZHVjdGlvbiBoYXMgdG8gdXNlIHRoZW0gdG9vLCBub3QgYSBkaWZmZXJlbnRseS1kZWZpbmVkIGxpYnJhcnkgbWV0cmljOgoKLSAqKmFjY3VyYWN5Kiog4oCUIHRvcCByZXBvcnRlZCBvcHRpb24gYWdhaW5zdCB0aGUgZ29sZCBsYWJlbCwgb3ZlciBldmVyeSBkZWNpc2lvbiAoY2hvaWNlLCBzY29yZQogIGFuZCBub3VsKS4KLSAqKkVDRSoqIOKAlCBgbGF5YS5jb21tb24uZWNlX3Njb3JlYCBvZiB0aGUgdG9wIHByb2JhYmlsaXR5IGFnYWluc3QgY29ycmVjdG5lc3MsIDE1IGVxdWFsLXdpZHRoCiAgYmlucywgb3ZlciBldmVyeSBkZWNpc2lvbi4KLSAqKmJyaWVyKiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIGRlY2lzaW9ucyBvZiBgc3VtKChwIC0gZ29sZCkqKjIpYC4gVGhlIG5vdGVib29rIGRvZXMKICBub3QgYXBwZW5kIHNjb3JlIGRlY2lzaW9ucyB0byB0aGlzIG1lYW4sIHNvIGEgc2NvcmUgY2hhbmdlIG5ldmVyIG1vdmVzIEJyaWVyOyBrZXB0IGFzLWlzIHNvIHRoZQogIG51bWJlciBjb21wYXJlcyB3aXRoIHRoZSByZWZlcmVuY2UuCi0gKipzb2Z0X2FjY3VyYWN5Kiog4oCUIG1lYW4gb3ZlciAqKmNob2ljZSBhbmQgbm91bCoqIG9mIGBzdW0ocCAqIGdvbGQpYC4KCmBnb2xkYCBpcyB0aGUgdGVhY2hlciBkaXN0cmlidXRpb24gKGBnb2xkLnByb2JhYmlsaXRpZXNgKSwgbm90IGEgb25lLWhvdCBsYWJlbCwgc28gQnJpZXIgYW5kIHNvZnQKYWNjdXJhY3kgbWVhc3VyZSB0aGUgZGlzdHJpYnV0aW9uIHRoZSBydW4gd2FzIHRyYWluZWQgdG8gbWF0Y2guCiIiIgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IGFubm90YXRpb25zCgpmcm9tIHR5cGluZyBpbXBvcnQgU2VxdWVuY2UKCmltcG9ydCBudW1weSBhcyBucAoKZnJvbSBsYXlhLmNvbW1vbiBpbXBvcnQgZWNlX3Njb3JlCgoKZGVmIGFjY3VyYWN5KGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICByZXR1cm4gZmxvYXQobnAubWVhbihjb3JyZWN0KSkgaWYgbGVuKGNvcnJlY3QpIGVsc2UgZmxvYXQoIm5hbiIpCgoKZGVmIGV4cGVjdGVkX2NhbGlicmF0aW9uX2Vycm9yKGNvbmZpZGVuY2VzOiBTZXF1ZW5jZVtmbG9hdF0sIGNvcnJlY3Q6IFNlcXVlbmNlW2Zsb2F0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJpbnM6IGludCA9IDE1KSAtPiBmbG9hdDoKICAgICIiIlRoaW4gd3JhcHBlciBvdmVyIHRoZSByZXBvc2l0b3J5J3MgYGNvbW1vbi5lY2Vfc2NvcmVgLCBhcyB0aGUgbm90ZWJvb2sgY2FsbHMgaXQuIiIiCiAgICBpZiBub3QgbGVuKGNvbmZpZGVuY2VzKToKICAgICAgICByZXR1cm4gZmxvYXQoIm5hbiIpCiAgICByZXR1cm4gZmxvYXQoZWNlX3Njb3JlKG5wLmFzYXJyYXkoY29uZmlkZW5jZXMsIGR0eXBlPWZsb2F0KSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbnAuYXNhcnJheShjb3JyZWN0LCBkdHlwZT1mbG9hdCksIGJpbnM9YmlucykpCgoKZGVmIGJyaWVyX3Njb3JlKHByb2JzOiBTZXF1ZW5jZVtmbG9hdF0sIGdvbGQ6IFNlcXVlbmNlW2Zsb2F0XSkgLT4gZmxvYXQ6CiAgICBwID0gbnAuYXNhcnJheShwcm9icywgZHR5cGU9ZmxvYXQpCiAgICBnID0gbnAuYXNhcnJheShnb2xkLCBkdHlwZT1mbG9hdCkKICAgIHJldHVybiBmbG9hdCgoKHAgLSBnKSAqKiAyKS5zdW0oKSkKCgpkZWYgc29mdF9hY2N1cmFjeShwcm9iczogU2VxdWVuY2VbZmxvYXRdLCBnb2xkOiBTZXF1ZW5jZVtmbG9hdF0pIC0+IGZsb2F0OgogICAgcCA9IG5wLmFzYXJyYXkocHJvYnMsIGR0eXBlPWZsb2F0KQogICAgZyA9IG5wLmFzYXJyYXkoZ29sZCwgZHR5cGU9ZmxvYXQpCiAgICByZXR1cm4gZmxvYXQoKHAgKiBnKS5zdW0oKSkK"
+}
+for _name, _b64 in _EMBEDDED.items():
+ (HARNESS / _name).write_bytes(base64.b64decode(_b64))
+sys.path.insert(0, str(HARNESS))
+
+
+def run(cmd):
+ print("\n$ " + " ".join(cmd), flush=True)
+ return subprocess.run(cmd, check=True)
+
+
+# --- one visible T4 -------------------------------------------------------------------------
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+import torch # noqa: E402
+
+n_gpu = torch.cuda.device_count()
+print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
+for _i in range(n_gpu):
+ _p = torch.cuda.get_device_properties(_i)
+ print(" GPU", _i, _p.name, round(_p.total_memory / 1e9, 1), "GB", flush=True)
+assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu
+
+# --- laya at the pinned SHA, without disturbing Kaggle's torch ------------------------------
+run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
+ "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
+for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
+ ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
+ try:
+ __import__(_mod)
+ except Exception:
+ run([sys.executable, "-m", "pip", "install", "-q", _pkg])
+
+# --- base checkpoint config readback --------------------------------------------------------
+from huggingface_hub import snapshot_download # noqa: E402
+
+base_dir = snapshot_download("convaiinnovations/laya")
+with open(os.path.join(base_dir, "rl_agent_config.json"), encoding="utf-8") as _f:
+ base_cfg = json.load(_f)
+base_keys = ("max_len", "head_max_len", "max_tokens_per_batch", "temperature",
+ "temperature_by_options", "option_layout")
+print("BASE_CONFIG " + json.dumps({k: base_cfg.get(k) for k in base_keys}), flush=True)
+
+# --- data -----------------------------------------------------------------------------------
+run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
+with open(WORK / "data" / "train.jsonl", encoding="utf-8") as _f:
+ n_train_cases = sum(1 for _l in _f if _l.strip())
+with open(WORK / "data" / "test_meta.json", encoding="utf-8") as _f:
+ test_meta = json.load(_f)
+print("SPLITS train_cases=%d test_cases=%d" % (n_train_cases, len(test_meta)), flush=True)
+
+# --- Gate R: rlcd, seed 0, notebook hyperparameters ----------------------------------------
+cmd = [sys.executable, "-m", "laya.train_cli",
+ "--data", str(WORK / "data" / "train.jsonl"),
+ "--out", str(WORK / "run_rlcd_seed0"),
+ "--base", "convaiinnovations/laya",
+ "--loss", "rlcd", "--seed", "0",
+ "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
+ "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
+ "--max-len", "1024", "--head-max-len", "256",
+ "--device", "cuda"]
+_t0 = time.time()
+_proc = subprocess.run(cmd, check=True, capture_output=True, text=True)
+train_seconds = round(time.time() - _t0, 1)
+print(_proc.stdout)
+if _proc.stderr:
+ print(_proc.stderr, file=sys.stderr)
+(WORK / "step4_train.log").write_text(_proc.stdout + "\n" + _proc.stderr, encoding="utf-8")
+
+train_split_line = next((ln for ln in _proc.stdout.splitlines() if ln.startswith("train items")), "")
+peak_gpu_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
+print("TRAIN_SECONDS", train_seconds, "PEAK_GPU_GB", peak_gpu_gb, flush=True)
+print("TRAIN_SPLIT", train_split_line, flush=True)
+
+# --- evaluate with the Step-4 evaluator -----------------------------------------------------
+from evaluate import evaluate_checkpoint # noqa: E402
+
+results = {
+ "meta": {
+ "pinned_sha": PINNED_SHA,
+ "torch": torch.__version__,
+ "gpu": torch.cuda.get_device_name(0),
+ "visible_gpus": n_gpu,
+ "train_seconds": train_seconds,
+ "peak_gpu_gb": peak_gpu_gb,
+ "train_split_line": train_split_line,
+ "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
+ "n_train_cases": n_train_cases,
+ "n_test_cases": len(test_meta),
+ }
+}
+for _label, _ckpt in (("rlcd_seed0", str(WORK / "run_rlcd_seed0")),
+ ("base", "convaiinnovations/laya"),
+ ("published", "convaiinnovations/laya-typed-decisions")):
+ try:
+ results[_label] = evaluate_checkpoint(_ckpt, test_meta, device="cuda", temperatures="shipped")
+ except Exception as _e: # noqa: BLE001 - a missing optional row must not kill the run
+ results[_label] = {"error": "%s: %s" % (type(_e).__name__, _e)}
+ print(_label, json.dumps(results[_label]), flush=True)
+
+with open(WORK / "step4_gate_r_results.json", "w", encoding="utf-8") as _f:
+ json.dump(results, _f, indent=2)
+print("\nRESULTS " + json.dumps(results, indent=2), flush=True)