Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

8.8 KB
Raw Blame
"""Generate the Gate D Kaggle kernel: the remaining 5 cells of the 3-seed x 2-loss panel.

One single-T4 session runs the five remaining trainings sequentially under the pinned Gate-R
contract, fail-closed after each: checkpoint exists, `training.laya_train` records the intended
loss/seed/config, counts match (5600/400/skipped {}), metrics finite, and the base sanity row stays
within tolerance of the Gate-R reference. A hard mismatch writes the partial results and aborts.

RLCD seed 0 is reused from the Gate R kernel (different session, same pinned SHA/data/command), as
the Coordinator explicitly allowed; it is embedded as the sixth cell.

    python research/scripts/step4_benchmark/kaggle/build_gate_d.py
    KAGGLE_API_TOKEN=$(kaggle auth print-access-token) kaggle kernels push -p <this dir>/gate_d
"""
from __future__ import annotations

import base64
import json
from pathlib import Path

HERE = Path(__file__).resolve().parent
HARNESS = HERE.parent
OUT = HERE / "gate_d"
EMBED = ("prepare_data.py", "evaluate.py", "metrics.py")
PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
KERNEL_ID = "yuyi722333/laya-step4-gate-d"

TEMPLATE = '''"""#887 Step 4 Gate D, generated by build_gate_d.py -- do not edit here."""
import base64
import json
import math
import os
import subprocess
import sys
import time
from pathlib import Path

PINNED_SHA = "__PINNED_SHA__"
WORK = Path("/kaggle/working")
HARNESS = WORK / "step4_benchmark"
HARNESS.mkdir(parents=True, exist_ok=True)

_EMBEDDED = __EMBEDDED__
for _n, _b in _EMBEDDED.items():
    (HARNESS / _n).write_bytes(base64.b64decode(_b))
sys.path.insert(0, str(HARNESS))

os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import torch  # noqa: E402

n_gpu = torch.cuda.device_count()
print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu
print("GPU", torch.cuda.get_device_name(0), flush=True)

RESULTS_PATH = WORK / "step4_gate_d_results.json"


def flush_results():
    RESULTS_PATH.write_text(json.dumps(RESULTS, indent=2), encoding="utf-8")


def run(cmd):
    print("\\n$ " + " ".join(cmd), flush=True)
    p = subprocess.run(cmd, check=True, text=True, capture_output=True)
    sys.stdout.write(p.stdout)
    if p.stderr:
        sys.stderr.write(p.stderr)
    sys.stdout.flush()
    return p.stdout


run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
     "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
                   ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
    try:
        __import__(_mod)
    except Exception:
        run([sys.executable, "-m", "pip", "install", "-q", _pkg])

from huggingface_hub import snapshot_download  # noqa: E402

base_dir = snapshot_download("convaiinnovations/laya")
base_cfg = json.loads((Path(base_dir) / "rl_agent_config.json").read_text(encoding="utf-8"))

run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
test_meta = json.loads((WORK / "data" / "test_meta.json").read_text(encoding="utf-8"))

from evaluate import evaluate_checkpoint  # noqa: E402

RESULTS = {
    "meta": {
        "pinned_sha": PINNED_SHA,
        "torch": torch.__version__,
        "gpu": torch.cuda.get_device_name(0),
        "visible_gpus": n_gpu,
        "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
        "n_train_cases": 1200,
        "n_test_cases": len(test_meta),
        "reused": {"rlcd_seed0": "Gate R kernel (separate session, same SHA/data/command)"},
    },
    "rlcd_seed0": {
        "checkpoint": "Gate R kernel run_rlcd_seed0 (reused)",
        "temperatures": "shipped", "n_cases": 400, "n_decisions": 2000,
        "accuracy": 0.775, "ece": 0.1504, "brier": 0.0519, "soft_accuracy": 0.5654,
        "per_type_accuracy": {"choice": 0.7417, "noul": 0.8517, "score": 0.7425},
        "loss": "rlcd", "seed": 0, "wall_clock_s": 2691.3,
    },
}

# --- base sanity row, must match the Gate-R reference -----------------------
base_row = evaluate_checkpoint("convaiinnovations/laya", test_meta, device="cuda",
                               temperatures="shipped")
RESULTS["base"] = base_row
base_ok = (abs(base_row["accuracy"] - 0.362) <= 0.01 and abs(base_row["ece"] - 0.1741) <= 0.01
           and abs(base_row["brier"] - 0.3287) <= 0.01)
print("BASE_SANITY", "OK" if base_ok else "FAIL", json.dumps(base_row), flush=True)
flush_results()
if not base_ok:
    raise SystemExit("GATE_D_PROTOCOL_PROBLEM: base sanity drift")

# --- the five remaining cells ----------------------------------------------
CELLS = [("rlcd", 1), ("soft-ce", 0), ("soft-ce", 1), ("rlcd", 2), ("soft-ce", 2)]


def train(loss, seed):
    out = WORK / ("run_" + loss.replace("-", "_") + "_seed%d" % seed)
    cmd = [sys.executable, "-m", "laya.train_cli",
           "--data", str(WORK / "data" / "train.jsonl"),
           "--out", str(out),
           "--base", "convaiinnovations/laya",
           "--loss", loss, "--seed", str(seed),
           "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
           "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
           "--max-len", "1024", "--head-max-len", "256",
           "--device", "cuda"]
    t0 = time.time()
    stdout = run(cmd)
    dt = round(time.time() - t0, 1)
    cfg = json.loads((out / "rl_agent_config.json").read_text(encoding="utf-8"))
    lt = (cfg.get("training") or {}).get("laya_train", {})
    split_line = next((ln for ln in stdout.splitlines() if ln.startswith("train items")), "")
    checks = {
        "loss": lt.get("loss") == loss,
        "seed": lt.get("seed") == seed,
        "epochs": lt.get("epochs") == 4,
        "micro_batch": lt.get("micro_batch") == 8,
        "grad_accum": lt.get("grad_accum") == 8,
        "encoder_lr": abs(float(lt.get("encoder_lr", 0)) - 2.5e-5) < 1e-12,
        "head_lr": abs(float(lt.get("head_lr", 0)) - 1e-4) < 1e-12,
        "max_len": lt.get("max_len") == 1024,
        "head_max_len": lt.get("head_max_len") == 256,
        "calib_seed": lt.get("calib_seed") == 20260922,
        "no_shuffle": list(lt.get("shuffle_options") or []) == [],
        "counts": "train items 5600, calibration items 400, skipped {}" in split_line,
    }
    return out, dt, cfg, checks, split_line


for loss, seed in CELLS:
    out, dt, cfg, checks, split_line = train(loss, seed)
    print("CELL %s seed%d checks %s" % (loss, seed, json.dumps(checks)), flush=True)
    bad = sorted(k for k, v in checks.items() if not v)
    if bad:
        RESULTS["%s_seed%d" % (loss, seed)] = {"error": "protocol mismatch: " + ",".join(bad),
                                               "split_line": split_line}
        flush_results()
        raise SystemExit("GATE_D_PROTOCOL_PROBLEM: %s seed%d -> %s" % (loss, seed, bad))

    res = evaluate_checkpoint(str(out), test_meta, device="cuda", temperatures="shipped")
    finite = all(isinstance(res.get(k), (int, float)) and math.isfinite(res[k])
                 for k in ("accuracy", "ece", "brier"))
    res.update({
        "loss": loss, "seed": seed, "wall_clock_s": dt,
        "temperature": cfg.get("temperature"),
        "calibration": (cfg.get("training") or {}).get("laya_train_calibration"),
        "split_line": split_line, "finite": finite,
    })
    RESULTS["%s_seed%d" % (loss, seed)] = res
    flush_results()
    print("RESULT %s seed%d %s" % (loss, seed, json.dumps(res)), flush=True)
    if not finite:
        raise SystemExit("GATE_D_PROTOCOL_PROBLEM: non-finite metric %s seed%d" % (loss, seed))

print("ALL_CELLS_DONE", flush=True)
print("RESULTS " + json.dumps(RESULTS, indent=2), flush=True)
'''


def main() -> int:
    OUT.mkdir(parents=True, exist_ok=True)
    embedded = {name: base64.b64encode((HARNESS / name).read_bytes()).decode("ascii")
                for name in EMBED}
    script = (TEMPLATE.replace("__PINNED_SHA__", PINNED_SHA)
              .replace("__EMBEDDED__", json.dumps(embedded, indent=1)))
    (OUT / "step4_gate_d.py").write_text(script, encoding="utf-8", newline="\n")

    metadata = {
        "id": KERNEL_ID,
        "title": "Laya Step 4 Gate D",
        "code_file": "step4_gate_d.py",
        "language": "python",
        "kernel_type": "script",
        "is_private": True,
        "enable_gpu": True,
        "enable_tpu": False,
        "enable_internet": True,
        "machine_shape": "NvidiaTeslaT4",
        "dataset_sources": [],
        "competition_sources": [],
        "kernel_sources": [],
        "model_sources": [],
    }
    (OUT / "kernel-metadata.json").write_text(json.dumps(metadata, indent=2) + "\n",
                                              encoding="utf-8", newline="\n")
    print("wrote", OUT / "step4_gate_d.py", "and", OUT / "kernel-metadata.json")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())