Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

7.6 KB
Raw Blame
"""Generate the self-contained Kaggle Gate R runner for #887 Step 4.

The harness under this directory is not in the upstream repo yet (public mutation authority is
NONE), so the kernel cannot clone it. This generator embeds the four harness files verbatim (as
base64) into one script, plus a `kernel-metadata.json` pinned to a single T4.

Single T4 on Kaggle means `machine_shape: "NvidiaTeslaT4"`; the kernel then pins
`CUDA_VISIBLE_DEVICES=0` and asserts exactly one visible GPU, so the run is single-device with no
torchrun and no 2-GPU assert — the shipped `laya-train` path.

Run the API with `KAGGLE_API_TOKEN` exported from `kaggle auth print-access-token`; the stored
credential is not picked up automatically by this CLI build.

    python research/scripts/step4_benchmark/kaggle/build_kernel.py
    KAGGLE_API_TOKEN=$(kaggle auth print-access-token) kaggle kernels push -p <dir>
"""
from __future__ import annotations

import base64
import json
import os
from pathlib import Path

HERE = Path(__file__).resolve().parent
HARNESS = HERE.parent
EMBED = ("prepare_data.py", "run_benchmark.py", "evaluate.py", "metrics.py")
PINNED_SHA = "a4a8921afebfd852bba0000475cfb6ab737a124c"
KERNEL_ID = "yuyi722333/laya-step4-gate-r"

TEMPLATE = '''"""#887 Step 4 Gate R, generated by build_kernel.py -- do not edit here.

Single-device T4 reproduction of the typed-decisions fine-tune through the shipped `laya-train`,
then the Step-4 evaluator. No torchrun, no 2-GPU assert, no early stopping, no option shuffling.
"""
import base64
import json
import os
import subprocess
import sys
import time
from pathlib import Path

PINNED_SHA = "__PINNED_SHA__"
WORK = Path("/kaggle/working")
HARNESS = WORK / "step4_benchmark"
HARNESS.mkdir(parents=True, exist_ok=True)

_EMBEDDED = __EMBEDDED__
for _name, _b64 in _EMBEDDED.items():
    (HARNESS / _name).write_bytes(base64.b64decode(_b64))
sys.path.insert(0, str(HARNESS))


def run(cmd):
    print("\\n$ " + " ".join(cmd), flush=True)
    return subprocess.run(cmd, check=True)


# --- one visible T4 -------------------------------------------------------------------------
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import torch  # noqa: E402

n_gpu = torch.cuda.device_count()
print("torch", torch.__version__, "| cuda", torch.cuda.is_available(), "| visible_gpus", n_gpu, flush=True)
for _i in range(n_gpu):
    _p = torch.cuda.get_device_properties(_i)
    print("  GPU", _i, _p.name, round(_p.total_memory / 1e9, 1), "GB", flush=True)
assert n_gpu == 1, "expected exactly one visible T4, got %d" % n_gpu

# --- laya at the pinned SHA, without disturbing Kaggle's torch ------------------------------
run([sys.executable, "-m", "pip", "install", "-q", "--no-deps",
     "git+https://github.com/NandhaKishorM/laya@%s" % PINNED_SHA])
for _mod, _pkg in (("safetensors", "safetensors"), ("datasets", "datasets"),
                   ("transformers", "transformers"), ("huggingface_hub", "huggingface_hub")):
    try:
        __import__(_mod)
    except Exception:
        run([sys.executable, "-m", "pip", "install", "-q", _pkg])

# --- base checkpoint config readback --------------------------------------------------------
from huggingface_hub import snapshot_download  # noqa: E402

base_dir = snapshot_download("convaiinnovations/laya")
with open(os.path.join(base_dir, "rl_agent_config.json"), encoding="utf-8") as _f:
    base_cfg = json.load(_f)
base_keys = ("max_len", "head_max_len", "max_tokens_per_batch", "temperature",
             "temperature_by_options", "option_layout")
print("BASE_CONFIG " + json.dumps({k: base_cfg.get(k) for k in base_keys}), flush=True)

# --- data -----------------------------------------------------------------------------------
run([sys.executable, str(HARNESS / "prepare_data.py"), "--out-dir", str(WORK / "data")])
with open(WORK / "data" / "train.jsonl", encoding="utf-8") as _f:
    n_train_cases = sum(1 for _l in _f if _l.strip())
with open(WORK / "data" / "test_meta.json", encoding="utf-8") as _f:
    test_meta = json.load(_f)
print("SPLITS train_cases=%d test_cases=%d" % (n_train_cases, len(test_meta)), flush=True)

# --- Gate R: rlcd, seed 0, notebook hyperparameters ----------------------------------------
cmd = [sys.executable, "-m", "laya.train_cli",
       "--data", str(WORK / "data" / "train.jsonl"),
       "--out", str(WORK / "run_rlcd_seed0"),
       "--base", "convaiinnovations/laya",
       "--loss", "rlcd", "--seed", "0",
       "--epochs", "4", "--micro-batch", "8", "--grad-accum", "8",
       "--encoder-lr", "2.5e-5", "--head-lr", "1e-4",
       "--max-len", "1024", "--head-max-len", "256",
       "--device", "cuda"]
_t0 = time.time()
_proc = subprocess.run(cmd, check=True, capture_output=True, text=True)
train_seconds = round(time.time() - _t0, 1)
print(_proc.stdout)
if _proc.stderr:
    print(_proc.stderr, file=sys.stderr)
(WORK / "step4_train.log").write_text(_proc.stdout + "\\n" + _proc.stderr, encoding="utf-8")

train_split_line = next((ln for ln in _proc.stdout.splitlines() if ln.startswith("train items")), "")
peak_gpu_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
print("TRAIN_SECONDS", train_seconds, "PEAK_GPU_GB", peak_gpu_gb, flush=True)
print("TRAIN_SPLIT", train_split_line, flush=True)

# --- evaluate with the Step-4 evaluator -----------------------------------------------------
from evaluate import evaluate_checkpoint  # noqa: E402

results = {
    "meta": {
        "pinned_sha": PINNED_SHA,
        "torch": torch.__version__,
        "gpu": torch.cuda.get_device_name(0),
        "visible_gpus": n_gpu,
        "train_seconds": train_seconds,
        "peak_gpu_gb": peak_gpu_gb,
        "train_split_line": train_split_line,
        "base_config": {k: base_cfg.get(k) for k in ("max_len", "head_max_len")},
        "n_train_cases": n_train_cases,
        "n_test_cases": len(test_meta),
    }
}
for _label, _ckpt in (("rlcd_seed0", str(WORK / "run_rlcd_seed0")),
                      ("base", "convaiinnovations/laya"),
                      ("published", "convaiinnovations/laya-typed-decisions")):
    try:
        results[_label] = evaluate_checkpoint(_ckpt, test_meta, device="cuda", temperatures="shipped")
    except Exception as _e:  # noqa: BLE001 - a missing optional row must not kill the run
        results[_label] = {"error": "%s: %s" % (type(_e).__name__, _e)}
    print(_label, json.dumps(results[_label]), flush=True)

with open(WORK / "step4_gate_r_results.json", "w", encoding="utf-8") as _f:
    json.dump(results, _f, indent=2)
print("\\nRESULTS " + json.dumps(results, indent=2), flush=True)
'''


def main() -> int:
    embedded = {}
    for name in EMBED:
        embedded[name] = base64.b64encode((HARNESS / name).read_bytes()).decode("ascii")

    script = (TEMPLATE
              .replace("__PINNED_SHA__", PINNED_SHA)
              .replace("__EMBEDDED__", json.dumps(embedded, indent=1)))
    (HERE / "step4_gate_r.py").write_text(script, encoding="utf-8", newline="\n")

    metadata = {
        "id": KERNEL_ID,
        "title": "Laya Step 4 Gate R",
        "code_file": "step4_gate_r.py",
        "language": "python",
        "kernel_type": "script",
        "is_private": True,
        "enable_gpu": True,
        "enable_tpu": False,
        "enable_internet": True,
        "machine_shape": "NvidiaTeslaT4",
        "dataset_sources": [],
        "competition_sources": [],
        "kernel_sources": [],
        "model_sources": [],
    }
    (HERE / "kernel-metadata.json").write_text(json.dumps(metadata, indent=2) + "\n",
                                               encoding="utf-8", newline="\n")
    print("wrote", HERE / "step4_gate_r.py", "and", HERE / "kernel-metadata.json")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())