Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
"""Train a frozen-backbone decision probe for the Bonsai 2 Vision readout.
Inputs are hardware-eval reports from scripts/check-webgpu-decisions.mjs run
with `--prompt verify --return-hidden` (one last-position hidden state and one
Yes-minus-No margin per option) on d1's balanced-V3 train, validation and
final-test splits, plus the split files for the labels.
Each option is scored as a * margin + w . standardize(hidden) + b_type, and a
softmax over a question's options gives its answer, the same grouped readout
d1 uses over its option markers. Only 2048 + a few parameters are trained;
the regularization strength is chosen on validation, and the test split is
scored once with the chosen model.
python scripts/models/bonsai/train_decision_probe.py --reports <dir> --data <split dir> --out probe.json
"""
import argparse
import base64
import json
from pathlib import Path
import numpy as np
from scipy.optimize import minimize
TYPES = ("choice", "noul", "score")
def decode(text):
return np.frombuffer(base64.b64decode(text), dtype=np.float16).astype(np.float32)
def load(report_path, split_path):
report = json.loads(Path(report_path).read_text())
answers = report["engines"]["bonsai"]["answers"]
split = json.loads(Path(split_path).read_text())
rows = []
for request in split["requests"]:
for entry in request["questions"]:
answer = answers.get(entry["id"])
if not answer or not answer.get("hidden") or any(h is None for h in answer["hidden"]):
continue
question = entry["question"]
if question["type"] == "noul":
names = ["no", "yes"]
elif question["type"] == "score":
names = [str(i) for i in range(len(question["criteria"]))]
else:
names = list(question["criteria"].keys())
label = entry["label"]
target = names.index(label["selected_option"]) if question["type"] != "score" else label["selected_index"]
rows.append(dict(entry=entry, type=question["type"], names=names, target=target,
margins=np.array(answer["margins"], dtype=np.float64),
hidden=np.stack([decode(h) for h in answer["hidden"]]).astype(np.float64)))
return rows
def scores(params, row, mean, std):
width = mean.shape[0]
a, w = params[0], params[1:1 + width]
bias = params[1 + width + TYPES.index(row["type"])]
return a * row["margins"] + ((row["hidden"] - mean) / std) @ w + bias
def objective(params, rows, mean, std, l2):
width = mean.shape[0]
loss, grad = 0.0, np.zeros_like(params)
for row in rows:
s = scores(params, row, mean, std)
s = s - s.max()
p = np.exp(s) / np.exp(s).sum()
loss -= np.log(p[row["target"]] + 1e-12)
d = p.copy()
d[row["target"]] -= 1
z = (row["hidden"] - mean) / std
grad[0] += d @ row["margins"]
grad[1:1 + width] += d @ z
grad[1 + width + TYPES.index(row["type"])] += d.sum()
n = len(rows)
loss = loss / n + l2 * np.sum(params[1:1 + width] ** 2)
grad = grad / n
grad[1:1 + width] += 2 * l2 * params[1:1 + width]
return loss, grad
def answers_for(rows, params, mean, std):
out = {}
for row in rows:
s = scores(params, row, mean, std)
p = np.exp(s - s.max())
p /= p.sum()
q = row["entry"]["question"]
if q["type"] == "noul":
out[row["entry"]["id"]] = {"type": "noul", "noul": float(p[1])}
elif q["type"] == "score":
out[row["entry"]["id"]] = {"type": "score", "score": float(p @ np.arange(len(p))),
"probabilities": {n: float(v) for n, v in zip(row["names"], p)}}
else:
out[row["entry"]["id"]] = {"type": "choice", "choice": row["names"][int(p.argmax())],
"probabilities": {n: float(v) for n, v in zip(row["names"], p)}}
return out
def metrics(rows, answers):
# Mirrors scripts/models/d1/decision-metrics.mjs.
t = dict(categorical=0, correct=0, pos=0, neg=0, tp=0, fp=0, unk=0, unk_ok=0, score_n=0, score_err=0.0, nll=0.0)
for row in rows:
entry, answer, q = row["entry"], answers[row["entry"]["id"]], row["entry"]["question"]
label = entry["label"]
if q["type"] == "score":
t["score_n"] += 1
t["score_err"] += abs(answer["score"] - label["expected_level"])
continue
t["categorical"] += 1
guess = answer["choice"] if q["type"] == "choice" else ("yes" if answer["noul"] >= 0.5 else "no")
right = guess == label["selected_option"]
t["correct"] += right
if entry.get("is_completion_claim"):
positives = entry.get("completion_positive_options", [])
if label["selected_option"] in positives:
t["pos"] += 1
t["tp"] += guess in positives
else:
t["neg"] += 1
t["fp"] += guess in positives
unknown = (entry.get("uncertainty_metadata") or {}).get("unknown_option")
if unknown and label["selected_option"] == unknown:
t["unk"] += 1
t["unk_ok"] += right
return {"categorical": f"{t['correct']}/{t['categorical']}", "categorical_accuracy": t["correct"] / max(1, t["categorical"]),
"completion_found": f"{t['tp']}/{t['pos']}",
"false_completions": f"{t['fp']}/{t['neg']}", "unknown": f"{t['unk_ok']}/{t['unk']}",
"score_mae": t["score_err"] / max(1, t["score_n"])}
def fit(rows, mean, std, l2):
width = mean.shape[0]
start = np.zeros(1 + width + len(TYPES))
start[0] = 1.0 # begin at the zero-shot verify readout
result = minimize(objective, start, args=(rows, mean, std, l2), jac=True, method="L-BFGS-B", options={"maxiter": 500})
return result.x
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--reports", type=Path, required=True)
parser.add_argument("--data", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
train = load(args.reports / "probe-train.json", args.data / "train.json")
val = load(args.reports / "probe-val.json", args.data / "validation.json")
test = load(args.reports / "probe-test.json", args.data / "final_test.json")
stacked = np.concatenate([row["hidden"] for row in train])
mean, std = stacked.mean(0), stacked.std(0) + 1e-3
width = mean.shape[0]
zero = np.zeros(1 + width + len(TYPES))
zero[0] = 1.0
report = {"rows": {"train": len(train), "validation": len(val), "test": len(test)},
"zero_shot_verify": {"validation": metrics(val, answers_for(val, zero, mean, std)),
"test": metrics(test, answers_for(test, zero, mean, std))},
"candidates": []}
best = None
for l2 in [1e-4, 1e-3, 1e-2, 3e-2, 1e-1, 3e-1, 1.0]:
params = fit(train, mean, std, l2)
val_loss, _ = objective(params, val, mean, std, 0.0)
val_metrics = metrics(val, answers_for(val, params, mean, std))
accuracy = val_metrics["categorical_accuracy"]
candidate = {"l2": l2, "validation_nll": float(val_loss), "validation": val_metrics,
"train": metrics(train, answers_for(train, params, mean, std))}
report["candidates"].append(candidate)
print(json.dumps(candidate), flush=True)
key = (accuracy, -val_loss)
if best is None or key > best[0]:
best = (key, l2, params)
_, l2, params = best
report["selected_l2"] = l2
report["test"] = metrics(test, answers_for(test, params, mean, std))
report["margin_weight"] = float(params[0])
report["type_bias"] = {t: float(params[1 + width + i]) for i, t in enumerate(TYPES)}
args.out.write_text(json.dumps(report, indent=2) + "\n")
np.savez(args.out.with_suffix(".npz"), params=params, mean=mean, std=std)
print("selected l2", l2, "test", json.dumps(report["test"]))
print("zero-shot verify test", json.dumps(report["zero_shot_verify"]["test"]))
if __name__ == "__main__":
main()