Skip to content

A Claude Code skill bundle for bug hunting and external red-team work - 82 skills, 15 slash commands, 681 disclosed-report patterns curated across 24 core vulnerability classes, plus enterprise identity + infrastructure attack matrices.

5.4 KB
Raw Blame
#!/usr/bin/env python3
"""
run_eval_ps_par.py — parallel PortSwigger eval. Same login/launch/oracle/ablation as
run_eval_ps_auto.py, but launches all instances up front then runs the agents
CONCURRENTLY (default 3 at a time) so wall-time is max-per-wave, not the sum.

  python3 eval/run_eval_ps_par.py --labs eval/ps_labs_hard.json --parallel 3
"""
import argparse
import json
import os
import sys
import time
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import run_eval as RE                 # noqa: E402
import run_eval_ps_auto as AUTO       # noqa: E402  (reuse ps_login, launch_lab, UA, TARGET_DESC, EVAL)
from oracle_portswigger import ps_solved  # noqa: E402


def do_run(lab, url, cond, model, max_turns, timeout):
    base = {"lab": lab["key"], "class": lab.get("class"), "condition": cond, "instance": url}
    pre, ok = ps_solved(url)
    if not ok:
        return {**base, "solved": None, "skipped": "unreadable"}
    if pre:
        return {**base, "solved": None, "skipped": "pre-solved"}
    r = RE.run_agent(RE.build_task(lab["objective"], url, AUTO.TARGET_DESC),
                     cond == "skills", model, max_turns, timeout)
    time.sleep(2)
    post, _ = ps_solved(url)
    return {**base, "solved": bool(post), **r}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--labs", default=os.path.join(AUTO.EVAL, "ps_labs.json"))
    ap.add_argument("--limit", type=int, default=0)
    ap.add_argument("--conditions", default="baseline,skills")
    ap.add_argument("--model", default="claude-sonnet-4-6")
    ap.add_argument("--max-turns", type=int, default=60)
    ap.add_argument("--timeout", type=int, default=600)
    ap.add_argument("--parallel", type=int, default=3)
    ap.add_argument("--out", default=os.path.join(AUTO.EVAL, "results", "ps_par.jsonl"))
    a = ap.parse_args()

    labs = json.load(open(a.labs))
    if a.limit:
        labs = labs[:a.limit]
    conds = [c.strip() for c in a.conditions.split(",") if c.strip()]
    os.makedirs(os.path.dirname(a.out), exist_ok=True)
    out = open(a.out, "a")
    rows = []

    # Phase 0: launch all instances (browser, sequential — fast)
    from playwright.sync_api import sync_playwright
    launched = []
    with sync_playwright() as p:
        b = p.chromium.launch(headless=True)
        ctx = b.new_context(user_agent=AUTO.UA, viewport={"width": 1280, "height": 900}, locale="en-US")
        pg = ctx.new_page()
        print("login...", flush=True)
        if not AUTO.ps_login(pg):
            sys.exit("login failed")
        print("  ok")
        for lab in labs:
            url = AUTO.launch_lab(pg, ctx, lab["slug"])
            print(f"  launched {lab['key']}: {url}", flush=True)
            if url:
                launched.append((lab, url))
        b.close()

    def record(rec):
        rec["ts"] = datetime.now(timezone.utc).isoformat()
        rows.append(rec)
        out.write(json.dumps(rec) + "\n"); out.flush()
        print(f"  [{rec['lab']:26s} {rec['condition']:8s}] solved={rec.get('solved')} "
              f"turns={rec.get('num_turns')} ${rec.get('cost_usd')} {rec.get('skipped','')}"
              f"{rec.get('agent_error','')}", flush=True)

    print(f"\n== parallel eval: {len(launched)} instance(s) | {conds} | par={a.parallel} | model={a.model} ==")
    base = {}
    if "baseline" in conds:
        print("-- baseline phase --", flush=True)
        with ThreadPoolExecutor(max_workers=a.parallel) as ex:
            futs = {ex.submit(do_run, lab, url, "baseline", a.model, a.max_turns, a.timeout): lab["key"]
                    for lab, url in launched}
            for f in as_completed(futs):
                rec = f.result(); record(rec); base[rec["lab"]] = rec

    if "skills" in conds:
        todo = ([(lab, url) for lab, url in launched if base.get(lab["key"], {}).get("solved") is False]
                if "baseline" in conds else launched)
        print(f"-- skills phase ({len(todo)} lab(s) where baseline failed) --", flush=True)
        with ThreadPoolExecutor(max_workers=a.parallel) as ex:
            futs = {ex.submit(do_run, lab, url, "skills", a.model, a.max_turns, a.timeout): lab["key"]
                    for lab, url in todo}
            for f in as_completed(futs):
                record(f.result())
    out.close()

    # summary
    print("\n" + "=" * 60 + "\nSUMMARY (parallel)")
    runs = [r for r in rows if r.get("solved") is not None]
    for c in conds:
        rs = [r for r in runs if r["condition"] == c]
        if rs:
            print(f"  {c:9s}: solved {sum(1 for r in rs if r['solved'])}/{len(rs)}   "
                  f"${sum((r.get('cost_usd') or 0) for r in rs):.2f}")
    print("\n  per-lab (baseline → skills):")
    for lab, _ in launched:
        lr = {r["condition"]: r for r in rows if r["lab"] == lab["key"]}
        def cell(c):
            r = lr.get(c)
            return "-" if not r else ("(skip)" if r.get("skipped") else ("✓" if r["solved"] else "✗"))
        print(f"    {lab['key']:26s} {lab.get('class',''):14s} base={cell('baseline'):>6s}  skills={cell('skills')}")
    delta = [k for k in base if base[k].get("solved") is False
             and any(r["lab"] == k and r["condition"] == "skills" and r.get("solved") for r in rows)]
    print(f"\n  SKILL DELTA (baseline ✗ → skills ✓): {len(delta)}{': ' + ', '.join(delta) if delta else ''}")
    print(f"  raw: {a.out}")


if __name__ == "__main__":
    sys.exit(main())