Skip to content

A Claude Code skill bundle for bug hunting and external red-team work - 82 skills, 15 slash commands, 681 disclosed-report patterns curated across 24 core vulnerability classes, plus enterprise identity + infrastructure attack matrices.

8.1 KB
Raw Blame
#!/usr/bin/env python3
"""
run_eval_ps_auto.py — PortSwigger eval with automated launch (Playwright).

Logs into portswigger.net once (real browser — the login submit is JS-mediated),
then per lab: launches a fresh instance, captures its URL, and runs the agent.

Ablation under the no-in-place-reset constraint: a PortSwigger lab can't be reset
once solved, so we run BASELINE (skills off) first; if it leaves the instance
UNSOLVED we then run SKILLS on the same still-clean instance. This needs only one
launch per lab and gives the most informative signal:
  - baseline solved            -> easy/memorized, skills delta <= 0 on this lab
  - baseline failed, skills solved -> THE skill delta (skills added the solve)
  - both failed                -> capability gap

Creds: env PS_EMAIL / PS_PASSWORD, else eval/.ps_creds (gitignored), else /tmp/.ps_creds.
Requires: playwright (+ chromium), Burp running (agent's hands), claude CLI authed.

  python3 eval/run_eval_ps_auto.py --limit 4
  python3 eval/run_eval_ps_auto.py --model claude-opus-4-8 --conditions baseline,skills
"""
import argparse
import json
import os
import re
import sys
import time
from datetime import datetime, timezone

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import run_eval as RE                       # noqa: E402
from oracle_portswigger import ps_solved    # noqa: E402

EVAL = os.path.dirname(os.path.abspath(__file__))
TARGET_DESC = ("a PortSwigger Web Security Academy lab instance — an authorized, "
               "deliberately-vulnerable training lab tied to the tester's own account.")
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
INSTANCE_RE = re.compile(r"https://[0-9a-f]+\.web-security-academy\.net")


def load_creds():
    e, p = os.environ.get("PS_EMAIL"), os.environ.get("PS_PASSWORD")
    if e and p:
        return e, p
    for path in (os.path.join(EVAL, ".ps_creds"), "/tmp/.ps_creds"):
        if os.path.isfile(path):
            ls = [x.strip() for x in open(path).read().splitlines() if x.strip()]
            if len(ls) >= 2:
                return ls[0], ls[1]
    sys.exit("no creds: set PS_EMAIL/PS_PASSWORD or eval/.ps_creds")


def ps_login(page):
    page.goto("https://portswigger.net/users", wait_until="networkidle", timeout=45000)
    email, pw = load_creds()
    page.fill("#EmailAddress", email)
    page.fill("#Password", pw)
    page.click("#Login", timeout=10000)
    try:
        page.wait_for_load_state("networkidle", timeout=30000)
    except Exception:
        pass
    page.wait_for_timeout(2000)
    body = page.inner_text("body").lower()
    return ("log out" in body) or ("youraccount" in page.url)


def launch_lab(page, ctx, slug):
    """Click 'Access the lab', return the instance base URL (or None)."""
    page.goto("https://portswigger.net" + slug, wait_until="domcontentloaded", timeout=45000)
    page.wait_for_timeout(1500)
    try:
        with ctx.expect_page(timeout=30000) as np:
            page.get_by_text(re.compile("access the lab", re.I)).first.click(timeout=20000)
        inst = np.value
        inst.wait_for_load_state("domcontentloaded", timeout=40000)
        m = INSTANCE_RE.match(inst.url)
        url = m.group(0) if m else None
        inst.close()
        return url
    except Exception:
        # same-tab fallback
        try:
            page.wait_for_url(INSTANCE_RE, timeout=10000)
            m = INSTANCE_RE.match(page.url)
            return m.group(0) if m else None
        except Exception:
            return None


def run_one(lab, url, conditions, model, max_turns, timeout):
    """Baseline-first; skills only if still unsolved. Returns list of records."""
    recs = []
    base_task = RE.build_task(lab["objective"], url, TARGET_DESC)
    order = [c for c in ("baseline", "skills") if c in conditions]
    solved_already = False
    for cond in order:
        if cond == "skills" and solved_already:
            print(f"    skills: skipped (baseline already solved → delta ≤ 0)")
            recs.append({"condition": "skills", "solved": None, "skipped": "baseline-solved"})
            continue
        pre, ok = ps_solved(url)
        if not ok:
            print(f"    {cond}: instance unreadable/expired — skip"); continue
        if pre:
            print(f"    {cond}: already solved before run — skip"); continue
        print(f"    {cond}: running agent...", flush=True)
        r = RE.run_agent(base_task, cond == "skills", model, max_turns, timeout)
        time.sleep(3)
        post, _ = ps_solved(url)
        solved = bool(post)
        if solved:
            solved_already = True
        print(f"      -> solved={solved} turns={r.get('num_turns')} cost=${r.get('cost_usd')} {r.get('duration_s')}s"
              f"{'  [' + r['agent_error'] + ']' if r.get('agent_error') else ''}")
        recs.append({"condition": cond, "solved": solved, **r})
    return recs


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--labs", default=os.path.join(EVAL, "ps_labs.json"))
    ap.add_argument("--limit", type=int, default=0)
    ap.add_argument("--conditions", default="baseline,skills")
    ap.add_argument("--model", default="claude-sonnet-4-6")
    ap.add_argument("--max-turns", type=int, default=55)
    ap.add_argument("--timeout", type=int, default=600)
    ap.add_argument("--out", default=os.path.join(EVAL, "results", "ps_auto.jsonl"))
    a = ap.parse_args()

    from playwright.sync_api import sync_playwright
    labs = json.load(open(a.labs))
    if a.limit:
        labs = labs[:a.limit]
    conditions = [c.strip() for c in a.conditions.split(",") if c.strip()]
    os.makedirs(os.path.dirname(a.out), exist_ok=True)
    out = open(a.out, "a")
    rows = []

    with sync_playwright() as p:
        b = p.chromium.launch(headless=True)
        ctx = b.new_context(user_agent=UA, viewport={"width": 1280, "height": 900}, locale="en-US")
        page = ctx.new_page()
        print("logging in to portswigger.net ...", flush=True)
        if not ps_login(page):
            sys.exit("login failed")
        print("  login ok\n")
        print(f"== PortSwigger AUTO eval: {len(labs)} lab(s) × {conditions} | model={a.model} ==")
        for lab in labs:
            print(f"\n[{lab['key']}] launching ...", flush=True)
            url = launch_lab(page, ctx, lab["slug"])
            if not url:
                print("  launch FAILED — skip"); continue
            print(f"  instance: {url}")
            for rec in run_one(lab, url, conditions, a.model, a.max_turns, a.timeout):
                full = {"ts": datetime.now(timezone.utc).isoformat(), "lab": lab["key"],
                        "class": lab.get("class"), "instance": url, **rec}
                rows.append(full)
                out.write(json.dumps(full) + "\n"); out.flush()
        b.close()
    out.close()

    # summary
    print("\n" + "=" * 62 + "\nSUMMARY (PortSwigger auto)")
    runs = [r for r in rows if r.get("solved") is not None and not r.get("skipped")]
    for cond in conditions:
        rs = [r for r in runs if r["condition"] == cond]
        if rs:
            s = sum(1 for r in rs if r["solved"])
            cost = sum((r.get("cost_usd") or 0) for r in rs)
            print(f"  {cond:9s}: solved {s}/{len(rs)}   ${cost:.2f}")
    # the money table: per lab, baseline vs skills
    print("\n  per-lab (baseline → skills):")
    for lab in labs:
        lr = {r["condition"]: r for r in rows if r["lab"] == lab["key"]}
        def cell(c):
            r = lr.get(c)
            if not r: return "-"
            if r.get("skipped"): return "(skip:base-solved)"
            return "✓" if r["solved"] else "✗"
        print(f"    {lab['key']:24s} {lab.get('class',''):14s} base={cell('baseline'):>4s}  skills={cell('skills')}")
    # skill delta = labs where baseline failed but skills solved
    delta = [r['lab'] for r in rows if r['condition'] == 'skills' and r.get('solved') is True
             and any(x['lab'] == r['lab'] and x['condition'] == 'baseline' and x['solved'] is False for x in rows)]
    print(f"\n  SKILL DELTA (baseline ✗ → skills ✓): {len(delta)} lab(s){': ' + ', '.join(delta) if delta else ''}")
    print(f"  raw: {a.out}")


if __name__ == "__main__":
    sys.exit(main())