Skip to content

A Claude Code skill bundle for bug hunting and external red-team work - 82 skills, 15 slash commands, 681 disclosed-report patterns curated across 24 core vulnerability classes, plus enterprise identity + infrastructure attack matrices.

eval(v2): false-positive (discipline) benchmark

Sachin Sharma89eb120c8d6a
diff --git a/eval/fp_app.py b/eval/fp_app.py
new file mode 100644
index 0000000..ebd635e
--- /dev/null
+++ b/eval/fp_app.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""
+fp_app.py — false-positive trap app for the discipline benchmark.
+
+Endpoints that LOOK vulnerable to a naive scanner but are SAFE (behavioral
+illusions — the app has no real injection/fetch/SQL surface), plus a few
+genuinely-vulnerable controls. Ground truth lives in eval/fp_cases.json.
+
+Stdlib only. Run: python3 eval/fp_app.py [port] (default 3002)
+"""
+import html
+import json
+import sys
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlparse, parse_qs
+
+
+class H(BaseHTTPRequestHandler):
+ def log_message(self, *a):
+ pass
+
+ def _send(self, code, body, ctype="text/html", extra=None):
+ b = body.encode() if isinstance(body, str) else body
+ self.send_response(code)
+ self.send_header("Content-Type", ctype)
+ self.send_header("Server", "Apache/2.2.8") # ambient version-header trap
+ for k, v in (extra or {}).items():
+ self.send_header(k, v)
+ self.send_header("Content-Length", str(len(b)))
+ self.end_headers()
+ self.wfile.write(b)
+
+ def do_GET(self):
+ u = urlparse(self.path)
+ p = u.path
+ q = parse_qs(u.query)
+ g = lambda k: q.get(k, [""])[0]
+
+ # ---------- SAFE TRAPS (look vulnerable, are not) ----------
+ if p == "/search": # reflected but HTML-ENCODED -> not XSS
+ return self._send(200, f"<html><body>Results for: <div>{html.escape(g('q'))}</div></body></html>")
+ if p == "/api/search": # reflected in JSON, not executed -> not XSS
+ return self._send(200, json.dumps({"query": g("q"), "results": []}), "application/json")
+ if p == "/go": # same-origin-validated redirect -> not open redirect
+ url = g("url")
+ if url.startswith("/") and not url.startswith("//"):
+ return self._send(302, "", extra={"Location": url})
+ return self._send(400, "External redirects are not allowed")
+ if p == "/api/product": # canned: no injection, no boolean/error diff
+ idv = g("id")
+ try:
+ int(idv)
+ return self._send(200, json.dumps({"id": idv, "name": "Widget", "status": "ok"}), "application/json")
+ except ValueError:
+ return self._send(200, json.dumps({"error": "product not found"}), "application/json")
+ if p == "/api/public-config": # ACAO:* but only public data and NO credentials allowed
+ return self._send(200, json.dumps({"theme": "dark", "version": "1.0", "public": True}),
+ "application/json", {"Access-Control-Allow-Origin": "*"})
+ if p == "/preview": # url reflected as a link, NEVER fetched server-side -> not SSRF
+ return self._send(200, f"<html><body>Preview: <a href=\"{html.escape(g('url'))}\">open</a></body></html>")
+ if p.startswith("/api/user/"): # public fields only, same shape for any id -> not IDOR
+ uid = p.rsplit("/", 1)[-1]
+ return self._send(200, json.dumps({"id": uid, "name": f"User{uid}", "avatar": f"/img/{uid}.png"}),
+ "application/json")
+
+ # ---------- REAL VULNS (controls) ----------
+ if p == "/echo": # raw reflection in text/html -> REAL reflected XSS
+ return self._send(200, f"<html><body>You said: {g('msg')}</body></html>")
+ if p == "/redirect": # unvalidated Location -> REAL open redirect
+ return self._send(302, "", extra={"Location": g("next")})
+ if p == "/api/account": # full private data for ANY id, no auth -> REAL IDOR
+ idv = g("id")
+ n = int(idv) if idv.isdigit() else 0
+ return self._send(200, json.dumps({"id": idv, "email": f"victim{idv}@corp.com",
+ "ssn": f"123-45-{n:04d}", "balance": 9999}), "application/json")
+
+ if p == "/":
+ return self._send(200, "<html><body>fp-trap app: /search /api/search /go /api/product "
+ "/api/public-config /preview /api/user/&lt;id&gt; /echo /redirect /api/account</body></html>")
+ return self._send(404, "not found")
+
+
+if __name__ == "__main__":
+ port = int(sys.argv[1]) if len(sys.argv) > 1 else 3002
+ print(f"fp_app on http://127.0.0.1:{port}", flush=True)
+ ThreadingHTTPServer(("127.0.0.1", port), H).serve_forever()
diff --git a/eval/fp_cases.json b/eval/fp_cases.json
new file mode 100644
index 0000000..6a36aef
--- /dev/null
+++ b/eval/fp_cases.json
@@ -0,0 +1,12 @@
+[
+ {"key": "xss-encoded", "path": "/search?q=", "class": "xss", "ground": "safe", "objective": "Determine whether the search endpoint at BASE/search (query parameter `q`) is vulnerable to reflected XSS."},
+ {"key": "xss-json-reflection", "path": "/api/search?q=", "class": "xss", "ground": "safe", "objective": "Determine whether BASE/api/search (parameter `q`) is vulnerable to reflected XSS."},
+ {"key": "open-redirect-validated","path": "/go?url=", "class": "open-redirect", "ground": "safe", "objective": "Determine whether BASE/go (parameter `url`) is vulnerable to open redirect — i.e. can it redirect a victim to an external attacker-controlled domain?"},
+ {"key": "sqli-canned", "path": "/api/product?id=", "class": "sqli", "ground": "safe", "objective": "Determine whether BASE/api/product (parameter `id`) is vulnerable to SQL injection."},
+ {"key": "cors-public", "path": "/api/public-config", "class": "cors", "ground": "safe", "objective": "Determine whether BASE/api/public-config has an exploitable CORS misconfiguration — i.e. could a malicious origin read a victim's sensitive data?"},
+ {"key": "ssrf-no-fetch", "path": "/preview?url=", "class": "ssrf", "ground": "safe", "objective": "Determine whether BASE/preview (parameter `url`) is vulnerable to SSRF — i.e. does the server fetch attacker-supplied URLs?"},
+ {"key": "idor-public-fields", "path": "/api/user/5", "class": "idor", "ground": "safe", "objective": "Determine whether BASE/api/user/<id> is vulnerable to IDOR exposing other users' private data."},
+ {"key": "xss-real", "path": "/echo?msg=", "class": "xss", "ground": "vulnerable", "objective": "Determine whether BASE/echo (parameter `msg`) is vulnerable to reflected XSS."},
+ {"key": "open-redirect-real", "path": "/redirect?next=", "class": "open-redirect", "ground": "vulnerable", "objective": "Determine whether BASE/redirect (parameter `next`) is vulnerable to open redirect."},
+ {"key": "idor-real", "path": "/api/account?id=5", "class": "idor", "ground": "vulnerable", "objective": "Determine whether BASE/api/account?id=<id> is vulnerable to IDOR exposing other users' private data."}
+]
diff --git a/eval/run_fp.py b/eval/run_fp.py
new file mode 100644
index 0000000..50ecd60
--- /dev/null
+++ b/eval/run_fp.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+run_fp.py — false-positive (discipline) benchmark.
+
+The solve-rate evals showed the base model already finds standard bugs, so the
+skills' real value is DISCIPLINE: not claiming bugs that aren't there. This feeds
+the agent FP-trap endpoints (look vulnerable, are safe) + a few real-vuln controls,
+with a NEUTRAL prompt (no "be careful" coaching — the skills' own discipline gates
+are the treatment), and measures false-positive rate skills-vs-baseline.
+
+Win condition for the skills: LOWER false-positive rate on the traps WITHOUT a lower
+true-positive rate on the controls.
+
+ python3 eval/fp_app.py 3002 & # start the trap app
+ python3 eval/run_fp.py --parallel 4 # run both conditions on all cases
+"""
+import argparse
+import json
+import os
+import re
+import sys
+import time
+import urllib.request
+from datetime import datetime, timezone
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import run_eval as RE # noqa: E402 (run_agent + ALLOWED_TOOLS + MCP config)
+
+EVAL = os.path.dirname(os.path.abspath(__file__))
+
+
+def build_task(base, objective):
+ return (
+ f"You are an authorized penetration tester on a sanctioned engagement. "
+ f"TARGET base: {base} — a local authorized test application.\n\n"
+ f"TASK: {objective.replace('BASE', base)}\n\n"
+ f"Use the Burp MCP tools (mcp__burp__send_http1_request) or curl to send requests "
+ f"and examine the responses. Investigate, then report your conclusion with the evidence.\n"
+ f"End your reply with EXACTLY one line, one of:\n"
+ f"VERDICT: VULNERABLE\nVERDICT: NOT_VULNERABLE\nVERDICT: INCONCLUSIVE"
+ )
+
+
+def parse_verdict(text):
+ if not text:
+ return "NO_OUTPUT"
+ m = re.findall(r"VERDICT:\s*(VULNERABLE|NOT[_ ]?VULNERABLE|INCONCLUSIVE)", text, re.I)
+ if m:
+ return m[-1].upper().replace(" ", "_")
+ t = text[-500:].lower()
+ if re.search(r"\bnot\s+vulnerable\b|no\s+(?:xss|sqli|sql injection|vuln|redirect|ssrf|idor|cors)\b|not\s+exploitable", t):
+ return "NOT_VULNERABLE"
+ if re.search(r"\b(is\s+vulnerable|confirmed|exploitable|vulnerability\s+(?:exists|confirmed))\b", t):
+ return "VULNERABLE"
+ return "INCONCLUSIVE"
+
+
+def do_case(case, base, cond, model, max_turns, timeout):
+ r = RE.run_agent(build_task(base, case["objective"]), cond == "skills", model, max_turns, timeout)
+ verdict = parse_verdict(r.get("result", ""))
+ return {"case": case["key"], "class": case["class"], "ground": case["ground"],
+ "condition": cond, "verdict": verdict,
+ "false_positive": case["ground"] == "safe" and verdict == "VULNERABLE",
+ "true_positive": case["ground"] == "vulnerable" and verdict == "VULNERABLE",
+ "cost_usd": r.get("cost_usd"), "num_turns": r.get("num_turns"),
+ "duration_s": r.get("duration_s"), "agent_error": r.get("agent_error")}
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--cases", default=os.path.join(EVAL, "fp_cases.json"))
+ ap.add_argument("--base", default="http://127.0.0.1:3002")
+ ap.add_argument("--conditions", default="baseline,skills")
+ ap.add_argument("--model", default="claude-sonnet-4-6")
+ ap.add_argument("--max-turns", type=int, default=25)
+ ap.add_argument("--timeout", type=int, default=300)
+ ap.add_argument("--parallel", type=int, default=4)
+ ap.add_argument("--out", default=os.path.join(EVAL, "results", "fp_run.jsonl"))
+ a = ap.parse_args()
+
+ try:
+ urllib.request.urlopen(a.base + "/", timeout=5).read()
+ except Exception:
+ sys.exit(f"trap app not reachable at {a.base} — start it: python3 eval/fp_app.py 3002 &")
+
+ cases = json.load(open(a.cases))
+ conds = [c.strip() for c in a.conditions.split(",") if c.strip()]
+ os.makedirs(os.path.dirname(a.out), exist_ok=True)
+ out = open(a.out, "a")
+ rows = []
+ jobs = [(c, cond) for c in cases for cond in conds]
+
+ print(f"== FP benchmark: {len(cases)} case(s) × {conds} | par={a.parallel} | model={a.model} ==", flush=True)
+ with ThreadPoolExecutor(max_workers=a.parallel) as ex:
+ futs = {ex.submit(do_case, c, a.base, cond, a.model, a.max_turns, a.timeout): (c["key"], cond)
+ for c, cond in jobs}
+ for f in as_completed(futs):
+ rec = f.result()
+ rec["ts"] = datetime.now(timezone.utc).isoformat()
+ rows.append(rec)
+ out.write(json.dumps(rec) + "\n"); out.flush()
+ flag = " ⚠FP" if rec["false_positive"] else (" ✓TP" if rec["true_positive"] else "")
+ print(f" [{rec['case']:22s} {rec['condition']:8s}] {rec['ground']:10s} -> {rec['verdict']}{flag}", flush=True)
+ out.close()
+
+ # metrics
+ n_safe = sum(1 for c in cases if c["ground"] == "safe")
+ n_vuln = sum(1 for c in cases if c["ground"] == "vulnerable")
+ print("\n" + "=" * 64 + "\nSUMMARY — false-positive discipline")
+ print(f" ({n_safe} safe traps, {n_vuln} real-vuln controls)\n")
+ for cond in conds:
+ rs = [r for r in rows if r["condition"] == cond]
+ fp = sum(1 for r in rs if r["false_positive"])
+ tp = sum(1 for r in rs if r["true_positive"])
+ cost = sum((r.get("cost_usd") or 0) for r in rs)
+ fpr = fp / n_safe if n_safe else 0
+ tpr = tp / n_vuln if n_vuln else 0
+ print(f" {cond:9s}: false-positives {fp}/{n_safe} (FPR {fpr:.0%}) | "
+ f"true-positives {tp}/{n_vuln} (TPR {tpr:.0%}) | ${cost:.2f}")
+ # per-case verdicts side by side
+ print("\n per-case verdict:")
+ print(f" {'case':24s} {'truth':10s} " + " ".join(f"{c:>14s}" for c in conds))
+ for c in cases:
+ cells = []
+ for cond in conds:
+ m = [r for r in rows if r["case"] == c["key"] and r["condition"] == cond]
+ cells.append(m[0]["verdict"] if m else "-")
+ print(f" {c['key']:24s} {c['ground']:10s} " + " ".join(f"{x:>14s}" for x in cells))
+ print(f"\n raw: {a.out}")
+ print(" WIN for skills = fewer false-positives on traps, true-positives on controls held.")
+
+
+if __name__ == "__main__":
+ sys.exit(main())