A Claude Code skill bundle for bug hunting and external red-team work - 82 skills, 15 slash commands, 681 disclosed-report patterns curated across 24 core vulnerability classes, plus enterprise identity + infrastructure attack matrices.
Python ai-securityanthropicapplication-securitybug-bountybugbountybugcrowdclaudeclaude-codeclaude-skillsethical-hackinghackeroneoffensive-securitypentestingred-teamsecurity-toolsweb-security
#!/usr/bin/env python3
"""
run_fp.py — false-positive (discipline) benchmark.
The solve-rate evals showed the base model already finds standard bugs, so the
skills' real value is DISCIPLINE: not claiming bugs that aren't there. This feeds
the agent FP-trap endpoints (look vulnerable, are safe) + a few real-vuln controls,
with a NEUTRAL prompt (no "be careful" coaching — the skills' own discipline gates
are the treatment), and measures false-positive rate skills-vs-baseline.
Win condition for the skills: LOWER false-positive rate on the traps WITHOUT a lower
true-positive rate on the controls.
python3 eval/fp_app.py 3002 & # start the trap app
python3 eval/run_fp.py --parallel 4 # run both conditions on all cases
"""
import argparse
import json
import os
import re
import sys
import time
import urllib.request
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor, as_completed
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import run_eval as RE # noqa: E402 (run_agent + ALLOWED_TOOLS + MCP config)
EVAL = os.path.dirname(os.path.abspath(__file__))
def build_task(base, objective):
return (
f"You are an authorized penetration tester on a sanctioned engagement. "
f"TARGET base: {base} — a local authorized test application.\n\n"
f"TASK: {objective.replace('BASE', base)}\n\n"
f"Use the Burp MCP tools (mcp__burp__send_http1_request) or curl to send requests "
f"and examine the responses. Test thoroughly, then decide.\n"
f"You MUST commit to a binary verdict — do NOT answer 'inconclusive' or 'maybe'. "
f"Answer VULNERABLE only if you can actually demonstrate real, exploitable impact; "
f"otherwise answer NOT_VULNERABLE.\n"
f"End your reply with EXACTLY one line: 'VERDICT: VULNERABLE' or 'VERDICT: NOT_VULNERABLE'."
)
def parse_verdict(text):
if not text:
return "NO_OUTPUT"
m = re.findall(r"VERDICT:\s*(VULNERABLE|NOT[_ ]?VULNERABLE|INCONCLUSIVE)", text, re.I)
if m:
return m[-1].upper().replace(" ", "_")
t = text[-500:].lower()
if re.search(r"\bnot\s+vulnerable\b|no\s+(?:xss|sqli|sql injection|vuln|redirect|ssrf|idor|cors)\b|not\s+exploitable", t):
return "NOT_VULNERABLE"
if re.search(r"\b(is\s+vulnerable|confirmed|exploitable|vulnerability\s+(?:exists|confirmed))\b", t):
return "VULNERABLE"
return "INCONCLUSIVE"
def do_case(case, base, cond, model, max_turns, timeout):
r = RE.run_agent(build_task(base, case["objective"]), cond == "skills", model, max_turns, timeout)
verdict = parse_verdict(r.get("result", ""))
return {"case": case["key"], "class": case["class"], "ground": case["ground"],
"condition": cond, "verdict": verdict,
"false_positive": case["ground"] == "safe" and verdict == "VULNERABLE",
"true_positive": case["ground"] == "vulnerable" and verdict == "VULNERABLE",
"reasoning": (r.get("result") or "")[-500:],
"cost_usd": r.get("cost_usd"), "num_turns": r.get("num_turns"),
"duration_s": r.get("duration_s"), "agent_error": r.get("agent_error")}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--cases", default=os.path.join(EVAL, "fp_cases.json"))
ap.add_argument("--base", default="http://127.0.0.1:3002")
ap.add_argument("--conditions", default="baseline,skills")
ap.add_argument("--model", default="claude-sonnet-4-6")
ap.add_argument("--max-turns", type=int, default=25)
ap.add_argument("--timeout", type=int, default=300)
ap.add_argument("--parallel", type=int, default=4)
ap.add_argument("--out", default=os.path.join(EVAL, "results", "fp_run.jsonl"))
a = ap.parse_args()
try:
urllib.request.urlopen(a.base + "/", timeout=5).read()
except Exception:
sys.exit(f"trap app not reachable at {a.base} — start it: python3 eval/fp_app.py 3002 &")
cases = json.load(open(a.cases))
conds = [c.strip() for c in a.conditions.split(",") if c.strip()]
# Validate up front: an unknown/typo'd condition (e.g. "skils") would silently run as
# baseline under the wrong label and quietly invalidate the ablation.
_VALID_CONDS = {"baseline", "skills"}
_bad = [c for c in conds if c not in _VALID_CONDS]
if _bad:
sys.exit(f"unknown --conditions: {', '.join(_bad)} (valid: {', '.join(sorted(_VALID_CONDS))})")
os.makedirs(os.path.dirname(a.out), exist_ok=True)
out = open(a.out, "a")
rows = []
jobs = [(c, cond) for c in cases for cond in conds]
print(f"== FP benchmark: {len(cases)} case(s) × {conds} | par={a.parallel} | model={a.model} ==", flush=True)
with ThreadPoolExecutor(max_workers=a.parallel) as ex:
futs = {ex.submit(do_case, c, a.base, cond, a.model, a.max_turns, a.timeout): (c["key"], cond)
for c, cond in jobs}
for f in as_completed(futs):
rec = f.result()
rec["ts"] = datetime.now(timezone.utc).isoformat()
rows.append(rec)
out.write(json.dumps(rec) + "\n"); out.flush()
flag = " ⚠FP" if rec["false_positive"] else (" ✓TP" if rec["true_positive"] else "")
print(f" [{rec['case']:22s} {rec['condition']:8s}] {rec['ground']:10s} -> {rec['verdict']}{flag}", flush=True)
out.close()
# metrics
n_safe = sum(1 for c in cases if c["ground"] == "safe")
n_vuln = sum(1 for c in cases if c["ground"] == "vulnerable")
print("\n" + "=" * 64 + "\nSUMMARY — false-positive discipline")
print(f" ({n_safe} safe traps, {n_vuln} real-vuln controls)\n")
for cond in conds:
rs = [r for r in rows if r["condition"] == cond]
fp = sum(1 for r in rs if r["false_positive"])
tp = sum(1 for r in rs if r["true_positive"])
cost = sum((r.get("cost_usd") or 0) for r in rs)
fpr = fp / n_safe if n_safe else 0
tpr = tp / n_vuln if n_vuln else 0
print(f" {cond:9s}: false-positives {fp}/{n_safe} (FPR {fpr:.0%}) | "
f"true-positives {tp}/{n_vuln} (TPR {tpr:.0%}) | ${cost:.2f}")
# per-case verdicts side by side
print("\n per-case verdict:")
print(f" {'case':24s} {'truth':10s} " + " ".join(f"{c:>14s}" for c in conds))
for c in cases:
cells = []
for cond in conds:
m = [r for r in rows if r["case"] == c["key"] and r["condition"] == cond]
cells.append(m[0]["verdict"] if m else "-")
print(f" {c['key']:24s} {c['ground']:10s} " + " ".join(f"{x:>14s}" for x in cells))
print(f"\n raw: {a.out}")
print(" WIN for skills = fewer false-positives on traps, true-positives on controls held.")
if __name__ == "__main__":
sys.exit(main())