The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
fix(sanitizer): integrate unreadable audit with portable filename quoting
haelyraa4bf7a45ca5a
diff --cc agents/opensource-sanitizer.mdindex a0d538508,1bb619c2a..36688a87a--- a/agents/opensource-sanitizer.md+++ b/agents/opensource-sanitizer.md@@@ -136,6 -139,180 +139,182 @@@ git log --oneline | wc -git log -p | grep -iE '(password|secret|api.?key|token)' | head -20```+ ### Step 7: Unreadable Files (WARNING)++ Steps 1-3 only match text, so a credential sitting in a spreadsheet, a PDF or an embedded database+ is invisible to them. This pass lists the files that don't read as text, for a human to open. It+ does not extract or scan their contents.++ The names of these files are untrusted, so this pass keeps them out of its stdout and out of its+ report section. The block writes one line per path to a manifest beside the project directory,+ `<project-dir>.UNREADABLE_FILES.txt`, where no scan step reaches it, and prints only a tally by+ type and a count. A person opens that file. Do not read it or copy anything from it.++ Run it as one shell call from the project directory (`cd PROJECT_DIR`, as in Step 6). The outer+ part is plain sh and the two find passes call bash. The last line it prints is always+ `Unreadable count:`. A number means the walk finished. `unknown`, or no such line at all+ (a timeout, a killed run), means the scan is incomplete: report the count as unknown, never as+ zero. A few thousand files can take a minute or more, so if the call times out, re-run it with a+ longer timeout. A non-zero or unknown count means at least PASS WITH WARNINGS. Run by the+ pipeline, it also stops until a person confirms; run on its own, it just reports the section.++ ```bash+ manifest=$(pwd).UNREADABLE_FILES.txt+ tab=$(printf "\t")++ # Classify by content, never by extension.+ # 1. First-byte signature: PK (docx/xlsx/pptx/odt/jar), OLE (legacy doc/xls), PDF, gzip, bzip2, zstd, 7z.+ # 2. Whole-file test with tr | cmp (grep -q exits on the first printable byte): a NUL or a control+ # byte other than tab, LF and CR is not plain text. Bytes 0x80-0xFF pass, since UTF-8 uses them.+ # This is the floor when file(1) is absent.+ # 3. file(1) only widens, against an opaque-binary allowlist (not "not text/*", which would flag+ # JSON and JavaScript), and fails closed on an error or on output that is not a mime type.+ # Over-listing (ANSI logs, form feeds, UTF-16) is the safe direction.+ #+ # The walk prints "label<TAB>path" records, "PROBLEM<TAB>reason" when the scan itself went wrong,+ # and END as its last record. A walk that dies before END gives an unknown count, not a short one.+ # LC_ALL=C printf %q keeps each path to one ASCII line, so a newline, a right-to-left override or a+ # zero-width name cannot pass for a second entry.+ step7_walk() {+ for t in dd od tr cmp; do+ command -v "$t" >/dev/null 2>&1 || printf "PROBLEM\t%s is missing, so files cannot be classified\n" "$t"+ done++ # Pass 1: files.+ find . \( -name .git -o -name node_modules -o -name __pycache__ \) -prune -o -type f -exec bash -c '++ export LC_ALL=C+ shape="^[A-Za-z0-9._+-]+/[A-Za-z0-9._+-]+; charset=[A-Za-z0-9._-]+\$"+ for f do+ [ -s "$f" ] || continue+ q=$(LC_ALL=C printf %q "$f")+ if ! [ -r "$f" ]; then+ printf "file not readable\t%s\n" "$q"+ continue+ fi+ label= ctl=+ sig=$(set -o pipefail; LC_ALL=C dd if="$f" bs=8 count=1 2>/dev/null | od -An -tx1 | tr -d " \n") || sig=+ case $sig in+ 504b0304*|504b0506*|504b0708*) label="ZIP/OOXML container (PK: docx, xlsx, pptx, odt, jar)";;+ d0cf11e0a1b11ae1*) label="OLE compound document (legacy doc/xls/ppt/msi)";;+ 25504446*) label="PDF";;+ 1f8b*) label="gzip";;+ 425a68*) label="bzip2";;+ 28b52ffd*) label="zstd";;+ 377abcaf271c*) label="7z";;+ esac+ LC_ALL=C tr -d "\000-\010\013\014\016-\037\177" < "$f" | cmp -s - "$f" || ctl=1+ if [ -z "$label" ] && [ -z "$sig" ]; then+ label="detector error, unverified"+ elif [ -z "$label" ] && [ -z "$ctl" ] && [ "${#sig}" -le 2 ]; then+ continue # one clean byte is text; file(1) calls every one-byte file binary+ elif [ -z "$label" ] && command -v file >/dev/null 2>&1; then+ out=$(file -b --mime-type --mime-encoding -- "$f" 2>/dev/null); rc=$?+ if [ "$rc" -ne 0 ] || ! [[ $out =~ $shape ]]; then+ [ -n "$ctl" ] || label="detector error, unverified"+ else+ mime=${out%%;*}+ case $mime in+ image/svg+xml) ;;+ image/*|audio/*|video/*|font/*|\+ application/pdf|application/zip|application/gzip|application/x-tar|\+ application/x-bzip2|application/x-xz|application/zstd|application/x-7z-compressed|\+ application/vnd.rar|application/x-sqlite3|application/vnd.sqlite3|\+ application/x-executable|application/x-pie-executable|application/x-sharedlib|\+ application/x-mach-binary|application/x-dosexec|application/wasm|\+ application/vnd.microsoft.portable-executable|\+ application/vnd.openxmlformats-officedocument.*|\+ application/vnd.oasis.opendocument.*|application/msword|\+ application/vnd.ms-excel|application/vnd.ms-powerpoint|\+ application/x-ole-storage|application/vnd.ms-office|application/octet-stream)+ label=$mime ;;+ esac+ if [ -z "$label" ]; then+ case ${out##*charset=} in+ binary) label="binary (file mime-encoding)" ;;+ unknown-8bit) label="unknown 8-bit encoding, unverified" ;;+ esac+ fi+ fi+ fi+ if [ -z "$label" ]; then+ [ -n "$ctl" ] || continue+ label="contains NUL or control bytes (not plain text)"+ fi+ printf "%s\t%s\n" "$label" "$q"+ done' _ {} + 2>/dev/null ||+ printf "PROBLEM\tthe file walk reported an error, so some files may not have been checked\n"++ # Pass 2: directories the walk cannot enter (an r/x test, since find error text differs between+ # GNU find, bfs and busybox).+ find . \( -name .git -o -name node_modules -o -name __pycache__ \) -prune -o -type d -exec bash -c '++ export LC_ALL=C+ for d do+ { [ -r "$d" ] && [ -x "$d" ]; } ||+ printf "directory not readable, contents unscanned\t%s\n" "$(LC_ALL=C printf %q "$d")"+ done' _ {} + 2>/dev/null ||+ printf "PROBLEM\tthe directory walk reported an error, so some directories may not have been checked\n"++ printf "END\tof walk\n"+ }++ # Paths go to the manifest, never to stdout. It sits beside the project, not inside it, so a+ # re-run of Steps 1-3 cannot grep it. Nothing here depends on a temp file or on sort: a failed+ # write or a scan problem turns the count into "unknown" instead of a short list.+ rm -f "$manifest" 2>/dev/null+ step7_walk | {+ n=0; problems=0; werr=; ended=+ while IFS="$tab" read -r label q; do+ if [ "$label" = PROBLEM ]; then+ problems=$((problems + 1)); echo "Scan problem: $q"+ elif [ "$label" = END ]; then+ ended=1+ else+ n=$((n + 1))+ { printf "%s\t%s\n" "$q" "$label" >> "$manifest"; } 2>/dev/null || werr=1+ fi+ done+ if [ -n "$werr" ]; then+ problems=$((problems + 1)); echo "Scan problem: could not write every path to the manifest"+ fi+ if [ -z "$ended" ]; then+ problems=$((problems + 1)); echo "Scan problem: the walk ended early, so some files were not checked"+ fi+ # Tally by type, from the second column only. It needs cut, sort and uniq; the count does not.+ if [ "$n" -gt 0 ]; then+ tally=$({ cut -f2 "$manifest" | LC_ALL=C sort | uniq -c; } 2>/dev/null)+ if [ -n "$tally" ]; then+ printf "%s\n" "$tally"+ else+ echo "No tally: cut, sort or uniq is missing. The count is unaffected."+ fi+ fi+ if [ "$problems" -gt 0 ]; then+ echo "Unreadable count: unknown ($n listed, $problems scan problems)"+ else+ echo "Unreadable count: $n"+ fi+ }+ ```++ Each line of the manifest is a path quoted with `printf %q`, a tab, and the reason, so an odd+ name shows up as `$'...'` and cannot pass for two entries. Every path there needs a person to+ open the file and say what is in it — do not infer from the extension or the mime type. A count of+ zero is not proof of a clean repo.++ Not listed, so not covered by this pass:++ - `.git`, `node_modules` and `__pycache__` at any depth (`-name` matches a basename, files included),+ content reachable only through a symlink, and zero-byte files. Git history is out of scope too:+ Step 6's `git log -p` shows a removed binary only as `Binary files differ`.+ - SVG. It is text, so Steps 1-3 scan it.+ - What Steps 1-3 and Step 6 print. They don't consult this list, so a match in a listed file with+ no NUL byte still prints its line, and grep names a binary file that matches.+ - Payloads stored as printable text: base64 in an `.eml`, a Git LFS pointer, or a hand-built file+ with no NUL, control byte or known signature. file(1) catches part of that last kind, when it+ reports an unknown 8-bit encoding, and nothing does when it is absent. They read as text, and the+ Steps 1-3 patterns don't match them either. A minified bundle is scanned by Step 1, but a+ high-entropy value in the middle of its one long line can still slip the line-anchored entropy check.+## Output FormatGenerate `SANITIZATION_REPORT.md` in the project directory:diff --cc manifests/context-packs/[email protected]index 2de007665,469c40a57..152abae64--- a/manifests/context-packs/[email protected]+++ b/manifests/context-packs/[email protected]@@@ -1,288 -1,1 +1,288 @@@-{"coverage":{"skills":293,"withTriggers":32},"generatedAt":"2026-09-24T23:51:22.784Z","id":"skill-triggers@1","model":{"effort":null,"id":"hand-seeded","source":"manual-curation-pending-regeneration"},"registryDigest":"f3a1ed9bcdffb71f0d10274d375cb002cb9e21f0cf5b1f4de1fa0cf814032deb","schemaVersion":1,"triggers":{"skill:api-connector-builder":["add api integration","new provider connector","match existing integration pattern"],"skill:api-design":["rest endpoint design","pagination api","status codes","api versioning","rate limiting api","resource naming","filtering api","api error responses","offset pagination","limit query parameter","pagination defaults"],"skill:backend-patterns":["express api","node backend architecture","nextjs api routes","server side patterns","data access layer","static file server","url path handling","file server"],"skill:browser-qa":["deployed feature test","visual regression screenshots","core web vitals check","axe accessibility audit","ship do not ship","staging verification"],"skill:canary-watch":["post deploy monitoring","smoke test url","production url check","console errors production","sse stream check","after deploy verification"],"skill:code-tour":["onboarding walkthrough","explain subsystem","architecture tour","pr walkthrough","rca tour"],"skill:coding-standards":["code review standards","naming conventions","readability review","immutability conventions","fix naming typo","export naming","consistent exports"],"skill:content-hash-cache-pattern":["cache file processing","content addressed cache","sha256 hash cache"],"skill:database-migrations":["zero downtime migration","schema change production","add column large table","backfill data","expand contract","concurrent index","migration rollback","prisma migration","django migration"],"skill:deployment-patterns":["ci cd setup","dockerize app","health checks","rollback strategy","production readiness","deploy pipeline","containerize application"],"skill:design-system":["design tokens","visual consistency audit","css custom properties","ui audit","design system bootstrap"],"skill:django-patterns":["django orm","drf api","django rest framework","django caching","django signals","django middleware"],"skill:django-security":["django authentication","csrf protection","sql injection prevention","xss prevention","django deployment security","role based access control","authorization middleware","permissions checks"],"skill:docker-patterns":["dockerfile review","docker compose setup","container security","multi service orchestration"],"skill:error-handling":["error types","retry logic","circuit breaker","user facing errors","exception handling patterns","typed errors","error boundaries","go error handling","custom error class","error codes","config validation"],"skill:evm-token-decimals":["token decimals","wei conversion","erc20 balance off","bridge token precision"],"skill:frontend-a11y":["aria attributes","screen reader support","focus management","semantic html","form labeling","keyboard navigation react","a11y lint errors"],"skill:git-workflow":["merge vs rebase","commit conventions","resolve merge conflict","branching strategy","clean up commits","pull request cleanup","git history tidy"],"skill:hexagonal-architecture":["ports and adapters","dependency injection boundaries","decouple domain from io"],"skill:kubernetes-patterns":["kubernetes manifests","kubectl debugging","pod probes","k8s rbac","autoscaling config","configmap secrets"],"skill:orch-fix-defect":["fix a bug","broken behavior","regression fix","reproduce bug","defect repair"],"skill:postgres-patterns":["slow postgres query","query optimization","index design","rls policies","supabase schema","postgres indexing","database performance","schema design postgres","postgres driver","node postgres","query planner"],"skill:python-patterns":["pythonic code","pep 8","type hints python","python code review","idiomatic python"],"skill:python-testing":["pytest fixtures","mocking python","parametrized tests","coverage python","tdd python"],"skill:redis-patterns":["cache aside pattern","distributed lock","redis rate limiting","cache invalidation"],"skill:regex-vs-llm-structured-text":["parse invoice","extract receipt data","text extraction pipeline","parse form fields","cheap document parser","extract table data","parse log lines","parse access logs","common log format","log line parsing"],"skill:rust-patterns":["rust ownership","borrow checker","rust error handling","traits rust","rust concurrency","idiomatic rust"],"skill:search-first":["find existing library","npm package research","before writing custom code","evaluate existing tools","add dependency research"],"skill:security-review":["security audit","authentication review","sanitize user input","secrets handling","payment security checklist","prevent injection attacks","secure api endpoints","authn authz review","vulnerability checklist","input validation security","parameterized queries","sql injection"],"skill:security-scan":["audit claude config","claudemd security","mcp server audit","agentshield scan","hook configuration audit","settings json security"],"skill:tdd-workflow":["write test first","failing test","red green refactor","test driven development","regression test first","write a regression test"],"skill:verification-loop":["pre pr checks","verification report","quality gates","build lint test coverage","before creating a pr"]},"triggersDigest":"25b97a9e06fc336c7cf95ab854ed1a41033a54bcd6e1fb1cf69dc906332462aa"}+{+ "coverage": {+ "skills": 302,+ "withTriggers": 32+ },+ "generatedAt": "2026-09-24T23:51:22.784Z",+ "id": "skill-triggers@1",+ "model": {+ "effort": null,+ "id": "hand-seeded",+ "source": "manual-curation-pending-regeneration"+ },- "registryDigest": "c03f6e6f4beb33dc933195c02fbce3192e98c65831243450c39f791984a32abc",++ "registryDigest": "1d84b5f3852a014edbaa36c038561382f6e11b7b69a011d8801c7b8abd22fbb7",+ "schemaVersion": 1,+ "triggers": {+ "skill:api-connector-builder": [+ "add api integration",+ "new provider connector",+ "match existing integration pattern"+ ],+ "skill:api-design": [+ "rest endpoint design",+ "pagination api",+ "status codes",+ "api versioning",+ "rate limiting api",+ "resource naming",+ "filtering api",+ "api error responses",+ "offset pagination",+ "limit query parameter",+ "pagination defaults"+ ],+ "skill:backend-patterns": [+ "express api",+ "node backend architecture",+ "nextjs api routes",+ "server side patterns",+ "data access layer",+ "static file server",+ "url path handling",+ "file server"+ ],+ "skill:browser-qa": [+ "deployed feature test",+ "visual regression screenshots",+ "core web vitals check",+ "axe accessibility audit",+ "ship do not ship",+ "staging verification"+ ],+ "skill:canary-watch": [+ "post deploy monitoring",+ "smoke test url",+ "production url check",+ "console errors production",+ "sse stream check",+ "after deploy verification"+ ],+ "skill:code-tour": [+ "onboarding walkthrough",+ "explain subsystem",+ "architecture tour",+ "pr walkthrough",+ "rca tour"+ ],+ "skill:coding-standards": [+ "code review standards",+ "naming conventions",+ "readability review",+ "immutability conventions",+ "fix naming typo",+ "export naming",+ "consistent exports"+ ],+ "skill:content-hash-cache-pattern": [+ "cache file processing",+ "content addressed cache",+ "sha256 hash cache"+ ],+ "skill:database-migrations": [+ "zero downtime migration",+ "schema change production",+ "add column large table",+ "backfill data",+ "expand contract",+ "concurrent index",+ "migration rollback",+ "prisma migration",+ "django migration"+ ],+ "skill:deployment-patterns": [+ "ci cd setup",+ "dockerize app",+ "health checks",+ "rollback strategy",+ "production readiness",+ "deploy pipeline",+ "containerize application"+ ],+ "skill:design-system": [+ "design tokens",+ "visual consistency audit",+ "css custom properties",+ "ui audit",+ "design system bootstrap"+ ],+ "skill:django-patterns": [+ "django orm",+ "drf api",+ "django rest framework",+ "django caching",+ "django signals",+ "django middleware"+ ],+ "skill:django-security": [+ "django authentication",+ "csrf protection",+ "sql injection prevention",+ "xss prevention",+ "django deployment security",+ "role based access control",+ "authorization middleware",+ "permissions checks"+ ],+ "skill:docker-patterns": [+ "dockerfile review",+ "docker compose setup",+ "container security",+ "multi service orchestration"+ ],+ "skill:error-handling": [+ "error types",+ "retry logic",+ "circuit breaker",+ "user facing errors",+ "exception handling patterns",+ "typed errors",+ "error boundaries",+ "go error handling",+ "custom error class",+ "error codes",+ "config validation"+ ],+ "skill:evm-token-decimals": [+ "token decimals",+ "wei conversion",+ "erc20 balance off",+ "bridge token precision"+ ],+ "skill:frontend-a11y": [+ "aria attributes",+ "screen reader support",+ "focus management",+ "semantic html",+ "form labeling",+ "keyboard navigation react",+ "a11y lint errors"+ ],+ "skill:git-workflow": [+ "merge vs rebase",+ "commit conventions",+ "resolve merge conflict",+ "branching strategy",+ "clean up commits",+ "pull request cleanup",+ "git history tidy"+ ],+ "skill:hexagonal-architecture": [+ "ports and adapters",+ "dependency injection boundaries",+ "decouple domain from io"+ ],+ "skill:kubernetes-patterns": [+ "kubernetes manifests",+ "kubectl debugging",+ "pod probes",+ "k8s rbac",+ "autoscaling config",+ "configmap secrets"+ ],+ "skill:orch-fix-defect": [+ "fix a bug",+ "broken behavior",+ "regression fix",+ "reproduce bug",+ "defect repair"+ ],+ "skill:postgres-patterns": [+ "slow postgres query",+ "query optimization",+ "index design",+ "rls policies",+ "supabase schema",+ "postgres indexing",+ "database performance",+ "schema design postgres",+ "postgres driver",+ "node postgres",+ "query planner"+ ],+ "skill:python-patterns": [+ "pythonic code",+ "pep 8",+ "type hints python",+ "python code review",+ "idiomatic python"+ ],+ "skill:python-testing": [+ "pytest fixtures",+ "mocking python",+ "parametrized tests",+ "coverage python",+ "tdd python"+ ],+ "skill:redis-patterns": [+ "cache aside pattern",+ "distributed lock",+ "redis rate limiting",+ "cache invalidation"+ ],+ "skill:regex-vs-llm-structured-text": [+ "parse invoice",+ "extract receipt data",+ "text extraction pipeline",+ "parse form fields",+ "cheap document parser",+ "extract table data",+ "parse log lines",+ "parse access logs",+ "common log format",+ "log line parsing"+ ],+ "skill:rust-patterns": [+ "rust ownership",+ "borrow checker",+ "rust error handling",+ "traits rust",+ "rust concurrency",+ "idiomatic rust"+ ],+ "skill:search-first": [+ "find existing library",+ "npm package research",+ "before writing custom code",+ "evaluate existing tools",+ "add dependency research"+ ],+ "skill:security-review": [+ "security audit",+ "authentication review",+ "sanitize user input",+ "secrets handling",+ "payment security checklist",+ "prevent injection attacks",+ "secure api endpoints",+ "authn authz review",+ "vulnerability checklist",+ "input validation security",+ "parameterized queries",+ "sql injection"+ ],+ "skill:security-scan": [+ "audit claude config",+ "claudemd security",+ "mcp server audit",+ "agentshield scan",+ "hook configuration audit",+ "settings json security"+ ],+ "skill:tdd-workflow": [+ "write test first",+ "failing test",+ "red green refactor",+ "test driven development",+ "regression test first",+ "write a regression test"+ ],+ "skill:verification-loop": [+ "pre pr checks",+ "verification report",+ "quality gates",+ "build lint test coverage",+ "before creating a pr"+ ]+ },+ "triggersDigest": "25b97a9e06fc336c7cf95ab854ed1a41033a54bcd6e1fb1cf69dc906332462aa"+}