Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

docs(ast-vuln): sanitizer-composition research + fingerprint index (~2700x fewer tokens)

Xerj Squad A2c7b28af3af2
diff --git a/docs/examples/ast-vuln-graph/wp_compose_index.py b/docs/examples/ast-vuln-graph/wp_compose_index.py
new file mode 100644
index 000000000..06a8a57fe
--- /dev/null
+++ b/docs/examples/ast-vuln-graph/wp_compose_index.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Improve XERJ for composition audits: compute each function's SANITIZER
+SEQUENCE fingerprint ONCE at index time, so the audit is a tiny structured query
+over fingerprints instead of pulling and reading hundreds of code bodies.
+
+Token cost of an audit drops from O(total code size) to O(true candidates)."""
+import glob, json, os, re, sys, urllib.request
+X="http://127.0.0.1:9200"; WP=sys.argv[1] if len(sys.argv)>1 else None
+
+ESC = ["esc_html","esc_attr","esc_sql","esc_url","esc_js","esc_textarea","wp_specialchars",
+ "addslashes","wp_slash","wp_kses","wp_kses_post","sanitize_text_field"]
+DEESC = ["html_entity_decode","wp_specialchars_decode","htmlspecialchars_decode","urldecode",
+ "rawurldecode","stripslashes","stripslashes_deep","wp_unslash","base64_decode"]
+RAW_SINK = r"->(query|get_results|get_var|get_row|get_col)\(" # attacker-dangerous
+SAFE_SINK = r"->(update|insert|delete|replace|prepare)\(" # self-escaping wpdb methods
+OUT_SINK = r"\becho\b|printf\(|\bprint\b"
+
+def fingerprint(code):
+ ev=[]
+ for name in ESC:
+ for m in re.finditer(r"\b"+re.escape(name)+r"\s*\(",code): ev.append((m.start(),"ESC:"+name))
+ for name in DEESC:
+ for m in re.finditer(r"\b"+re.escape(name)+r"\s*\(",code): ev.append((m.start(),"DEE:"+name))
+ for m in re.finditer(RAW_SINK,code): ev.append((m.start(),"SNKraw"))
+ for m in re.finditer(SAFE_SINK,code): ev.append((m.start(),"SNKsafe"))
+ for m in re.finditer(OUT_SINK,code): ev.append((m.start(),"SNKout"))
+ seq=[t for _,t in sorted(ev)]
+ # danger-order = a de-escape after some escape, then a sink after that de-escape
+ danger=False
+ esc_i=[i for i,t in enumerate(seq) if t.startswith("ESC")]
+ if esc_i:
+ first_esc=esc_i[0]
+ for i,t in enumerate(seq):
+ if t.startswith("DEE") and i>first_esc and any(seq[j].startswith("SNK") for j in range(i+1,len(seq))):
+ danger=True; break
+ has_raw = any(t=="SNKraw" for t in seq)
+ return seq, danger, has_raw
+
+def process(code):
+ seq,danger,has_raw=fingerprint(code)
+ # audit-grade flag: dangerous order AND a RAW sink (the only order that can inject/XSS)
+ return {"sanitizer_seq":seq, "compose_danger":danger, "sink_raw":has_raw,
+ "compose_audit":bool(danger and (has_raw or "SNKout" in seq))}
+
+if WP: # (re)build a compose index from source
+ from tree_sitter import Language, Parser
+ import tree_sitter_php as tsphp
+ parser=Parser(Language(tsphp.language_php()))
+ def txt(n): return n.text.decode("utf8","replace")
+ docs=[]
+ for path in glob.glob(f"{WP}/**/*.php",recursive=True):
+ root=parser.parse(open(path,"rb").read()).root_node
+ rel=os.path.relpath(path,WP)
+ def walk(n):
+ if n.type in ("function_definition","method_declaration"):
+ nm=n.child_by_field_name("name"); nm=txt(nm) if nm else "<anon>"
+ d={"file":rel,"func":nm,"line":n.start_point[0]+1}; d.update(process(txt(n)))
+ if d["sanitizer_seq"]: docs.append(d)
+ for c in n.children: walk(c)
+ walk(root)
+ try: urllib.request.urlopen(urllib.request.Request(f"{X}/wpcompose",method="DELETE")).read()
+ except Exception: pass
+ m={"mappings":{"properties":{"file":{"type":"keyword"},"func":{"type":"keyword"},
+ "line":{"type":"integer"},"sanitizer_seq":{"type":"keyword"},
+ "compose_danger":{"type":"boolean"},"sink_raw":{"type":"boolean"},
+ "compose_audit":{"type":"boolean"}}}}
+ urllib.request.urlopen(urllib.request.Request(f"{X}/wpcompose",data=json.dumps(m).encode(),
+ headers={"Content-Type":"application/json"},method="PUT")).read()
+ lines=[]
+ for d in docs: lines.append(json.dumps({"index":{}})); lines.append(json.dumps(d))
+ r=urllib.request.Request(f"{X}/wpcompose/_bulk",data=("\n".join(lines)+"\n").encode(),
+ headers={"Content-Type":"application/x-ndjson"},method="POST")
+ resp=json.loads(urllib.request.urlopen(r).read())
+ print(f"wpcompose: indexed {len(docs)} functions with sanitizer fingerprints, errors={resp.get('errors')}")
diff --git a/docs/research/wordpress-read-first-findings.md b/docs/research/wordpress-read-first-findings.md
index 66e29a95a..ddf6ed1e0 100644
--- a/docs/research/wordpress-read-first-findings.md
+++ b/docs/research/wordpress-read-first-findings.md
@@ -89,3 +89,80 @@ an un-re-verified object relationship. XERJ's contribution is to make each such
semantic invariant, once discovered by reading, a **stored, queryable
completeness check** over the whole indexed codebase. Precision comes from the
read; durability and scale come from XERJ.
+
+---
+
+# Part 2 — Sanitizer composition, and making the audit cheap
+
+## The class: two protections that interact and one undoes the other
+
+The richest composition bug is **de-escape after escape before a sink**: a value
+is escaped, then a later filter (`stripslashes`, `wp_unslash`, `urldecode`,
+`html_entity_decode`) *removes* the escaping before it reaches SQL or output.
+`esc_sql($x)` then `stripslashes(...)` = injection; `esc_html($x)` then
+`html_entity_decode(...)` = XSS.
+
+Reading core's compositions, the invariant it holds everywhere is **escaper-last
+before the sink**:
+
+- `WP_Term_Query::get_terms` — `sanitize_term_field` (slashed) → `stripslashes`
+ (clean) → **`esc_sql` last** → query. Correct; the comment documents it.
+- `wp_update_term` — `sanitize_term` → `wp_unslash` → **`$wpdb->update()`**, which
+ re-escapes via format specifiers. Unslashing *before* `$wpdb->update/insert` is
+ the *required* WP convention (not doing it double-slashes). Correct.
+- `wp_widget_rss_output` — `$desc = html_entity_decode(...)` then
+ `$desc = esc_attr( wp_trim_words($desc) )` **re-escapes last**. Correct.
+
+An order-aware detector flagged **23** candidates; every one cleared on reading,
+via four false-positive drivers now understood: (1) a *safe* self-escaping
+`$wpdb->update/insert/prepare` sink misread as raw; (2) decode-then-re-escape on
+the actual output variable; (3) escape and decode on *different* variables; (4)
+plaintext-email output where decode is correct. **Core composes correctly.** But,
+as with SSRF, that verdict required *reading* — and reading 23 bodies is
+expensive.
+
+## Making it cheap: the sanitizer-sequence fingerprint (the XERJ improvement)
+
+`wp_compose_index.py` computes, once at index time, each function's ordered
+**sanitizer sequence** — a compact keyword list like
+`["ESC:esc_html","DEE:html_entity_decode","ESC:esc_attr","SNKout"]` — plus a
+`sink_raw` flag distinguishing attacker-dangerous sinks (`$wpdb->query`,
+`get_results`, `echo`) from self-escaping ones (`$wpdb->update/insert/prepare`).
+It stores these as indexed fields. The whole composition audit then becomes a
+single structured query:
+
+```
+GET wpcompose { compose_danger:true AND sink_raw:true } → 6 candidates + their fingerprints
+```
+
+The fingerprints are **self-triaging**: `wp_update_term`'s
+`[ESC:wp_slash, DEE:wp_unslash, …, SNKsafe]` shows the unslashed value reaching a
+*safe* sink; `wp_notify_moderator`'s long `SNKout` tail is email. Most candidates
+clear without pulling a single line of code.
+
+**Measured token cost of the same audit:**
+
+| approach | what it reads | tokens |
+|---|---|--:|
+| read every sanitizer-bearing function to find the bad order | 3,624 function bodies | **~1,329,000** |
+| one query over the sanitizer-sequence fingerprint | 6 fingerprints (+ read only the un-clearable) | **~490** |
+
+**≈2,700×.** The audit cost drops from O(total code size) to O(true candidates),
+because the semantic invariant was compiled into an indexed fingerprint once and
+queried, instead of re-derived by reading every time.
+
+## The generalizable pattern for XERJ
+
+This is the reusable shape for *every* invariant found by reading:
+
+1. **Read once** to discover the invariant (escaper-last; block 169.254; re-verify
+ the object relationship).
+2. **Compile it to a fingerprint field** at index time — the ordered sanitizer
+ sequence, the covered IP ranges, the checked-vs-used object keys.
+3. **Audit by querying the fingerprint**, returning only violators — tiny result,
+ no code transfer.
+4. **Read only the survivors** to confirm.
+
+Steps 2–4 are where XERJ turns a million-token reading pass into a few-hundred-token
+query — and, run against the plugin ecosystem, the same fingerprints will surface
+the escaper-last violations and incomplete validators that core doesn't have.