~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python3
"""Keep copy-pasteable attack payloads out of the published documentation.
This repository writes unusually candid security documentation, which is a
virtue -- but three times during the FastAPI migration a doc landed carrying a
*working input* against code that is byte-identical to the released version,
and each was caught only by after-the-fact review:
* c7b6216b2 / 395e38de1 -- a ``?type=pdf`` SSRF request, removed in 32ab42937
* bc24a446b -- a verdict table listing the IPv6 link-local addresses that
pass ``is_safe_custom_llm_endpoint``, removed in 2d24e0aa1
The distinction this hook enforces is the one those redactions settled on:
describing a MECHANISM is fine and useful; publishing an INPUT that a reader
can paste at a shipped version is not. So naming a CIDR range -- the IPv4 or
IPv6 link-local block -- passes, because that is how you explain the fix, while
the same address written as a ``scheme://host`` URL does not, because that is
the request itself rather than a description of it.
(This docstring deliberately does not spell out an example URL: the repo's
whitelist-check flags literal external IPs in source, and quoting one here to
illustrate the rule would be the very thing the rule exists to prevent.)
If a payload genuinely must appear (a test fixture quoted in a doc, a CVE
write-up for an already-patched-and-released version), add the marker
``<!-- payload-ok: reason -->`` on the line or the line above. The marker is
deliberately noisy so it shows up in review.
"""
import re
import sys
from pathlib import Path
MARKER = "payload-ok:"
# A scheme plus a host that only makes sense as an SSRF target. Anchored on
# `scheme://` so prose and CIDR ranges are not matched.
_HOST_PAYLOAD = re.compile(
r"""[a-z][a-z0-9+.-]*:// # scheme://
\[? # optional IPv6 bracket
(?:
169\.254\.\d{1,3}\.\d{1,3} # IPv4 link-local / AWS IMDS
| fe80::[0-9a-f:]* # IPv6 link-local
| fd00:ec2::[0-9a-f:]* # AWS IMDS over IPv6
| fd20:ce::[0-9a-f:]* # GCP metadata over IPv6
| fd00:42::[0-9a-f:]* # Scaleway metadata over IPv6
| fd00:a9fe:[0-9a-f:]* # Akamai / Linode metadata over IPv6
| fd00:c1::[0-9a-f:]* # OCI IMDS over IPv6
| metadata\.google\.internal
| metadata\.goog
)""",
re.IGNORECASE | re.VERBOSE,
)
# Schemes whose only use in a doc is an SSRF demonstration.
_SSRF_SCHEME = re.compile(r"\b(?:gopher|dict|ldap)://", re.IGNORECASE)
CHECKS = (
(
_HOST_PAYLOAD,
"a link-local / cloud-metadata URL (an SSRF request, not a range)",
),
(_SSRF_SCHEME, "an SSRF-only URL scheme"),
)
def check(path: Path) -> list[str]:
try:
lines = path.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError):
return []
issues = []
# A marker may be a multi-line HTML comment sitting a few lines above the
# payload, so look back over a small window rather than a single line.
LOOKBACK = 6
for i, line in enumerate(lines, 1):
window = lines[max(0, i - 1 - LOOKBACK) : i]
if any(MARKER in w for w in window):
continue
for pattern, why in CHECKS:
m = pattern.search(line)
if m:
issues.append(
f"{path}:{i}: {why}: {m.group(0)!r}\n"
f" Describe the mechanism instead (a CIDR range is fine), or add\n"
f" <!-- {MARKER} why this is safe to publish --> if it must stay."
)
break
return issues
def main(argv: list[str]) -> int:
issues: list[str] = []
for name in argv:
p = Path(name)
if p.suffix == ".md" and p.is_file():
issues.extend(check(p))
if issues:
print("Copy-pasteable attack payload(s) in documentation:\n")
print("\n".join(issues))
print(
"\nThis repo redacted three of these during the FastAPI migration "
"(32ab42937, 2d24e0aa1). Publishing a working input against "
"released code is what those commits removed."
)
return 1
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))