Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

2.5 KB
Raw Blame
#!/usr/bin/env python3
"""
Dump the FastAPI route table to a file as one absolute URL per line.

Used by the Nuclei DAST workflow to seed a URL list so the scanner
probes the actual application surface (authenticated routes, API
endpoints, routers) instead of just the index page.

Parameterized routes (`/research/{research_id}/status`) are emitted
with a converter-appropriate placeholder so Nuclei still exercises the
path. The substituted URL will usually 404 (the resource doesn't exist
for the test user), but that is fine — Nuclei probes path traversal,
parameter injection, and SQLi templates against the URL pattern, not
against a specific resource.

Skips:
  - Mounts (StaticFiles asset serving and the Socket.IO ASGI app) —
    no per-route app logic to probe.
  - Routes that don't accept GET — Nuclei templates almost exclusively
    issue GET probes, so POST-only endpoints just generate 405s.

Usage:
    python scripts/ci/dump_url_map.py http://127.0.0.1:5000 > urls.txt
"""

import re
import sys


# Map Starlette path converters to a placeholder that satisfies the
# converter so the router dispatches the request to the handler instead
# of 404-ing at the converter stage. Anything not listed falls back to a
# plain string.
_PLACEHOLDERS = {
    "int": "1",
    "float": "1",
    "uuid": "00000000-0000-0000-0000-000000000000",
    "path": "nuclei",
}
_DEFAULT_PLACEHOLDER = "nuclei"

# Starlette syntax: {name} or {name:converter}
_PARAM_RE = re.compile(
    r"\{(?P<name>[a-zA-Z_][a-zA-Z0-9_]*)(?::(?P<conv>[a-zA-Z_][a-zA-Z0-9_]*))?\}"
)


def _substitute(match: "re.Match[str]") -> str:
    return _PLACEHOLDERS.get(match.group("conv") or "", _DEFAULT_PLACEHOLDER)


def main() -> int:
    if len(sys.argv) != 2:
        print(f"Usage: {sys.argv[0]} BASE_URL", file=sys.stderr)
        return 2

    base_url = sys.argv[1].rstrip("/")

    from starlette.routing import Route

    from local_deep_research.web.fastapi_app import app

    seen: set[str] = set()
    for route in app.routes:
        # Mounts (StaticFiles, the Socket.IO sub-app) have no methods and
        # no per-route handler logic worth probing.
        if not isinstance(route, Route):
            continue
        if "GET" not in (route.methods or set()):
            continue
        path = _PARAM_RE.sub(_substitute, route.path)
        url = f"{base_url}{path}"
        if url in seen:
            continue
        seen.add(url)
        print(url)

    return 0


if __name__ == "__main__":
    sys.exit(main())