Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.9 KB
Raw Blame
"""
SSRF-hardened HTTP fetch for URL-backed knowledge sources.

Responsibilities:
- Validate URL (reuses `posthog.security.url_validation`) before each hop.
- Pin validated IPs so ``requests`` connects to the exact IPs we checked,
  eliminating the DNS-rebinding TOCTOU window.
- Manual redirect handling — we refuse the default `requests` redirect chain
  because it does *not* re-validate the target IP, which is the classic
  DNS-rebinding / open-redirect bypass for SSRF.
- Stream-read the body with a hard byte cap so a malicious server can't blow
  up process memory by advertising `Content-Length: 0` and then never closing.
- Conditional GET (`If-None-Match`) so repeat refreshes are cheap.
"""

from __future__ import annotations

import hashlib
import urllib.parse as urlparse
from dataclasses import dataclass

import requests
import structlog

from posthog.security.pinned_requests import PinnedIPAdapter, select_pinned_ip
from posthog.security.url_validation import strip_userinfo, validate_url_and_pin_ips

from .constants import URL_CONNECT_TIMEOUT, URL_MAX_BYTES, URL_MAX_REDIRECTS, URL_READ_TIMEOUT, URL_USER_AGENT

logger = structlog.get_logger(__name__)


class UrlFetchError(Exception):
    """Generic fetch failure. Message is user-safe (no internal detail)."""


@dataclass(frozen=True)
class FetchResult:
    status: int  # 200 on success, 304 on Not Modified
    body: bytes | None  # None on 304
    content_type: str | None
    etag: str | None
    final_url: str


def normalize_url(raw: str) -> str:
    """
    Canonicalize a user-submitted URL before validating or storing it.

    Intentionally minimal: strip userinfo + fragment, lowercase the scheme
    and host. We do NOT touch the path/query — a lot of docs sites care about
    trailing slashes and case in path segments.
    """

    raw = raw.strip()
    # urlparse silently drops tab and newline characters and keeps the text after them,
    # so reject control characters before parsing.
    if any(ord(char) < 0x20 or ord(char) == 0x7F for char in raw):
        raise UrlFetchError("Invalid URL.")
    parsed = urlparse.urlparse(raw)
    if not parsed.scheme or not parsed.netloc:
        raise UrlFetchError("Invalid URL.")
    scheme = parsed.scheme.lower()
    host = (parsed.hostname or "").lower()
    if not host:
        raise UrlFetchError("Invalid URL.")
    netloc = host
    if parsed.port:
        netloc = f"{netloc}:{parsed.port}"
    return urlparse.urlunparse((scheme, netloc, parsed.path, parsed.params, parsed.query, ""))


def sha256_of(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8")).hexdigest()


def prefetch_key(url: str) -> str:
    """
    Stable cache key for a URL, shared by the crawl prefetch path. Discovery
    caches fetched bodies under this key and the fetch phase looks them up by
    it — they MUST agree, so both go through this one function.
    """

    try:
        return normalize_url(url)
    except UrlFetchError:
        return url


# --- Shared SSRF-safe fetch core ---------------------------------------------


def _read_capped(response: requests.Response, cap: int) -> bytes:
    """Stream the body into memory but abort as soon as we cross `cap` bytes."""

    chunks: list[bytes] = []
    total = 0
    for chunk in response.iter_content(chunk_size=64 * 1024):
        if not chunk:
            continue
        total += len(chunk)
        if total > cap:
            raise UrlFetchError("Remote response exceeds the maximum allowed size.")
        chunks.append(chunk)
    return b"".join(chunks)


def _ssrf_safe_get(
    url: str,
    *,
    headers: dict[str, str],
    max_bytes: int = URL_MAX_BYTES,
    etag: str | None = None,
) -> FetchResult:
    """
    Core SSRF-hardened GET with manual redirect handling and DNS pinning.

    Re-validates ``validate_url_and_pin_ips`` on every redirect hop and
    pins the validated IPs so ``requests`` connects to the exact addresses
    we checked (no DNS rebinding window). Stream-reads the body with a
    hard byte cap.

    Raises ``UrlFetchError`` with a user-safe message on any failure.
    """

    current = strip_userinfo(normalize_url(url))
    adapter = PinnedIPAdapter()
    session = requests.Session()
    session.mount("http://", adapter)  # nosemgrep: request-session-with-http
    session.mount("https://", adapter)
    try:
        for _hop in range(URL_MAX_REDIRECTS + 1):
            verdict = validate_url_and_pin_ips(current)
            if not verdict.allowed:
                logger.warning(
                    "business_knowledge.url_fetch.ssrf_blocked",
                    url=current,
                    reason=verdict.reason,
                )
                raise UrlFetchError("URL is not reachable from this environment.")

            # select_pinned_ip prefers IPv4. An arbitrary member of pinned_ips can be
            # an AAAA record, and an IPv4-only worker then fails the connection.
            parsed = urlparse.urlparse(current)
            hostname = (parsed.hostname or "").lower()
            chosen_ip = select_pinned_ip(verdict.pinned_ips)
            if chosen_ip is not None:
                adapter.pin(hostname, chosen_ip)

            merged_headers = dict(headers)
            if etag:
                merged_headers["If-None-Match"] = etag

            try:
                response = session.get(
                    current,
                    headers=merged_headers,
                    timeout=(URL_CONNECT_TIMEOUT, URL_READ_TIMEOUT),
                    allow_redirects=False,
                    stream=True,
                )
            except requests.RequestException as exc:
                logger.info(
                    "business_knowledge.url_fetch.transport_error",
                    url=current,
                    error_type=type(exc).__name__,
                )
                raise UrlFetchError("Failed to fetch the URL.") from exc

            try:
                if response.status_code in (301, 302, 303, 307, 308):
                    location = response.headers.get("Location")
                    if not location:
                        raise UrlFetchError("Redirect without Location header.")
                    next_url = strip_userinfo(urlparse.urljoin(current, location))
                    current = normalize_url(next_url)
                    continue

                if response.status_code == 304:
                    return FetchResult(
                        status=304,
                        body=None,
                        content_type=response.headers.get("Content-Type"),
                        etag=response.headers.get("ETag") or etag,
                        final_url=current,
                    )

                if response.status_code >= 400:
                    raise UrlFetchError(f"Remote responded with status {response.status_code}.")

                declared = response.headers.get("Content-Length")
                if declared and declared.isdigit() and int(declared) > max_bytes:
                    raise UrlFetchError("Remote response exceeds the maximum allowed size.")

                body = _read_capped(response, max_bytes)
                return FetchResult(
                    status=200,
                    body=body,
                    content_type=response.headers.get("Content-Type"),
                    etag=response.headers.get("ETag"),
                    final_url=current,
                )
            finally:
                response.close()

        raise UrlFetchError("Too many redirects.")
    finally:
        session.close()


# --- Public API ---------------------------------------------------------------


_CONTENT_HEADERS = {
    "User-Agent": URL_USER_AGENT,
    "Accept": "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.1",
    "Accept-Language": "en;q=0.9,*;q=0.5",
}

_METADATA_HEADERS = {
    "User-Agent": URL_USER_AGENT,
    "Accept": "application/xml,text/xml,text/html;q=0.9,text/plain;q=0.5,*/*;q=0.1",
}


def fetch_url(url: str, *, etag: str | None = None) -> FetchResult:
    """
    Fetch `url` with SSRF re-validation on every redirect hop.

    Raises `UrlFetchError` with a user-safe message on any failure.
    """

    return _ssrf_safe_get(url, headers=_CONTENT_HEADERS, etag=etag)


def fetch_text(url: str, *, max_bytes: int = URL_MAX_BYTES) -> str:
    """
    SSRF-safe GET returning decoded text. Used by discover for machine-readable
    metadata (sitemap.xml, robots.txt, HTML for link extraction).

    Raises `UrlFetchError` on failure.
    """

    result = _ssrf_safe_get(url, headers=_METADATA_HEADERS, max_bytes=max_bytes)
    if result.body is None:
        return ""
    return result.body.decode("utf-8", errors="replace")


def is_html_content_type(content_type: str | None) -> bool:
    if not content_type:
        # Missing header — trafilatura can still do something sensible with
        # raw HTML, and we fall back to bs4 if it can't.
        return True
    lowered = content_type.split(";", 1)[0].strip().lower()
    return lowered in {"text/html", "application/xhtml+xml", "text/plain"}