Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

19.9 KB
Raw Blame
"""SSRF-protected HTTP client for generic outbound requests.

Use this module when the URL represents a normal external HTTP interaction that
must go through network/proxy policy exactly as requested, such as HTTP Request
nodes, provider/API integrations, auth discovery, or custom tool calls.

Do not use this directly for "remote file" retrieval. File downloads, probes,
and metadata checks should use `core.file.remote_fetcher` instead so Dify-signed
file URLs can be resolved through DB + storage before falling back to this SSRF
client.
"""

import logging
import time
import urllib.request
from collections.abc import Generator, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from http.client import HTTPResponse
from http.cookiejar import Cookie, CookieJar
from typing import Any, override

import httpx
from pydantic import TypeAdapter, ValidationError

from configs import dify_config
from core.helper.http_client_pooling import get_pooled_http_client
from core.tools.errors import ToolSSRFError
from graphon.http.response import HttpResponse

logger = logging.getLogger(__name__)

SSRF_DEFAULT_MAX_RETRIES = dify_config.SSRF_DEFAULT_MAX_RETRIES

BACKOFF_FACTOR = 0.5
STATUS_FORCELIST = [429, 500, 502, 503, 504]

type Headers = dict[str, str]
_HEADERS_ADAPTER: TypeAdapter[Headers] = TypeAdapter(Headers)

_SSL_VERIFIED_POOL_KEY = "ssrf:verified"
_SSL_UNVERIFIED_POOL_KEY = "ssrf:unverified"
_SSRF_CLIENT_LIMITS = httpx.Limits(
    max_connections=dify_config.SSRF_POOL_MAX_CONNECTIONS,
    max_keepalive_connections=dify_config.SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS,
    keepalive_expiry=dify_config.SSRF_POOL_KEEPALIVE_EXPIRY,
)


class MaxRetriesExceededError(ValueError):
    """Raised when the maximum number of retries is exceeded.

    'status_code' is the status of the last response that was received, so a
    caller can report what the upstream actually answered. It stays None when
    every attempt failed before a response arrived.
    """

    def __init__(self, message: str, status_code: int | None = None) -> None:
        super().__init__(message)
        self.status_code = status_code


class ResponseLimitError(ValueError):
    """Base error for responses that cannot be safely bounded."""

    pass


class ResponseTooLargeError(ResponseLimitError):
    """Raised when an identity response exceeds the configured byte limit."""

    pass


class UnsupportedResponseEncodingError(ResponseLimitError):
    """Raised when response encoding prevents safe decoded-size enforcement."""

    pass


request_error = httpx.RequestError
max_retries_exceeded_error = MaxRetriesExceededError

_send_cookie_jar: ContextVar[CookieJar | None] = ContextVar("ssrf_send_cookie_jar", default=None)


class _SendScopedCookieJar(CookieJar):
    """Client cookie jar that keeps response cookies only inside ``_send_cookie_scope``.

    The pooled SSRF clients are shared by every tenant in the process, and httpx stores each response's
    ``Set-Cookie`` in the client jar and replays it on later requests. While sending, httpx writes the client
    jar only through ``extract_cookies`` and reads it only by iterating it, so both go to the jar of the current
    scope: redirect hops inside one scope still carry cookies set by earlier hops, and nothing survives the
    scope. Outside a scope the jar is empty and stores nothing. Cookies passed explicitly on a request never
    enter this jar.
    """

    @override
    def extract_cookies(self, response: HTTPResponse, request: urllib.request.Request) -> None:
        scope_jar = _send_cookie_jar.get()
        if scope_jar is not None:
            scope_jar.extract_cookies(response, request)

    @override
    def __iter__(self) -> Iterator[Cookie]:
        scope_jar = _send_cookie_jar.get()
        if scope_jar is None:
            return iter(())
        return iter(scope_jar)


@contextmanager
def _send_cookie_scope() -> Generator[None]:
    """Give one request attempt, including its redirect hops, a fresh cookie jar on the SSRF clients."""
    token = _send_cookie_jar.set(CookieJar())
    try:
        yield
    finally:
        _send_cookie_jar.reset(token)


def _create_proxy_mounts(verify: bool) -> dict[str, httpx.HTTPTransport]:
    """Build per-scheme proxy transports with the same TLS policy as the SSRF client."""
    return {
        "http://": httpx.HTTPTransport(
            proxy=dify_config.SSRF_PROXY_HTTP_URL,
            verify=verify,
        ),
        "https://": httpx.HTTPTransport(
            proxy=dify_config.SSRF_PROXY_HTTPS_URL,
            verify=verify,
        ),
    }


def _build_ssrf_client(verify: bool) -> httpx.Client:
    if dify_config.SSRF_PROXY_ALL_URL:
        return httpx.Client(
            proxy=dify_config.SSRF_PROXY_ALL_URL,
            verify=verify,
            limits=_SSRF_CLIENT_LIMITS,
            cookies=_SendScopedCookieJar(),
        )

    if dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL:
        return httpx.Client(
            mounts=_create_proxy_mounts(verify=verify),
            verify=verify,
            limits=_SSRF_CLIENT_LIMITS,
            cookies=_SendScopedCookieJar(),
        )

    return httpx.Client(verify=verify, limits=_SSRF_CLIENT_LIMITS, cookies=_SendScopedCookieJar())


def _get_ssrf_client(ssl_verify_enabled: bool) -> httpx.Client:
    if not isinstance(ssl_verify_enabled, bool):
        raise ValueError("SSRF client verify flag must be a boolean")

    return get_pooled_http_client(
        _SSL_VERIFIED_POOL_KEY if ssl_verify_enabled else _SSL_UNVERIFIED_POOL_KEY,
        lambda: _build_ssrf_client(verify=ssl_verify_enabled),
    )


def _get_user_provided_host_header(headers: Headers | None) -> str | None:
    """
    Extract the user-provided Host header from the headers dict.

    This is needed because when using a forward proxy, httpx may override the Host header.
    We preserve the user's explicit Host header to support virtual hosting and other use cases.
    """
    if not headers:
        return None
    # Case-insensitive lookup for Host header
    for key, value in headers.items():
        if key.lower() == "host":
            return value
    return None


def _inject_trace_headers(headers: Headers | None) -> Headers:
    """
    Inject W3C traceparent header for distributed tracing.

    When OTEL is enabled, HTTPXClientInstrumentor handles trace propagation automatically.
    When OTEL is disabled, we manually inject the traceparent header.
    """
    if headers is None:
        headers = {}

    # Skip if already present (case-insensitive check)
    for key in headers:
        if key.lower() == "traceparent":
            return headers

    # Skip if OTEL is enabled - HTTPXClientInstrumentor handles this automatically
    if dify_config.ENABLE_OTEL:
        return headers

    # Generate and inject traceparent for non-OTEL scenarios
    try:
        from core.helper.trace_id_helper import generate_traceparent_header

        traceparent = generate_traceparent_header()
        if traceparent:
            headers["traceparent"] = traceparent
    except Exception:
        # Silently ignore errors to avoid breaking requests
        logger.debug("Failed to generate traceparent header", exc_info=True)

    return headers


def make_request(
    method: str,
    url: str,
    max_retries: int = SSRF_DEFAULT_MAX_RETRIES,
    stream_response: bool = False,
    **kwargs: Any,
) -> httpx.Response:
    """Send one SSRF-protected request with optional streaming.

    Cookies set by responses are sent only on the redirect hops of the same attempt, never on later requests;
    callers that need a cookie pass it explicitly.

    Args:
        method: HTTP method sent through the configured SSRF client.
        url: Absolute request URL.
        max_retries: Number of retry attempts after the initial request.
        stream_response: Return an open streaming response that the caller must close.
        **kwargs: Additional keyword arguments forwarded to ``httpx.Client``.

    Returns:
        A buffered response, or an open response when ``stream_response`` is true.

    Raises:
        ToolSSRFError: The configured SSRF proxy rejects the destination.
        MaxRetriesExceededError: All configured request attempts fail.
        httpx.RequestError: A request fails while retries are disabled.
        ValueError: The SSL verification option or request headers are invalid.
    """
    # Convert requests-style allow_redirects to httpx-style follow_redirects
    if "allow_redirects" in kwargs:
        allow_redirects = kwargs.pop("allow_redirects")
        if "follow_redirects" not in kwargs:
            kwargs["follow_redirects"] = allow_redirects

    if "timeout" not in kwargs:
        kwargs["timeout"] = httpx.Timeout(
            timeout=dify_config.SSRF_DEFAULT_TIME_OUT,
            connect=dify_config.SSRF_DEFAULT_CONNECT_TIME_OUT,
            read=dify_config.SSRF_DEFAULT_READ_TIME_OUT,
            write=dify_config.SSRF_DEFAULT_WRITE_TIME_OUT,
        )

    # prioritize per-call option, which can be switched on and off inside the HTTP node on the web UI
    verify_option = kwargs.pop("ssl_verify", dify_config.HTTP_REQUEST_NODE_SSL_VERIFY)
    if not isinstance(verify_option, bool):
        raise ValueError("ssl_verify must be a boolean")
    client = _get_ssrf_client(verify_option)

    # Inject traceparent header for distributed tracing (when OTEL is not enabled)
    try:
        headers: Headers = _HEADERS_ADAPTER.validate_python(kwargs.get("headers") or {})
    except ValidationError as e:
        raise ValueError("headers must be a mapping of string keys to string values") from e
    headers = _inject_trace_headers(headers)
    kwargs["headers"] = headers

    # Preserve user-provided Host header
    # When using a forward proxy, httpx may override the Host header based on the URL.
    # We extract and preserve any explicitly set Host header to support virtual hosting.
    user_provided_host = _get_user_provided_host_header(headers)
    send_kwargs: dict[str, Any] = {}
    if "auth" in kwargs:
        send_kwargs["auth"] = kwargs.pop("auth")
    if "follow_redirects" in kwargs:
        send_kwargs["follow_redirects"] = kwargs.pop("follow_redirects")

    retries = 0
    last_status: int | None = None
    last_request_error: httpx.RequestError | None = None
    while retries <= max_retries:
        try:
            # Preserve the user-provided Host header
            # httpx may override the Host header when using a proxy
            headers = {k: v for k, v in headers.items() if k.lower() != "host"}
            if user_provided_host is not None:
                headers["host"] = user_provided_host
            kwargs["headers"] = headers
            with _send_cookie_scope():
                request = client.build_request(method=method, url=url, **kwargs)
                if stream_response:
                    response = client.send(request, stream=True, **send_kwargs)
                else:
                    response = client.send(request, **send_kwargs)
            last_request_error = None

            last_status = response.status_code

            # Check for SSRF protection by Squid proxy
            if response.status_code in (401, 403):
                # Only a response that Squid generated itself proves that the
                # proxy denied the destination: such an error page carries
                # ``X-Squid-Error`` and identifies Squid in ``Server``. Squid
                # appends ``Via`` to *every* proxied response, so a Squid-based
                # CDN forwards upstream 401/403s with a ``Via`` header too and
                # matching it would misreport a public destination as blocked
                # (issue #44785). ``Server`` is optional, so an absent origin
                # header cannot turn ``Via`` alone into denial evidence.
                server_header = response.headers.get("server", "").lower()
                squid_error_header = response.headers.get("x-squid-error", "").lower()
                is_squid_denial = "squid" in server_header or "err_access_denied" in squid_error_header

                if is_squid_denial:
                    # The deny ACL is usually ``to_private_networks`` (RFC1918 +
                    # loopback / link-local / CGN / IPv6 ULA, etc.). We don't know
                    # which specific ACL tripped from Squid's response alone, but
                    # the actionable remediation is the same in every case:
                    # allowlist the destination in the SSRF proxy. Tell the user
                    # exactly which env var to set so they don't have to grep the
                    # squid config. Mention a concrete example CIDR (e.g. the
                    # 172.21.0.0/16 from the bug report) so they can copy-paste it.
                    response.close()
                    raise ToolSSRFError(
                        f"Access to '{url}' was blocked by SSRF protection "
                        f"(e.g. SSRF_PROXY_ALLOW_PRIVATE_IPS=172.21.0.0/16 to "
                        f"allow 172.21.0.0/16). The URL resolves to a private, "
                        f"loopback, link-local, or otherwise non-public network "
                        f"address. See https://github.com/langgenius/dify/issues/38443."
                    )

            if response.status_code not in STATUS_FORCELIST or max_retries == 0:
                return response
            else:
                logger.warning(
                    "Received status code %s for URL %s which is in the force list",
                    response.status_code,
                    url,
                )
                response.close()

        except httpx.RequestError as e:
            last_request_error = e
            logger.warning("Request to URL %s failed on attempt %s: %s", url, retries + 1, e)
            if max_retries == 0:
                raise

        retries += 1
        if retries <= max_retries:
            time.sleep(BACKOFF_FACTOR * (2 ** (retries - 1)))
    error = MaxRetriesExceededError(
        f"Reached maximum retries ({max_retries}) for URL {url}",
        status_code=last_status,
    )
    if last_request_error is not None:
        raise error from last_request_error
    raise error


def buffer_response(response: httpx.Response, *, max_response_bytes: int) -> httpx.Response:
    """Consume one open identity response under a decoded byte limit and close its stream."""
    if max_response_bytes <= 0:
        raise ValueError("max_response_bytes must be positive")

    try:
        content_encoding = response.headers.get("content-encoding", "identity").strip().lower()
        if content_encoding not in {"", "identity"}:
            raise UnsupportedResponseEncodingError(f"content encoding {content_encoding} cannot be safely bounded")
        content = bytearray()
        for chunk in response.iter_bytes():
            if len(content) + len(chunk) > max_response_bytes:
                raise ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
            content.extend(chunk)
        decoded_headers = {
            name: value
            for name, value in response.headers.items()
            if name.lower() not in {"content-encoding", "content-length", "transfer-encoding"}
        }
        try:
            request = response.request
        except RuntimeError:
            request = None
        return httpx.Response(
            response.status_code,
            headers=decoded_headers,
            content=bytes(content),
            request=request,
            extensions=response.extensions,
            history=response.history,
            default_encoding=response.default_encoding,
        )
    finally:
        response.close()


def get(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("GET", url, max_retries=max_retries, **kwargs)


def post(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("POST", url, max_retries=max_retries, **kwargs)


def put(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("PUT", url, max_retries=max_retries, **kwargs)


def patch(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("PATCH", url, max_retries=max_retries, **kwargs)


def delete(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("DELETE", url, max_retries=max_retries, **kwargs)


def head(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
    return make_request("HEAD", url, max_retries=max_retries, **kwargs)


class SSRFProxy:
    """
    Adapter exposing SSRF-protected HTTP helpers behind HttpClientProtocol.

    This is intentionally a thin wrapper over the existing module-level functions so callers can inject it
    where a protocol-typed HTTP client is expected.
    """

    @property
    def max_retries_exceeded_error(self) -> type[Exception]:
        return max_retries_exceeded_error

    @property
    def request_error(self) -> type[Exception]:
        return request_error

    def get(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return get(url=url, max_retries=max_retries, **kwargs)

    def head(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return head(url=url, max_retries=max_retries, **kwargs)

    def post(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return post(url=url, max_retries=max_retries, **kwargs)

    def put(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return put(url=url, max_retries=max_retries, **kwargs)

    def delete(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return delete(url=url, max_retries=max_retries, **kwargs)

    def patch(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
        return patch(url=url, max_retries=max_retries, **kwargs)


def _to_graphon_http_response(response: httpx.Response) -> HttpResponse:
    """Convert an ``httpx`` response into Graphon's transport-agnostic wrapper."""
    return HttpResponse(
        status_code=response.status_code,
        headers=dict(response.headers),
        content=response.content,
        url=str(response.url) if response.url else None,
        reason_phrase=response.reason_phrase,
        fallback_text=response.text,
    )


class GraphonSSRFProxy:
    """Adapter exposing SSRF helpers behind Graphon's ``HttpClientProtocol``."""

    @property
    def max_retries_exceeded_error(self) -> type[Exception]:
        return max_retries_exceeded_error

    @property
    def request_error(self) -> type[Exception]:
        return request_error

    def get(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(get(url=url, max_retries=max_retries, **kwargs))

    def head(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(head(url=url, max_retries=max_retries, **kwargs))

    def post(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(post(url=url, max_retries=max_retries, **kwargs))

    def put(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(put(url=url, max_retries=max_retries, **kwargs))

    def delete(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(delete(url=url, max_retries=max_retries, **kwargs))

    def patch(self, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> HttpResponse:
        return _to_graphon_http_response(patch(url=url, max_retries=max_retries, **kwargs))


ssrf_proxy = SSRFProxy()
graphon_ssrf_proxy = GraphonSSRFProxy()