Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

11.3 KB
Raw Blame
"""Tests for the SSRF-guarded webhook sender (spec §6.7).

``requests`` is patched at the ``webhook_notify`` import site; DNS is patched
inside ``unstract.core.network.ssrf``, which is where the egress decision now
lives (this module no longer has a resolver of its own -- Greptile review #4
replaced its hand-rolled ``_host_is_public`` with the shared guard). So no real
network/DNS activity occurs.

The behaviours asserted here are therefore the SINK's contract -- scheme
policy, the ``allow_insecure`` waiver, status-code handling, and the fact that
an unsafe destination never reaches ``requests.post``. The guard's own
address-classification rules are covered by
``unstract/core/tests/test_ssrf_guard.py``; the cases kept below are the ones
this sink must keep refusing regardless of which implementation is behind it.
"""

import socket
from unittest import mock

from shared.utils import webhook_notify as wn

from unstract.core.network import ssrf


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_private_ip_refused(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("10.0.0.5", 443))]
    assert wn.send_webhook("https://internal.example/x", {"a": 1}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_metadata_ip_refused(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("169.254.169.254", 80))]
    assert wn.send_webhook("https://md.example/x", {}) is False


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_cgnat_shared_address_space_refused(m_gai, m_requests):
    """RFC 6598 100.64.0.0/10. `ipaddress.is_private` does not cover this
    range, which is why the old local guard needed a hand-maintained
    `_CGNAT_RANGE` constant for it (and why SonarCloud flagged that hardcoded
    address). The shared guard gets it from `is_global`, an allowlist kept
    against the IANA registries, so the range is covered without naming it.
    """
    m_gai.return_value = [(2, 1, 6, "", ("100.64.0.1", 443))]
    assert wn.send_webhook("https://cgnat.example/x", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_nat64_embedded_loopback_refused(m_gai, m_requests):
    """``64:ff9b::7f00:1`` is the RFC 6052 NAT64 prefix carrying 127.0.0.1 in
    its low 32 bits; on a deployment with a NAT64 route, that is what gets
    dialled.

    A REGRESSION GUARD, not a fixed bug: the previous local guard refused this
    too, via ``ipaddress.is_reserved`` (checked on 3.12.9 -- the claim in
    Greptile review #4 that it was delivered to does not hold). It is pinned
    here because that refusal was incidental to a version-dependent flag,
    whereas the shared guard refuses it deliberately by re-checking the
    embedded IPv4. Both implementations pass; only one of them means to.
    """
    m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("64:ff9b::7f00:1", 443, 0, 0))]
    assert wn.send_webhook("https://nat64.example/x", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_nat64_embedded_public_ip_still_allowed(m_gai, m_requests):
    """The companion to the test above, and the case that actually CHANGES
    behaviour.

    ``64:ff9b::5db8:d822`` embeds the public 93.184.216.34 -- a legitimate
    destination through a NAT64 translator. The old guard refused it, by the
    same blanket ``is_reserved`` that refused the loopback one: it could not
    tell the two apart, so it rejected the whole prefix. This test fails
    against that implementation and passes against the shared guard, which
    judges the embedded address.
    """
    m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("64:ff9b::5db8:d822", 443, 0, 0))]
    m_requests.post.return_value.status_code = 200
    assert wn.send_webhook("https://nat64-ok.example/hook", {}) is True


@mock.patch.object(wn, "requests")
def test_localhost_subdomain_refused_without_dns(m_requests):
    """RFC 6761 reserves ``*.localhost`` to loopback, and the shared guard
    decides that WITHOUT a resolver.

    No resolver is patched in this test, which is the point: the old guard had
    no such rule and could only reach the same answer through DNS, so its
    verdict depended on the image's libc (glibc's nss-myhostname answers for
    the whole subtree, musl only for the bare apex, a minimal container for
    neither). Same outcome, reached by luck rather than by rule.
    """
    assert wn.send_webhook("https://api.localhost/hook", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_normal_public_ip_still_allowed(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]
    m_requests.post.return_value.status_code = 200
    assert wn.send_webhook("https://cgnat-sibling.example/x", {}) is True


def test_http_scheme_refused_by_default():
    assert wn.send_webhook("http://example.com/x", {}) is False


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_public_host_posted_no_redirects(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]
    m_requests.post.return_value.status_code = 200
    assert wn.send_webhook("https://example.com/hook", {"job_id": "j"}) is True
    kw = m_requests.post.call_args.kwargs
    assert kw["allow_redirects"] is False
    # A (connect, read) pair, not a scalar. The scalar read as one total
    # budget and was not: `requests` applies it per socket operation.
    assert kw["timeout"] == (wn._CONNECT_TIMEOUT, wn._READ_TIMEOUT)
    assert kw["stream"] is True


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_delivery_error_returns_false_never_raises(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]
    m_requests.post.side_effect = Exception("boom")
    assert wn.send_webhook("https://example.com/hook", {}) is False


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_ipv6_loopback_refused(m_gai, m_requests):
    """Pins the 4-tuple IPv6 sockaddr shape: (addr, port, flowinfo, scope_id)."""
    m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("::1", 443, 0, 0))]
    assert wn.send_webhook("https://v6.example/x", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_ipv6_public_posted(m_gai, m_requests):
    m_gai.return_value = [(socket.AF_INET6, 1, 6, "", ("2606:4700::1111", 443, 0, 0))]
    m_requests.post.return_value.status_code = 200
    assert wn.send_webhook("https://v6.example/hook", {}) is True


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_empty_resolution_refused(m_gai, m_requests):
    m_gai.return_value = []
    assert wn.send_webhook("https://nowhere.example/x", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_multi_record_any_unsafe_refuses(m_gai, m_requests):
    """One public + one private record for the same host: must refuse (ALL
    resolved addresses must be public, not just the first).
    """
    m_gai.return_value = [
        (2, 1, 6, "", ("93.184.216.34", 443)),
        (2, 1, 6, "", ("10.0.0.5", 443)),
    ]
    assert wn.send_webhook("https://mixed.example/x", {}) is False
    assert not m_requests.post.called


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_server_error_status_returns_false(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]
    m_requests.post.return_value.status_code = 500
    assert wn.send_webhook("https://example.com/hook", {}) is False


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_not_found_status_returns_false(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 443))]
    m_requests.post.return_value.status_code = 404
    assert wn.send_webhook("https://example.com/hook", {}) is False


@mock.patch.object(wn, "requests")
@mock.patch.object(ssrf.socket, "getaddrinfo")
def test_allow_http_true_posts_public_host(m_gai, m_requests):
    m_gai.return_value = [(2, 1, 6, "", ("93.184.216.34", 80))]
    m_requests.post.return_value.status_code = 200
    assert wn.send_webhook("http://example.com/hook", {}, allow_http=True) is True
    assert m_requests.post.called


class TestAllowInsecure:
    """``allow_insecure`` waives scheme + public-host guards (test stacks only)."""

    @mock.patch.object(wn, "requests")
    def test_http_private_host_delivered_when_insecure(self, m_requests):
        m_requests.post.return_value = mock.Mock(status_code=200)
        ok = wn.send_webhook(
            "http://host.docker.internal:18099/hook",
            {"job_id": "j", "status": "completed"},
            allow_insecure=True,
        )
        assert ok is True
        m_requests.post.assert_called_once()

    @mock.patch.object(wn, "requests")
    def test_default_still_refuses_http_and_private(self, m_requests):
        assert (
            wn.send_webhook("http://host.docker.internal:18099/hook", {"job_id": "j"})
            is False
        )
        assert not m_requests.post.called


# ---------------------------------------------------------------------------
# The response body is never read.
# ---------------------------------------------------------------------------


def test_the_body_is_never_read_and_the_response_is_closed(monkeypatch):
    """`timeout` is per-read, not total, so reading the body is unbounded work.

    This runs inline on the `ide_callback` consumer -- concurrency 2 by
    default and shared with Prompt Studio -- so a receiver that answers 200
    and then trickles one byte every nine seconds, or returns a very large
    body, held one of those two slots for as long as it liked while resetting
    the read timeout on every chunk. Cheap to trigger with any API key:
    submit, cancel immediately, and the cancellation webhook fires with no LLM
    spend. Reported by @athul-rs on #2317.

    Asserting `stream=True` AND the close, because either alone is
    insufficient: without `stream` the body is already read by the time the
    call returns, and without the close the connection is held until GC.
    """
    captured = {}

    class _Resp:
        status_code = 200

        def __init__(self):
            self.closed = False

        def close(self):
            self.closed = True

        @property
        def text(self):  # pragma: no cover - must never be reached
            raise AssertionError("the response body must not be read")

    resp = _Resp()

    def _post(url, **kwargs):
        captured.update(kwargs)
        return resp

    monkeypatch.setattr(wn.requests, "post", _post)
    monkeypatch.setattr(wn, "is_safe_webhook_url", lambda *a, **k: True)

    assert wn.send_webhook("https://example.com/hook", {"job_id": "j"})

    assert captured["stream"] is True
    assert resp.closed, "the connection must be released without draining the body"
    # A tuple, so connect and read read as separate budgets rather than one
    # total -- which is what the old scalar looked like.
    assert isinstance(captured["timeout"], tuple)