Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

17.7 KB
Raw Blame
import io
import threading
import urllib.error
import unittest
import time
from urllib.parse import parse_qs, unquote, urlsplit
from unittest.mock import patch, MagicMock

from lib import http


class Test429RetryLimit(unittest.TestCase):
    """429 retries must be capped at max_429_retries to avoid wasting latency."""

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")  # Don't actually sleep in tests
    def test_429_retries_limited_to_2_by_default(self, mock_sleep, mock_urlopen):
        """With default max_429_retries=2, should attempt 2 times then raise."""
        error = urllib.error.HTTPError(
            "http://example.com", 429, "Too Many Requests", {}, None
        )
        mock_urlopen.side_effect = error

        with self.assertRaises(http.HTTPError) as ctx:
            http.request("GET", "http://example.com", retries=5)

        self.assertEqual(ctx.exception.status_code, 429)
        # Should be called exactly 2 times (initial + 1 retry), not 5
        self.assertEqual(mock_urlopen.call_count, 2)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_non_429_errors_still_use_full_retries(self, mock_sleep, mock_urlopen):
        """500 errors should still retry up to the full retries count."""
        error = urllib.error.HTTPError(
            "http://example.com", 500, "Internal Server Error", {}, None
        )
        mock_urlopen.side_effect = error

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://example.com", retries=3)

        self.assertEqual(mock_urlopen.call_count, 3)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    @patch("lib.http.time.monotonic", return_value=0.5)
    def test_shared_deadline_stops_retry_before_backoff_crosses_it(
        self,
        _mock_monotonic,
        mock_sleep,
        mock_urlopen,
    ):
        mock_urlopen.side_effect = urllib.error.HTTPError(
            "http://example.com", 500, "Internal Server Error", {}, None
        )

        with self.assertRaises(http.HTTPError) as caught:
            http.request(
                "GET",
                "http://example.com",
                retries=3,
                deadline_monotonic=1.0,
            )

        self.assertEqual(http.health.TIMEOUT, caught.exception.outcome_state)
        self.assertEqual(1, mock_urlopen.call_count)
        mock_sleep.assert_not_called()

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.monotonic", side_effect=[0.0, 0.5, 1.5])
    def test_shared_deadline_rejects_response_that_finishes_late(
        self,
        _mock_monotonic,
        mock_urlopen,
    ):
        mock_urlopen.return_value = _mock_response()

        with self.assertRaises(http.DeadlineExceeded):
            http.request(
                "GET",
                "http://example.com",
                retries=1,
                deadline_monotonic=1.0,
            )

    @patch("lib.http.urllib.request.urlopen")
    def test_shared_deadline_stops_waiting_during_slow_body_read(
        self,
        mock_urlopen,
    ):
        response = _mock_response()

        def slow_read():
            time.sleep(0.2)
            return b'{"ok": true}'

        response.read.side_effect = slow_read
        mock_urlopen.return_value = response
        started = time.monotonic()

        with self.assertRaises(http.DeadlineExceeded):
            http.request(
                "GET",
                "http://example.com",
                retries=1,
                deadline_monotonic=started + 0.02,
            )

        self.assertLess(time.monotonic() - started, 0.12)

    @patch("lib.http.urllib.request.urlopen")
    def test_shared_deadline_stops_waiting_during_error_body_read(self, mock_urlopen):
        release = threading.Event()
        read_started = threading.Event()

        class SlowBody(io.BytesIO):
            def read(self, *args, **kwargs):
                read_started.set()
                release.wait(1)
                return super().read(*args, **kwargs)

        mock_urlopen.side_effect = urllib.error.HTTPError(
            "https://example.com", 500, "Server error", {}, SlowBody(b"error")
        )
        started = time.monotonic()
        try:
            with self.assertRaises(http.DeadlineExceeded):
                http.request(
                    "GET", "https://example.com", retries=1,
                    deadline_monotonic=started + 0.05,
                )
            self.assertTrue(read_started.is_set())
            self.assertLess(time.monotonic() - started, 0.5)
            self.assertEqual(mock_urlopen.call_count, 1)
        finally:
            release.set()

    @patch("lib.http.urllib.request.urlopen")
    def test_error_body_within_deadline_keeps_status_and_body(self, mock_urlopen):
        mock_urlopen.side_effect = urllib.error.HTTPError(
            "https://example.com", 401, "Unauthorized", {}, io.BytesIO(b"bad token")
        )
        with self.assertRaises(http.HTTPError) as caught:
            http.request(
                "GET", "https://example.com", retries=1,
                deadline_monotonic=time.monotonic() + 1,
            )
        self.assertEqual(caught.exception.status_code, 401)
        self.assertEqual(caught.exception.body, "bad token")
        self.assertEqual(caught.exception.outcome_state, http.health.AUTH_FAILED)

    @patch("lib.http.urllib.request.urlopen")
    def test_worker_socket_timeout_is_not_wall_deadline_expiration(
        self,
        mock_urlopen,
    ):
        mock_urlopen.side_effect = TimeoutError("early socket timeout")

        with self.assertRaises(http.HTTPError) as caught:
            http.request(
                "GET",
                "http://example.com",
                retries=1,
                deadline_monotonic=time.monotonic() + 600,
            )

        self.assertNotIsInstance(caught.exception, http.DeadlineExceeded)
        self.assertEqual(http.health.TIMEOUT, caught.exception.outcome_state)


def _mock_response(body: str = '{"ok": true}', status: int = 200):
    resp = MagicMock()
    resp.__enter__ = MagicMock(return_value=resp)
    resp.__exit__ = MagicMock(return_value=False)
    resp.read.return_value = body.encode("utf-8")
    resp.status = status
    return resp


class TestParamsEncoding(unittest.TestCase):
    """request() should urlencode the params dict into the URL."""

    def _sent_url(self, mock_urlopen) -> str:
        request_arg = mock_urlopen.call_args[0][0]
        return request_arg.full_url

    @patch("lib.http.urllib.request.urlopen")
    def test_params_appended_to_url(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search", params={"q": "test", "limit": 10})
        sent_url = self._sent_url(mock_urlopen)
        self.assertIn("q=test", sent_url)
        self.assertIn("limit=10", sent_url)

    @patch("lib.http.urllib.request.urlopen")
    def test_params_appended_with_existing_query_string(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search?api_key=secret", params={"q": "test"})
        sent_url = self._sent_url(mock_urlopen)
        self.assertTrue(sent_url.startswith("https://api.example.com/search?api_key=secret&"))
        self.assertIn("q=test", sent_url)

    @patch("lib.http.urllib.request.urlopen")
    def test_none_values_dropped(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search", params={"q": "test", "filter": None})
        sent_url = self._sent_url(mock_urlopen)
        self.assertIn("q=test", sent_url)
        self.assertNotIn("filter", sent_url)

    @patch("lib.http.urllib.request.urlopen")
    def test_empty_params_leaves_url_unchanged(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search", params={})
        sent_url = self._sent_url(mock_urlopen)
        self.assertEqual(sent_url, "https://api.example.com/search")

    @patch("lib.http.urllib.request.urlopen")
    def test_no_params_kwarg_leaves_url_unchanged(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search")
        sent_url = self._sent_url(mock_urlopen)
        self.assertEqual(sent_url, "https://api.example.com/search")

    @patch("lib.http.urllib.request.urlopen")
    def test_int_and_bool_params_stringified(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search", params={"count": 25, "raw": True})
        sent_url = self._sent_url(mock_urlopen)
        self.assertIn("count=25", sent_url)
        self.assertIn("raw=True", sent_url)

    @patch("lib.http.urllib.request.urlopen")
    def test_arabic_query_param_is_percent_encoded(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        arabic_topic = "اسعار التمريض المنزلي"
        http.get("https://api.scrapecreators.com/v1/tiktok/search/keyword",
                 params={"query": arabic_topic})
        sent_url = self._sent_url(mock_urlopen)
        sent_url.encode("ascii")
        self.assertIn("%D8%A7", sent_url)
        self.assertEqual(parse_qs(urlsplit(sent_url).query)["query"], [arabic_topic])

    @patch("lib.http.urllib.request.urlopen")
    def test_cjk_query_param_is_percent_encoded(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search", params={"q": "人工智能"})
        sent_url = self._sent_url(mock_urlopen)
        sent_url.encode("ascii")
        self.assertEqual(parse_qs(urlsplit(sent_url).query)["q"], ["人工智能"])

    @patch("lib.http.urllib.request.urlopen")
    def test_non_ascii_in_base_url_path_is_percent_encoded(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search/العربية")
        sent_url = self._sent_url(mock_urlopen)
        sent_url.encode("ascii")
        self.assertEqual(unquote(urlsplit(sent_url).path), "/search/العربية")

    @patch("lib.http.urllib.request.urlopen")
    def test_non_ascii_in_raw_url_query_string_is_percent_encoded(self, mock_urlopen):
        mock_urlopen.return_value = _mock_response()
        http.get("https://api.example.com/search?q=العربية")
        sent_url = self._sent_url(mock_urlopen)
        sent_url.encode("ascii")
        self.assertEqual(parse_qs(urlsplit(sent_url).query)["q"], ["العربية"])


class TestDNSResolutionRetry(unittest.TestCase):
    """DNS resolution failures (gaierror) must retry with exponential backoff.

    Caller-passed `retries` values smaller than MIN_DNS_RETRIES are expanded
    on the first gaierror so a transient resolution failure doesn't wipe a
    request just because the caller passed retries=2.
    """

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_gaierror_retries_up_to_min_dns_retries_even_when_caller_passes_fewer(
        self, mock_sleep, mock_urlopen
    ):
        """Caller passed retries=2; gaierror should still get MIN_DNS_RETRIES attempts."""
        import socket
        err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
        mock_urlopen.side_effect = err

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://nonexistent.example", retries=2)

        # Caller passed retries=2, but the budget expanded to MIN_DNS_RETRIES=3.
        self.assertEqual(mock_urlopen.call_count, http.MIN_DNS_RETRIES)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_dns_failure_widens_single_attempt_budget(self, mock_sleep, mock_urlopen):
        """DNS failures widen the budget to MIN_DNS_RETRIES, even from retries=1."""
        import socket
        mock_urlopen.side_effect = urllib.error.URLError(socket.gaierror(-2, "DNS failure"))

        with self.assertRaises(http.HTTPError):
            http.request("GET", "https://example.com", retries=1)

        self.assertEqual(mock_urlopen.call_count, http.MIN_DNS_RETRIES)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_gaierror_succeeds_after_transient_failure(self, mock_sleep, mock_urlopen):
        """gaierror on attempt 1, then success — should NOT raise."""
        import socket
        success_response = MagicMock()
        success_response.read.return_value = b'{"ok": true}'
        success_response.status = 200
        success_response.__enter__ = lambda self: self
        success_response.__exit__ = lambda *args: None

        err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
        mock_urlopen.side_effect = [err, success_response]

        result = http.request("GET", "http://flaky.example", retries=2)

        self.assertEqual(result, {"ok": True})
        self.assertEqual(mock_urlopen.call_count, 2)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_gaierror_uses_exponential_backoff(self, mock_sleep, mock_urlopen):
        """Backoff delays for gaierror should be 1s, 2s, 4s — not the linear default."""
        import socket
        err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
        mock_urlopen.side_effect = err

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://nonexistent.example", retries=4)

        sleep_delays = [call.args[0] for call in mock_sleep.call_args_list]
        self.assertEqual(sleep_delays, [1, 2, 4])
        self.assertEqual(mock_urlopen.call_count, 4)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_non_dns_urlerror_uses_linear_backoff_not_dns_branch(
        self, mock_sleep, mock_urlopen
    ):
        """A URLError that's NOT a gaierror must NOT expand the retry budget."""
        # ConnectionRefusedError-style URLError reason (not gaierror)
        err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
        mock_urlopen.side_effect = err

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://refused.example", retries=2)

        # Caller passed retries=2, and non-DNS URLError doesn't expand it.
        self.assertEqual(mock_urlopen.call_count, 2)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_dns_widening_does_not_leak_into_subsequent_non_dns_urlerror(
        self, mock_sleep, mock_urlopen
    ):
        """Mixed sequence: DNS-then-non-DNS must respect caller's original retries.

        Without the fix, the first gaierror widens effective_retries from 2 to
        MIN_DNS_RETRIES=3, and a subsequent ConnectionRefused on attempt 1
        slips into a third overall attempt — exceeding what the caller asked
        for. Each non-DNS error path must gate on the original `retries`.
        """
        import socket
        dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
        conn_err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
        mock_urlopen.side_effect = [dns_err, conn_err, conn_err]  # 3rd would only fire if budget leaked

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://flaky.example", retries=2)

        # Caller asked for at most 2 attempts. DNS widening must not give us a 3rd.
        self.assertEqual(mock_urlopen.call_count, 2)

    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_dns_widening_does_not_leak_into_subsequent_oserror(
        self, mock_sleep, mock_urlopen
    ):
        """Mixed sequence: DNS-then-OSError must respect caller's original retries."""
        import socket
        dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known"))
        mock_urlopen.side_effect = [dns_err, TimeoutError("timed out"), TimeoutError("timed out")]

        with self.assertRaises(http.HTTPError):
            http.request("GET", "http://flaky.example", retries=2)

        self.assertEqual(mock_urlopen.call_count, 2)


class TestDebugLogRedaction(unittest.TestCase):
    """Debug log lines must not echo credentials the request carried."""

    SECRET = "sk-live-abcdef1234567890"

    def _logged(self, mock_log) -> str:
        return "\n".join(call.args[0] for call in mock_log.call_args_list)

    @patch("lib.http.log")
    @patch("lib.http.urllib.request.urlopen")
    def test_error_body_echoing_bearer_token_is_redacted(self, mock_urlopen, mock_log):
        mock_urlopen.side_effect = urllib.error.HTTPError(
            "https://example.com", 401, "Unauthorized", {},
            io.BytesIO(f"invalid key Bearer {self.SECRET}".encode("utf-8")),
        )

        with self.assertRaises(http.HTTPError):
            http.request(
                "POST", "https://example.com", retries=1,
                headers={"Authorization": f"Bearer {self.SECRET}"},
                json_data={"q": "x"},
            )

        logged = self._logged(mock_log)
        self.assertIn("Error body:", logged)
        self.assertIn("<redacted>", logged)
        self.assertNotIn(self.SECRET, logged)

    @patch("lib.http.log")
    @patch("lib.http.urllib.request.urlopen")
    @patch("lib.http.time.sleep")
    def test_url_error_echoing_api_key_is_redacted(self, _mock_sleep, mock_urlopen, mock_log):
        mock_urlopen.side_effect = urllib.error.URLError(f"proxy refused key {self.SECRET}")

        with self.assertRaises(http.HTTPError):
            http.request(
                "GET", "https://example.com", retries=1,
                headers={"X-Api-Key": self.SECRET},
            )

        logged = self._logged(mock_log)
        self.assertIn("URL Error:", logged)
        self.assertNotIn(self.SECRET, logged)