Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

37.5 KB
Raw Blame
"""WF-LOCAL-003: extraction and local orchestration have no rendering side effects.

Tables distinguish target failures from policy/credential refusals; fake transports
are confined to this fixture, while the socket boundary is tested separately.
"""
import asyncio
import io
import json
import sys
import time

import pytest

from core.web_fetch_worker import worker, transport
from core.handlers.model_hub import local_fetch
from core.web_fetch_worker.extract import extract_page, inert_markdown
from core.web_fetch_worker.policy import CheckedTarget, FetchFailure, parse_url
from core.web_fetch_worker.transport import HopResponse


@pytest.mark.parametrize("body,mime,expected", [
    (b"# Page\n\nReadable text.", "text/markdown", "Readable text."),
    (b"plain text", "text/plain", "plain text"),
    (b'{"hello": [1,2]}', "application/json", '"hello"'),
    (b"{not json", "application/json", "{not json"),
    (b"<html><title>Heading</title><article><h1>Heading</h1><p>A complete readable article explains the test document.</p></article></html>", "text/html", "complete readable article"),
    ("<meta charset='gb18030'><article><p>这是网页的正文,编码由元数据提供。</p></article>".encode("gb18030"), "text/html", "这是网页的正文"),
])
def test_formats_are_read_without_network(body, mime, expected):
    page = extract_page(body, mime, "https://example.com/docs/page")
    assert expected in page.content
    assert page.content_type == "text/markdown"
    assert page.source_content_type == mime


@pytest.mark.parametrize("body,mime,code,terminal", [
    (b"", "text/plain", "fetch_target_unreadable", False),
    (b"  ", "application/json", "fetch_target_unreadable", False),
    (b"", "text/html", "fetch_target_unreadable", False),
    (b"  ", "text/html", "fetch_target_unreadable", False),
    (b"<!-- only a comment -->", "text/html", "fetch_target_unreadable", False),
    (b"PK...", "application/zip", "fetch_unsupported_content", False),
    (b"<html><script src='/app.js'></script><div id='app'></div></html>", "text/html", "fetch_js_shell", False),
    (b"<header><form><input type='password'></form></header><div id='root'></div><script src='/app.js'></script>", "text/html", "fetch_js_shell", False),
    (b"<form id='challenge-form'>Checking</form>", "text/html", "fetch_challenge", False),
    (b"<form><input type='password'></form>", "text/html", "fetch_refused", True),
    (b"Access denied", "text/plain", "fetch_challenge", False),
])
def test_unreadable_formats_fail_over_but_login_is_terminal(body, mime, code, terminal):
    with pytest.raises(FetchFailure) as raised:
        extract_page(body, mime, "https://example.com/")
    assert (raised.value.code, raised.value.terminal) == (code, terminal)


@pytest.mark.parametrize("body,mime,expected", [
    (b"caf\xe9", "text/plain; Charset=ISO-8859-1", "café"),
    (b"caf\xe9", "TEXT/PLAIN; CHARSET=ISO-8859-1", "café"),
    (b"<meta charset='ISO-8859-1'><article><p>A readable caf\xe9 article.</p></article>", "TEXT/HTML", "café"),
])
def test_charset_declarations_are_case_insensitive(body, mime, expected):
    page = extract_page(body, mime, "https://example.com/")
    assert expected in page.content and not page.charset_fallback


def test_markdown_html_keeps_visible_text_and_literal_code_without_active_markup():
    text = '''# Install

<Warning>
Rotate the key before continuing.
</Warning>

<details><summary>Reason</summary>The visible explanation.</details>

<script>never_execute()</script>

```html
<script>literal example</script>
```

Done.'''
    content = inert_markdown(text, "https://example.com/")
    for visible in ("Rotate the key", "Reason", "visible explanation", "<script>literal example</script>", "Done."):
        assert visible in content
    assert "<Warning>" not in content and "<details>" not in content and "never_execute" not in content


@pytest.mark.parametrize("hint", ["/bad path.md", "/bad\\path.md", "/bad\npath.md", "https://example.com:/bad"])
def test_malformed_optional_hint_keeps_readable_html(hint):
    markup = f'<link rel="alternate" type="text/markdown" href="{hint}"><article><p>The original article remains readable.</p></article>'
    page = extract_page(markup.encode(), "text/html", "https://example.com/")
    assert page.follow_url is None and "original article" in page.content


def test_public_article_with_login_widget_and_xhtml_declaration_is_readable():
    text = b'''<?xml version="1.0" encoding="UTF-8"?><html><head><title>Public article</title></head>
    <body><header><form><input type="password">Sign in</form></header><article><h1>Public article</h1>
    <p>This complete readable article is available without authentication. Its header includes an unrelated login widget.</p>
    </article></body></html>'''
    for mime in ("text/html", "application/xhtml+xml"):
        assert "without authentication" in extract_page(text, mime, "https://example.com/").content


@pytest.mark.parametrize("mime", ["text/plain", "text/markdown"])
def test_large_text_is_bounded_before_markdown_token_construction(mime):
    source = ("A readable paragraph with a [link](https://example.com/).\n\n" * 150000).encode()
    page = extract_page(source, mime, "https://example.com/")
    assert page.content_overflow and "readable paragraph" in page.content
    assert len(page.content) < 50000


def test_plain_text_preserves_literal_markup_and_numbered_lines():
    text = 'Usage: tool <input-file> <output-file>\nstd::vector<int> v;\n\n<div>literal</div>\n\n[1]: https://example.com/ref\n1. One\n2. Two'
    page = extract_page(text.encode(), "text/plain", "https://example.com/")
    assert text in page.content
    numbered = inert_markdown("7. Seven\n8. Eight", "https://example.com/")
    assert "7. Seven" in numbered and "8. Eight" in numbered


def test_pdf_text_is_quoted_not_interpreted_as_markup():
    from pypdf import PdfWriter
    from pypdf.generic import DictionaryObject, NameObject, DecodedStreamObject
    writer = PdfWriter()
    page = writer.add_blank_page(width=600, height=800)
    font = DictionaryObject({NameObject("/Type"): NameObject("/Font"), NameObject("/Subtype"): NameObject("/Type1"), NameObject("/BaseFont"): NameObject("/Helvetica")})
    page[NameObject("/Resources")] = DictionaryObject({NameObject("/Font"): DictionaryObject({NameObject("/F1"): font})})
    stream = DecodedStreamObject()
    stream.set_data(b"BT /F1 12 Tf 20 700 Td (tool <input-file> std::vector<int>) Tj ET")
    page[NameObject("/Contents")] = stream
    output = io.BytesIO()
    writer.write(output)
    result = extract_page(output.getvalue(), "application/pdf", "https://example.com/")
    assert "[Page 1]" in result.content and "tool <input-file> std::vector<int>" in result.content


def test_inert_links_images_html_and_metadata():
    page = extract_page(b'''<html><title>A title</title><link rel="canonical" href="../canonical">
    <article><h1>A heading</h1><p>Readable article with <a href="../next">next page</a>
    and <a href="javascript:alert(1)">unsafe link</a>, and meaningful paragraph text.</p>
    <script>alert('secret')</script><img src="https://images.example/x"><form>unwanted</form></article></html>''',
                        "text/html", "https://example.com/docs/page")
    assert page.title == "A title"
    assert page.canonical_url == "https://example.com/canonical"
    assert "https://example.com/next" in page.content
    for unsafe in ("javascript:", "<script", "images.example", "unwanted", "secret"):
        assert unsafe not in page.content
    result = inert_markdown("[relative](../x) ![alt](https://img.example/x) <script>bad()</script> [ref][r]\n\n[r]: https://good.example/x", "https://example.com/docs/p")
    assert "https://example.com/x" in result and "https://good.example/x" in result
    assert "img.example" not in result and "bad()" not in result


def test_exact_challenge_text_inside_document_is_not_a_refusal():
    page = extract_page(b"This article discusses the message Access denied in a web application.", "text/plain", "https://example.com/")
    assert "Access denied" in page.content


def test_charset_fallback_alternate_and_refresh():
    assert extract_page(b"text \xff", "text/plain", "https://example.com/").charset_fallback
    for kind, markup in (("alternate", '<link rel="alternate" type="text/markdown" href="/page.md">'),
                         ("refresh", '<meta http-equiv="refresh" content="0; url=/next">')):
        page = extract_page((markup + "<article><p>Readable document for the alternative format.</p></article>").encode(),
                            "text/html", "https://example.com/")
        assert page.follow_kind == kind
        assert page.follow_url.startswith("https://example.com/")
    page = extract_page(b'<link rel="alternate" type="text/markdown" href="https://other.example/x"><p>Retained text.</p>',
                        "text/html", "https://example.com/")
    assert page.follow_url is None


def test_encrypted_and_textless_pdf_fail_over():
    from pypdf import PdfWriter
    for encrypted in (False, True):
        writer = PdfWriter()
        writer.add_blank_page(width=100, height=100)
        if encrypted:
            writer.encrypt("not-available")
        data = io.BytesIO()
        writer.write(data)
        with pytest.raises(FetchFailure, match="fetch_target_unreadable"):
            extract_page(data.getvalue(), "application/pdf", "https://example.com/document")


@pytest.fixture
def local_fixture(monkeypatch):
    calls, gates, events = [], [], []
    responses = {}
    def resolve(url):
        address = parse_url(url)  # Real syntax/literal policy, only DNS is fixture-owned.
        gates.append(address.url)
        return CheckedTarget(address, ("8.8.8.8",))
    def hop(target, headers, budget, timeout, on_sent):
        on_sent()
        calls.append((target.address.url, headers))
        item = responses[target.address.url]
        if isinstance(item, Exception):
            raise item
        return item
    monkeypatch.setattr(worker, "resolve_target", resolve)
    monkeypatch.setattr(transport, "native_available", lambda: (True, None))
    monkeypatch.setattr(transport, "curl_hop", hop)
    monkeypatch.setattr(transport, "httpx_hop", lambda *a: pytest.fail("sent target failure retried on degraded transport"))
    return responses, calls, gates, events


def test_cookie_jar_bounds_outgoing_header_and_validates_prefixes():
    jar = worker.OriginCookies()
    address = parse_url("https://example.com/doc")
    jar.receive(address, [("set-cookie", f"cookie{i}={'x' * 3900}; Path=/") for i in range(70)])
    assert len(jar.values) == 64
    assert len(jar.header(address).encode()) <= 48 * 1024
    jar = worker.OriginCookies()
    jar.receive(address, [("set-cookie", "__Host-bad=1; Path=/"),
                          ("set-cookie", "__Secure-bad=1"),
                          ("set-cookie", "good=1; Secure; Path=/")])
    assert jar.header(address) == "good=1"


def test_rfc_response_cookies_ignore_attributes_but_remain_origin_only():
    jar = worker.OriginCookies()
    address = parse_url("https://www.example.com/doc")
    jar.receive(address, [("set-cookie", 'prefs={"a":1}; Domain=.example.com; Path=/; Secure; HttpOnly; Priority=High; foo=bar'),
                          ("set-cookie", "partitioned=yes; Secure; Partitioned; Path=/")])
    assert jar.header(address) == 'prefs={"a":1}; partitioned=yes'
    assert jar.header(parse_url("https://other.example.com/doc")) == ""
    assert jar.header(parse_url("https://example.com/doc")) == ""


def test_redirect_cookie_scope_and_one_logical_send(local_fixture):
    responses, calls, gates, events = local_fixture
    responses.update({
        "https://example.com/start": HopResponse(302, [("location", "/next"), ("set-cookie", "visit=1; Path=/; Secure")], b""),
        "https://example.com/next": HopResponse(302, [("location", "https://other.example/final")], b""),
        "https://other.example/final": HopResponse(200, [("content-type", "text/markdown")], b"# Document\n\nUseful content."),
    })
    result = worker.fetch_document("https://example.com/start", "en", events.append)
    assert result["final_url"] == "https://other.example/final"
    assert calls[1][1]["Cookie"] == "visit=1" and "Cookie" not in calls[2][1]
    assert [e for e in events if e["event"] == "sent"] == [{"event": "sent"}]
    assert all(url in gates for url, _ in calls)
    events.clear()
    worker.fetch_document("https://other.example/final", "en", events.append)
    assert "Cookie" not in calls[-1][1]


def test_native_load_failure_selects_honest_transport_before_send(local_fixture, monkeypatch):
    responses, calls, gates, events = local_fixture
    monkeypatch.setattr(transport, "native_available", lambda: (False, "native_library_unavailable"))
    def degraded(target, headers, budget, timeout, on_sent):
        assert headers["User-Agent"].startswith("Avibe/")
        assert not any(key.lower().startswith(("sec-ch-", "sec-fetch-")) for key in headers)
        on_sent()
        return HopResponse(200, [("content-type", "text/plain")], b"Readable degraded result.")
    monkeypatch.setattr(transport, "httpx_hop", degraded)
    monkeypatch.setattr(transport, "curl_hop", lambda *a: pytest.fail("unavailable native library was used"))
    result = worker.fetch_document("https://example.com/", "en", events.append)
    assert result["transport_degraded"] and result["transport_degraded_reason"] == "native_library_unavailable"


@pytest.mark.parametrize("status,code,terminal", [
    (401, "fetch_refused", True), (403, "fetch_target_unreadable", False),
    (404, "fetch_target_unreadable", False), (410, "fetch_target_unreadable", False),
    (429, "service_rate_limited", False), (503, "fetch_unavailable", False),
])
def test_local_http_outcome_table(local_fixture, status, code, terminal):
    responses, calls, gates, events = local_fixture
    responses["https://example.com/"] = HopResponse(status, [("retry-after", "3600")], b"not content")
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append)
    assert (raised.value.code, raised.value.terminal) == (code, terminal)
    assert len(calls) == 1
    if status in {429, 503}:
        assert raised.value.retry_after == "3600"


def test_503_explicit_challenge_has_no_transient_host_class(local_fixture):
    responses, calls, gates, events = local_fixture
    responses["https://example.com/"] = HopResponse(503, [("cf-mitigated", "challenge")], b"challenge")
    with pytest.raises(FetchFailure, match="fetch_challenge"):
        worker.fetch_document("https://example.com/", "en", events.append)


@pytest.mark.parametrize("status,header,value,code", [
    (202, "x-amzn-waf-action", "challenge", "fetch_challenge"),
    (405, "x-amzn-waf-action", "captcha", "fetch_challenge"),
    (200, "X-Amzn-Waf-Action", "CHALLENGE", "fetch_challenge"),
    (401, "x-amzn-waf-action", "captcha", "fetch_challenge"),
    (429, "x-amzn-waf-action", "challenge", "fetch_challenge"),
    (302, "x-amzn-waf-action", "challenge", "fetch_challenge"),
    (503, "cf-mitigated", "challenge", "fetch_challenge"),
    (200, "cf-mitigated", "challenge", "fetch_challenge"),
    (202, "x-amzn-waf-action", "allow", "fetch_target_unreadable"),
    (405, "unrelated", "captcha", "fetch_target_unreadable"),
    (503, "cf-mitigated", "other", "fetch_unavailable"),
    (403, "cf-mitigated", "other", "fetch_target_unreadable"),
    (403, "cf-mitigated", "challenge", "fetch_challenge"),
])
def test_explicit_waf_challenge_headers_on_any_status(local_fixture, status, header, value, code):
    """Header facts, not page prose or a broad status set, identify challenges."""
    responses, calls, _, events = local_fixture
    responses["https://example.com/"] = HopResponse(status, [(header, value)], b"")
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append)
    assert raised.value.code == code and not raised.value.terminal
    assert len(calls) == 1


def test_redirect_to_private_terminates_without_second_connect(local_fixture):
    responses, calls, gates, events = local_fixture
    responses["https://example.com/"] = HopResponse(302, [("location", "http://169.254.169.254/latest")], b"")
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append)
    assert (raised.value.code, raised.value.terminal) == ("fetch_url_blocked", True)
    assert len(calls) == 1


@pytest.mark.parametrize("alternate_outcome", ["unreadable", "blocked", "success", "rate_limited"])
def test_one_markdown_alternate_retains_original_except_policy_refusal(local_fixture, monkeypatch, alternate_outcome):
    responses, calls, gates, events = local_fixture
    responses["https://example.com/"] = HopResponse(200, [("content-type", "text/html")],
        b'<title>Original title</title><link rel="canonical" href="/canonical"><link rel="alternate" type="text/markdown" href="/alternate"><article><p>Original readable HTML article text.</p></article>')
    responses["https://example.com/alternate"] = (HopResponse(200, [("content-type", "text/markdown")], b"Better Markdown body.")
                                               if alternate_outcome == "success" else HopResponse(404, [], b""))
    if alternate_outcome == "rate_limited":
        responses["https://example.com/alternate"] = HopResponse(429, [("retry-after", "3600")], b"")
    if alternate_outcome == "blocked":
        original = worker.resolve_target
        def resolve(url):
            if url.endswith("/alternate"):
                raise FetchFailure("fetch_dns_unverified", terminal=True)
            return original(url)
        monkeypatch.setattr(worker, "resolve_target", resolve)
        with pytest.raises(FetchFailure, match="fetch_dns_unverified"):
            worker.fetch_document("https://example.com/", "en", events.append)
        assert len(calls) == 1
    else:
        result = worker.fetch_document("https://example.com/", "en", events.append)
        assert ("Better Markdown" if alternate_outcome == "success" else "Original readable") in result["content"]
        if alternate_outcome == "rate_limited":
            assert result["cooldown_code"] == "service_rate_limited" and result["retry_after"] == "3600"
        assert result["title"] == "Original title" and result["canonical_url"] == "https://example.com/canonical"
        assert len(calls) == 2


@pytest.mark.parametrize("shape", ["loop", "missing_location", "hop_limit"])
def test_redirect_protocol_failure_is_target_scoped(local_fixture, shape):
    responses, _, _, events = local_fixture
    if shape == "loop":
        responses["https://example.com/"] = HopResponse(302, [("location", "/")], b"")
    elif shape == "missing_location":
        responses["https://example.com/"] = HopResponse(302, [], b"")
    else:
        for n in range(6):
            url = "https://example.com/" + (str(n) if n else "")
            responses[url] = HopResponse(302, [("location", f"/{n + 1}")], b"")
    with pytest.raises(FetchFailure, match="fetch_target_unreadable"):
        worker.fetch_document("https://example.com/", "en", events.append)


def test_short_attempt_dns_deadline_differs_from_real_dns_failure(local_fixture, monkeypatch):
    _, _, _, events = local_fixture
    original = worker.resolve_target
    def slow(url):
        time.sleep(.04)
        return original(url)
    monkeypatch.setattr(worker, "resolve_target", slow)
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append, remaining_seconds=.005)
    assert raised.value.code == "service_busy" and not raised.value.terminal and raised.value.cooldown_suppressed
    def unavailable(url):
        raise FetchFailure("fetch_dns_unverified", terminal=True)
    monkeypatch.setattr(worker, "resolve_target", unavailable)
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append)
    assert raised.value.code == "fetch_dns_unverified" and raised.value.terminal


def test_gate_worker_does_not_load_http_or_extraction_stack(tmp_path):
    """The mandatory gate works even when no local fetch stack can be loaded."""
    import subprocess
    from pathlib import Path

    code = '''import importlib.abc,sys
sys.path.insert(0,sys.argv.pop(1))
class PolicyOnly(importlib.abc.MetaPathFinder):
 def find_spec(self,fullname,path=None,target=None):
  if fullname.rsplit(".",1)[-1] in {"transport","extract"} or fullname.split(".",1)[0] in {"httpx","httpcore","curl_cffi","lxml","trafilatura","pypdf"}:
   raise AssertionError("gate attempted unrelated import: " + fullname)
sys.meta_path.insert(0,PolicyOnly())
from core.web_fetch_worker.worker import main
main()
'''
    result = subprocess.run(
        [sys.executable, "-I", "-B", "-c", code, str(Path(__file__).resolve().parents[1])],
        input=json.dumps({"operation": "gate", "url": "https://8.8.8.8/"}) + "\n",
        capture_output=True, text=True, cwd=tmp_path, timeout=5,
    )
    assert result.returncode == 0, result.stderr
    assert json.loads(result.stdout) == {"event": "result", "result": {"url": "https://8.8.8.8/"}}


async def test_gate_worker_and_initial_policy_failure_do_not_send():
    engine = local_fetch.LocalFetchEngine()
    assert await engine.check("https://8.8.8.8/", deadline=time.monotonic() + 5) == "https://8.8.8.8/"
    with pytest.raises(FetchFailure, match="fetch_url_blocked"):
        await engine.fetch("http://127.0.0.1/", deadline=time.monotonic() + 5)


async def test_host_lock_serializes_requests_and_scrubs_worker_environment(monkeypatch):
    monkeypatch.setenv("OPENAI_API_KEY", "test-owned-value-must-not-be-inherited")
    script = '''import sys,time,json,os
assert "OPENAI_API_KEY" not in os.environ
sys.stdin.readline()
print(json.dumps({"event":"host","host":"example.com"}),flush=True); sys.stdin.readline()
print(json.dumps({"event":"sent"}),flush=True); sys.stdin.readline()
time.sleep(.2)
print(json.dumps({"event":"result","result":{"content":"done"}}),flush=True)
'''
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine, starts = local_fetch.LocalFetchEngine(), []
    results = await asyncio.gather(*(engine.fetch("https://example.com/", deadline=time.monotonic() + 5,
                                                  on_sent=lambda: starts.append(time.monotonic())) for _ in range(2)))
    assert all(row["sent"] for row in results)
    assert len(starts) == 2 and starts[1] - starts[0] >= .18


@pytest.mark.parametrize("retained_html", [False, True])
async def test_local_retry_after_is_host_scoped_and_degraded_failure_is_recorded(monkeypatch, retained_html):
    script = '''import sys,json
from urllib.parse import urlsplit
request=json.loads(sys.stdin.readline())
for event in [{"event":"transport","transport_degraded":True,"transport_degraded_reason":"native_library_unavailable"},
              {"event":"host","host":urlsplit(request["url"]).hostname}, {"event":"sent"}]:
 print(json.dumps(event),flush=True); sys.stdin.readline()
print(json.dumps({"event":"failure","error_code":"service_rate_limited","retry_after":"3600","terminal":False}),flush=True)
'''
    if retained_html:
        script = script.replace('{"event":"failure","error_code":"service_rate_limited","retry_after":"3600","terminal":False}',
                                '{"event":"result","result":{"content":"retained HTML","cooldown_code":"service_rate_limited","retry_after":"3600"}}')
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine, sent = local_fetch.LocalFetchEngine(), []
    async def fetch(host):
        return await engine.fetch(f"https://{host}/", deadline=time.monotonic() + 5, on_sent=lambda: sent.append(host))
    first, second, other = await fetch("example.com"), await fetch("example.com"), await fetch("other.example")
    assert first["transport_degraded"] and first["transport_degraded_reason"] == "native_library_unavailable"
    if retained_html:
        assert first["content"] == "retained HTML" and "error_code" not in first and "cooldown_code" not in first
    assert engine._cooldowns["example.com"][0] >= time.time() + 3590
    assert second["skipped"] == "cooling" and not second["sent"]
    assert other["sent"] and sent == ["example.com", "other.example"]


@pytest.mark.parametrize("retained_html", [False, True])
async def test_queued_redirect_observes_rate_limit_before_host_is_released(monkeypatch, retained_html):
    """A queued redirect cannot slip between worker cleanup and health publication."""
    limited_result = ({"event": "result", "result": {"content": "retained HTML", "cooldown_code": "service_rate_limited", "retry_after": "3600"}}
                      if retained_html else {"event": "failure", "error_code": "service_rate_limited", "retry_after": "3600", "terminal": False})
    script = '''import sys,json
from urllib.parse import urlsplit
request=json.loads(sys.stdin.readline())
host=urlsplit(request["url"]).hostname
for event in [{"event":"host","host":host}, {"event":"sent"}]:
 print(json.dumps(event),flush=True); sys.stdin.readline()
if host == "limited.example":
 print(json.dumps(LIMITED_RESULT),flush=True)
else:
 print(json.dumps({"event":"host","host":"limited.example"}),flush=True)
 if sys.stdin.readline():
  print(json.dumps({"event":"result","result":{"content":"bypassed Retry-After"}}),flush=True)
'''.replace("LIMITED_RESULT", repr(limited_result))
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine = local_fetch.LocalFetchEngine()
    started, queued = asyncio.Event(), asyncio.Event()
    change_host = engine._change_host
    async def observe_redirect(host, state):
        if host == "limited.example" and state["held_host"] == "other.example":
            queued.set()
        await change_host(host, state)
    monkeypatch.setattr(engine, "_change_host", observe_redirect)
    async def first_sent():
        started.set()
        await queued.wait()
    first = asyncio.create_task(engine.fetch("https://limited.example/", deadline=time.monotonic() + 5, on_sent=first_sent))
    second = None
    try:
        await asyncio.wait_for(started.wait(), 3)
        second = asyncio.create_task(engine.fetch("https://other.example/", deadline=time.monotonic() + 5))
        limited, redirected = await asyncio.gather(first, second)
        assert limited.get("content") == "retained HTML" if retained_html else limited["error_code"] == "service_rate_limited"
        assert redirected.get("skipped") == "cooling" and "content" not in redirected
        assert engine._cooldowns["limited.example"][0] >= time.time() + 3590
        assert not engine._hosts and engine._workers["fetch"] == 0
    finally:
        for task in (first, second):
            if task is not None:
                task.cancel()
        await asyncio.gather(*(task for task in (first, second) if task is not None), return_exceptions=True)


async def test_busy_host_does_not_starve_policy_gate_or_other_hosts(monkeypatch):
    script = '''import sys,time,json
from urllib.parse import urlsplit
r=json.loads(sys.stdin.readline())
if r["operation"] == "gate":
 print(json.dumps({"event":"result","result":{"url":r["url"]}}),flush=True); sys.exit()
host=urlsplit(r["url"]).hostname
print(json.dumps({"event":"host","host":host}),flush=True); sys.stdin.readline()
print(json.dumps({"event":"sent"}),flush=True); sys.stdin.readline()
if host == "example.com": time.sleep(60)
print(json.dumps({"event":"result","result":{"content":"other host"}}),flush=True)
'''
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine, started = local_fetch.LocalFetchEngine(), asyncio.Event()
    spawned, spawn = [], asyncio.create_subprocess_exec
    async def capture(*args, **kwargs):
        spawned.append(1)
        return await spawn(*args, **kwargs)
    monkeypatch.setattr(asyncio, "create_subprocess_exec", capture)
    tasks = [asyncio.create_task(engine.fetch("https://example.com/", deadline=time.monotonic() + 10, on_sent=started.set)) for _ in range(40)]
    try:
        await asyncio.wait_for(started.wait(), 5)
        assert await engine.check("https://other.example/", deadline=time.monotonic() + 2) == "https://other.example/"
        other = await engine.fetch("https://other.example/", deadline=time.monotonic() + 2)
        assert other["sent"] and other["content"] == "other host"
        queued = await engine.fetch("https://example.com/", deadline=time.monotonic() + .3)
        assert queued["error_code"] == "service_busy" and not queued["sent"]
        assert not engine._cooldowns
        assert len(spawned) <= 4  # One same-host worker, independent gate and other host; not 40 parked workers.
    finally:
        for task in tasks:
            task.cancel()
        await asyncio.gather(*tasks, return_exceptions=True)


async def test_parent_memory_limit_kills_worker_and_keeps_known_alternate_fallback(monkeypatch):
    script = '''import sys,time,json
sys.stdin.readline()
print(json.dumps({"event":"sent"}),flush=True); sys.stdin.readline()
print(json.dumps({"event":"fallback","result":{"content":"original HTML","final_url":"https://example.com/"}}),flush=True); sys.stdin.readline()
time.sleep(60)
'''
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine = local_fetch.LocalFetchEngine()
    result = await engine.fetch("https://example.com/", deadline=time.monotonic() + .5)
    assert result["sent"] and result["content"] == "original HTML" and "error_code" not in result
    # Separate actual RSS-supervision case: lower only this fixture's threshold.
    monkeypatch.setattr(local_fetch, "_MEMORY_LIMIT", 1)
    monkeypatch.setattr(local_fetch, "_WORKER", "import sys,time; sys.stdin.readline(); time.sleep(60)")
    started = time.monotonic()
    result = await engine.fetch("https://example.com/", deadline=time.monotonic() + 5)
    assert result["error_code"] == "fetch_unavailable" and time.monotonic() - started < 3


@pytest.mark.parametrize("sent", [False, True])
@pytest.mark.parametrize("phase", ["policy", "fetch"])
async def test_attempt_expiry_during_revalidation_is_not_a_dns_refusal(monkeypatch, sent, phase):
    script = '''import sys,time,json
sys.stdin.readline()
for event in EVENTS:
 print(json.dumps(event),flush=True); sys.stdin.readline()
time.sleep(60)
'''.replace("EVENTS", repr([{"event": "host", "host": "example.com"}] +
                        ([{"event": "sent"}] if sent else []) + [{"event": "phase", "phase": phase}]))
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    engine = local_fetch.LocalFetchEngine()
    result = await engine.fetch("https://example.com/", deadline=time.monotonic() + .3)
    code = "service_busy" if not sent and phase == "policy" else "fetch_timeout"
    assert result["error_code"] == code and not result["terminal"]
    assert result["sent"] == sent and not engine._cooldowns
    if code == "service_busy":
        assert result["skipped"] == "unavailable"
    with pytest.raises(FetchFailure, match="fetch_dns_unverified"):
        await engine.check("https://example.com/", deadline=time.monotonic() + .3)


@pytest.mark.parametrize("remaining,elapsed,sent,timeout_type,suppressed", [
    (30, 5, False, "connect", False),
    (1, 1, False, "connect", True),
    (30, 30, True, "read", True),
])
def test_hop_timeout_only_cools_when_it_had_its_own_connect_budget(
    local_fixture, monkeypatch, remaining, elapsed, sent, timeout_type, suppressed
):
    import httpcore
    from types import SimpleNamespace
    _, _, _, events = local_fixture
    clock = SimpleNamespace(now=0)
    monkeypatch.setattr(worker, "time", SimpleNamespace(monotonic=lambda: clock.now, time=time.time))

    def timed_out(target, headers, budget, timeout, on_sent):
        if sent:
            on_sent()
        clock.now += elapsed
        raise (httpcore.ReadTimeout if timeout_type == "read" else httpcore.ConnectTimeout)()

    monkeypatch.setattr(transport, "curl_hop", timed_out)
    with pytest.raises(FetchFailure) as raised:
        worker.fetch_document("https://example.com/", "en", events.append, remaining_seconds=remaining)
    assert raised.value.code == "fetch_timeout" and raised.value.cooldown_suppressed == suppressed

async def test_stop_during_success_cleanup_is_rethrown_after_reap(monkeypatch):
    script = 'import sys; sys.stdin.readline(); print(\'{"event":"result","result":{"content":"page"}}\',flush=True)'
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    cleaning, release = asyncio.Event(), asyncio.Event()
    async def monitor(process):
        try:
            await asyncio.Future()
        except asyncio.CancelledError:
            cleaning.set()
            await release.wait()
            raise
    monkeypatch.setattr(local_fetch.LocalFetchEngine, "_monitor", staticmethod(monitor))
    task = asyncio.create_task(local_fetch.LocalFetchEngine().fetch("https://example.com/", deadline=time.monotonic() + 5))
    await asyncio.wait_for(cleaning.wait(), 5)
    task.cancel()
    await asyncio.sleep(0)
    assert not task.done()
    release.set()
    with pytest.raises(asyncio.CancelledError):
        await task


@pytest.mark.parametrize("spawn_delay", [0, 0.4], ids=["normal-start", "slow-start"])
async def test_active_worker_cap_and_cancellation_safe_host_release(monkeypatch, spawn_delay):
    script = '''import sys,time,json
from urllib.parse import urlsplit
request=json.loads(sys.stdin.readline())
print(json.dumps({"event":"host","host":urlsplit(request["url"]).hostname}),flush=True); sys.stdin.readline()
print('{"event":"sent"}',flush=True); sys.stdin.readline()
time.sleep(60)
'''
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    children, spawn, spawn_attempts = [], asyncio.create_subprocess_exec, []
    reaping, release = asyncio.Event(), asyncio.Event()
    async def capture(*args, **kwargs):
        spawn_attempts.append(1)
        if spawn_delay:
            await asyncio.sleep(spawn_delay)
        child = await spawn(*args, **kwargs)
        children.append(child)
        if len(children) == 1:
            wait = child.wait
            async def delayed_reap():
                reaping.set()
                await release.wait()
                return await wait()
            monkeypatch.setattr(child, "wait", delayed_reap)
        return child
    monkeypatch.setattr(asyncio, "create_subprocess_exec", capture)
    engine, began = local_fetch.LocalFetchEngine(), asyncio.Event()
    first = asyncio.create_task(engine.fetch("https://example.com/", deadline=time.monotonic() + 10, on_sent=began.set))
    await asyncio.wait_for(began.wait(), 5)
    first.cancel()
    await asyncio.wait_for(reaping.wait(), 5)
    first.cancel()  # Stop arriving during timeout/Stop cleanup must not strand admission.
    release.set()
    with pytest.raises(asyncio.CancelledError):
        await first
    assert children[0].returncode is not None
    began.clear()
    again = asyncio.create_task(engine.fetch("https://example.com/", deadline=time.monotonic() + 10, on_sent=began.set))
    try:
        await asyncio.wait_for(began.wait(), 2)
    finally:
        again.cancel()
        await asyncio.gather(again, return_exceptions=True)
    children.clear()
    spawn_attempts.clear()
    # Distinct hosts still have one aggregate process cap.
    sent, ready = 0, asyncio.Event()
    worker_cap = local_fetch._WORKERS
    def mark_sent():
        nonlocal sent
        sent += 1
        if sent == worker_cap:
            ready.set()
    tasks = [asyncio.create_task(engine.fetch(f"https://host{i}.example/", deadline=time.monotonic() + 10,
                                             on_sent=mark_sent)) for i in range(20)]
    try:
        await asyncio.wait_for(ready.wait(), 10)
        assert sent == worker_cap
        assert len(spawn_attempts) == len(children) == worker_cap == 8
    finally:
        for task in tasks:
            task.cancel()
        await asyncio.gather(*tasks, return_exceptions=True)
    assert all(child.returncode is not None for child in children)


@pytest.mark.skipif(sys.platform == "win32", reason="POSIX inherited rlimits")
def test_worker_respects_stricter_inherited_kernel_limits():
    import subprocess
    from pathlib import Path
    code = '''import resource,sys,json
sys.path.insert(0,sys.argv[1])
from core.web_fetch_worker.worker import resource_limits
resource.setrlimit(resource.RLIMIT_CPU,(20,20))
resource.setrlimit(resource.RLIMIT_FSIZE,(65536,65536))
resource_limits()
print(json.dumps([resource.getrlimit(resource.RLIMIT_CPU),resource.getrlimit(resource.RLIMIT_FSIZE)]))
'''
    result = subprocess.run([sys.executable, "-I", "-B", "-c", code, str(Path(__file__).resolve().parents[1])],
                            capture_output=True, text=True, check=True, timeout=30)
    assert json.loads(result.stdout) == [[20, 20], [65536, 65536]]


async def test_timeout_and_stop_kill_and_reap_worker(monkeypatch):
    # A test-owned worker simulates a parser that cannot yield to cancellation.
    # The real parent must terminate it; no production network override exists.
    script = "import sys,time; sys.stdin.readline(); print('{\"event\":\"sent\"}',flush=True); sys.stdin.readline(); time.sleep(60)"
    monkeypatch.setattr(local_fetch, "_WORKER", script)
    children = []
    spawn = asyncio.create_subprocess_exec
    async def capture(*args, **kwargs):
        child = await spawn(*args, **kwargs)
        children.append(child)
        return child
    monkeypatch.setattr(asyncio, "create_subprocess_exec", capture)
    engine = local_fetch.LocalFetchEngine()
    sent = []
    result = await engine.fetch("https://example.com/", deadline=time.monotonic() + 0.5, on_sent=lambda: sent.append(1))
    assert result["error_code"] == "fetch_timeout" and result["sent"]
    assert sent == [1] and children[0].returncode is not None
    called = asyncio.Event()
    task = asyncio.create_task(engine.fetch("https://example.com/", deadline=time.monotonic() + 30, on_sent=called.set))
    await asyncio.wait_for(called.wait(), 5)
    task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await task
    assert children[1].returncode is not None