Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

16.0 KB
Raw Blame
"""Disposable web_fetch worker: all DNS, sockets and parsers die with this process.

The parent supervises a 30s wall deadline and RSS ceiling. POSIX adds kernel
CPU/address-space limits; Windows adds a process-memory Job limit. IPC carries
only bounded facts (never cookies, headers, or credentials).
"""
from __future__ import annotations

import hashlib
import json
import os
import re
import sys
import time
from email.utils import parsedate_to_datetime
from urllib.parse import urljoin, urlsplit

from .policy import FetchFailure, parse_url, resolve_target


MEMORY_LIMIT = 768 * 1024 * 1024
_JOB = None


def resource_limits() -> None:
    global _JOB
    if os.name == "posix":
        import resource

        def cap(kind, soft, hard):
            _, inherited = resource.getrlimit(kind)
            if inherited != resource.RLIM_INFINITY:
                soft, hard = min(soft, inherited), min(hard, inherited)
            resource.setrlimit(kind, (soft, hard))

        cap(resource.RLIMIT_CPU, 30, 31)
        # macOS has no enforceable per-process address-space limit here; the
        # parent supervises RSS. Linux additionally enforces this in the kernel.
        if sys.platform != "darwin":
            cap(resource.RLIMIT_AS, MEMORY_LIMIT, MEMORY_LIMIT)
        cap(resource.RLIMIT_FSIZE, 1024 * 1024, 1024 * 1024)
    elif os.name == "nt":
        import ctypes
        from ctypes import wintypes

        class Basic(ctypes.Structure):
            _fields_ = [("per_process", ctypes.c_int64), ("per_job", ctypes.c_int64),
                        ("flags", wintypes.DWORD), ("min_ws", ctypes.c_size_t), ("max_ws", ctypes.c_size_t),
                        ("active", wintypes.DWORD), ("affinity", ctypes.c_size_t), ("priority", wintypes.DWORD),
                        ("scheduling", wintypes.DWORD)]
        class Limits(ctypes.Structure):
            _fields_ = [("basic", Basic), ("io", ctypes.c_uint64 * 6),
                        ("process_memory", ctypes.c_size_t), ("job_memory", ctypes.c_size_t),
                        ("peak_process", ctypes.c_size_t), ("peak_job", ctypes.c_size_t)]
        kernel = ctypes.WinDLL("kernel32", use_last_error=True)
        kernel.CreateJobObjectW.restype = wintypes.HANDLE
        kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
        kernel.SetInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
        kernel.GetCurrentProcess.restype = wintypes.HANDLE
        kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
        _JOB = kernel.CreateJobObjectW(None, None)
        limits = Limits()
        limits.basic.flags = 0x100  # JOB_OBJECT_LIMIT_PROCESS_MEMORY
        limits.process_memory = MEMORY_LIMIT
        if (not _JOB or not kernel.SetInformationJobObject(_JOB, 9, ctypes.byref(limits), ctypes.sizeof(limits))
                or not kernel.AssignProcessToJobObject(_JOB, kernel.GetCurrentProcess())):
            raise FetchFailure("fetch_unavailable")


class OriginCookies:
    def __init__(self):
        self.values = {}

    def receive(self, address, headers):
        for name, value in headers:
            if name.lower() != "set-cookie" or len(value.encode("utf-8")) > 4096:
                continue
            # RFC 6265's response parser has ONE name=value pair. SimpleCookie
            # parses a Cookie header instead and mistakes unknown attributes for
            # more cookies (or drops modern Partitioned/Priority cookies).
            pair, *pieces = value.split(";")
            if "=" not in pair:
                continue
            key, content = (part.strip() for part in pair.split("=", 1))
            if not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", key) or any(ord(c) < 32 or ord(c) > 126 for c in content):
                continue
            attributes = {}
            for piece in pieces:
                name, _, attribute = piece.strip().partition("=")
                attributes[name.lower()] = attribute.strip()
            secure = "secure" in attributes
            domain = attributes.get("domain", "").lstrip(".").lower()
            if (domain and not (address.host == domain or (not address.literal and "." in domain and address.host.endswith("." + domain)))):
                continue
            if secure and address.scheme != "https":
                continue
            if key.startswith(("__Secure-", "__Host-")) and not secure:
                continue
            if key.startswith("__Host-") and (domain or attributes.get("path") != "/"):
                continue
            default_path = urlsplit(address.url).path.rpartition("/")[0] or "/"
            path = attributes.get("path", default_path)
            if not path.startswith("/"):
                path = default_path
            identity = (address.origin, path, key)
            expiry = float("inf")
            try:
                if attributes.get("expires"):
                    expiry = parsedate_to_datetime(attributes["expires"]).timestamp()
            except (ValueError, TypeError, OverflowError):
                pass
            try:
                if re.fullmatch(r"-?[0-9]+", attributes.get("max-age", "")):
                    expiry = time.time() + int(attributes["max-age"])
            except (ValueError, OverflowError):
                pass
            if expiry <= time.time():
                self.values.pop(identity, None)
            elif identity in self.values or len(self.values) < 64:
                # Domain influences acceptance only, never where a cookie goes.
                self.values[identity] = (f"{key}={content}", secure, expiry)

    def header(self, address):
        request_path = urlsplit(address.url).path
        parts, size = [], 0
        for (origin, path, key), (value, secure, expiry) in self.values.items():
            if (origin != address.origin or not (request_path == path or request_path.startswith(path.rstrip("/") + "/"))
                    or (secure and address.scheme != "https") or expiry <= time.time()):
                continue
            # Reserve 16 KiB for the URL and fixed headers under the 64 KiB
            # outgoing-header ceiling. Excess cookies are simply not sent.
            cost = len(value.encode("utf-8")) + 2
            if size + cost <= 48 * 1024:
                parts.append(value)
                size += cost
        return "; ".join(parts)


def fetch_document(url: str, locale: str, exchange, *, remaining_seconds: float = 30) -> dict:
    deadline = time.monotonic() + max(0, min(remaining_seconds, 30))
    # The independent URL/DNS gate must not pay HTTP/parser startup cost or
    # depend on those packages loading. Imports stay inside the bounded fetch.
    from .extract import extract_page
    from .transport import TransferBudget, curl_hop, httpx_hop, native_available, request_headers

    def resolve(value):
        # A genuine DNS failure/full DNS allowance is terminal. Exhausting the
        # attempt's smaller remaining budget is admission/timeout, not a refusal;
        # any next candidate must gate the original URL again. The resolver
        # thread is confined to this disposable process and dies with it.
        import queue
        import threading

        exchange({"event": "phase", "phase": "policy"})
        if parse_url(value).literal:
            result = resolve_target(value)
        else:
            replies = queue.Queue(maxsize=1)
            def lookup():
                try:
                    replies.put((resolve_target(value), None))
                except Exception as error:
                    replies.put((None, error))
            threading.Thread(target=lookup, daemon=True).start()
            window = max(0, min(5, deadline - time.monotonic()))
            try:
                result, error = replies.get(timeout=window)
            except queue.Empty:
                if window < 5:
                    raise FetchFailure("fetch_timeout" if sent else "service_busy", cooldown_suppressed=True) from None
                raise FetchFailure("fetch_dns_unverified", terminal=True) from None
            if error:
                raise error
        exchange({"event": "phase", "phase": "fetch"})
        return result

    budget = TransferBudget(on_response=lambda status: exchange({"event": "response", "http_status": status}))
    cookies, seen = OriginCookies(), set()
    native, reason = native_available()
    exchange({"event": "transport", "transport_degraded": not native, "transport_degraded_reason": reason})
    hop = curl_hop if native else httpx_hop
    original = parse_url(url).url
    current, previous = original, None
    used_follow = set()
    retained = None
    sent = False

    def on_sent():
        nonlocal sent
        if not sent:
            exchange({"event": "sent"})
            sent = True

    for index in range(6):
        # Release/acquire is parent-owned and counted in the attempt deadline.
        # Re-resolve AFTER the lock wait, immediately before each numeric connect.
        address = parse_url(current)
        resolve(current)  # Policy precedes even a host-cooldown skip.
        exchange({"event": "host", "host": address.host})
        target = resolve(current)
        cookie = cookies.header(address)
        # Only effective cookies for this URL may justify a revisit. Unrelated
        # origins/paths and refreshed expiry timestamps cannot disguise a loop.
        visit = (target.address.url, hashlib.sha256(cookie.encode("utf-8")).digest())
        if visit in seen:
            exchange({"event": "phase", "phase": "loop"})
            raise FetchFailure("fetch_target_unreadable")
        seen.add(visit)
        headers = request_headers(degraded=not native, locale=locale, previous_origin=previous, target=target)
        if cookie:
            headers["Cookie"] = cookie
        stage = "fetch"
        allowance = max(0.01, deadline - time.monotonic())
        hop_sent = False

        def mark_hop_sent():
            nonlocal hop_sent
            if not hop_sent:
                on_sent()
                exchange({"event": "hop"})
                hop_sent = True

        try:
            response = hop(target, headers, budget, allowance, mark_hop_sent)
            cookies.receive(address, response.headers)
            # Explicit WAF headers are authoritative on any status (including
            # AWS's 202/405). Do not broaden status-only detection or scan prose.
            if (response.header("cf-mitigated").strip().lower() == "challenge"
                    or response.header("x-amzn-waf-action").strip().lower() in {"challenge", "captcha"}):
                raise FetchFailure("fetch_challenge")
            if response.status == 401:
                raise FetchFailure("fetch_refused", terminal=True)
            if response.status == 429:
                raise FetchFailure("service_rate_limited", retry_after=response.header("retry-after") or None)
            if response.status in {301, 302, 303, 307, 308}:
                location = response.header("location")
                if not location:
                    raise FetchFailure("fetch_target_unreadable")
                previous, current = address.origin, urljoin(address.url, location)
                # Always gate the selected redirect, even at the hop bound.
                resolve(current)
                continue
            if response.status in {403, 404, 410, 408}:
                raise FetchFailure("fetch_target_unreadable")
            if response.status != 200:
                raise FetchFailure("fetch_unavailable" if response.status >= 500 else "fetch_target_unreadable",
                                   retry_after=response.header("retry-after") or None)
            stage = "extract"
            exchange({"event": "phase", "phase": "extract"})
            page = extract_page(response.body, response.header("content-type"), address.url)
            if page.follow_url and page.follow_kind not in used_follow:
                # A forbidden selected alternate/refresh is terminal, not a reason
                # to return retained HTML or disclose the original to a provider.
                resolve(page.follow_url)
                if index < 5:
                    used_follow.add(page.follow_kind)
                    if page.follow_kind == "alternate" and page.content:
                        retained = (page, address.url, len(response.body))
                        exchange({"event": "fallback", "result": {
                            **page.payload(), "final_url": address.url, "source_size_bytes": len(response.body),
                            "transport_degraded": not native, "transport_degraded_reason": reason,
                        }})
                    previous, current = address.origin, page.follow_url
                    continue
            if not page.content.strip():
                raise FetchFailure("fetch_target_unreadable")
            if retained is not None:
                original_page = retained[0]
                page.title = page.title or original_page.title
                page.canonical_url = page.canonical_url or original_page.canonical_url
            result = {**page.payload(), "final_url": address.url, "source_size_bytes": len(response.body)}
            break
        except FetchFailure as error:
            if error.code == "fetch_timeout" and (hop_sent or allowance < 5):
                error.cooldown_suppressed = True
            if error.terminal or retained is None:
                raise
            page, final_url, size = retained
            result = {**page.payload(), "final_url": final_url, "source_size_bytes": size,
                      "cooldown_code": error.code, "retry_after": error.retry_after,
                      "cooldown_suppressed": error.cooldown_suppressed}
            break
        except Exception as error:
            # No target error retries on the other transport. Native-load failure
            # was the only fallback, chosen before the first socket existed.
            code = "fetch_target_unreadable" if stage == "extract" else "fetch_timeout" if "timeout" in type(error).__name__.lower() else "fetch_unavailable"
            # Connect/TLS has an independent 5 s bound. After send, the only
            # timeout is the remaining attempt budget; a shortened connect
            # allowance likewise cannot prove this new host is unhealthy.
            suppressed = code == "fetch_timeout" and (hop_sent or allowance < 5)
            if retained is not None:
                page, final_url, size = retained
                result = {**page.payload(), "final_url": final_url, "source_size_bytes": size,
                          "cooldown_code": code, "cooldown_suppressed": suppressed}
                break
            raise FetchFailure(code, cooldown_suppressed=suppressed) from None
    else:
        exchange({"event": "phase", "phase": "loop"})
        raise FetchFailure("fetch_target_unreadable")
    for field in ("follow_url", "follow_kind"):
        result.pop(field, None)
    return {**result, "transport_degraded": not native, "transport_degraded_reason": reason}


def main() -> None:
    def emit(payload):
        sys.stdout.write(json.dumps(payload, ensure_ascii=True) + "\n")
        sys.stdout.flush()

    def exchange(payload):
        emit(payload)
        if sys.stdin.readline().strip() != "ok":
            # Never return control to a native callback after denied admission.
            os._exit(1)

    try:
        resource_limits()
        request = json.loads(sys.stdin.readline(65536))
        if request["operation"] == "gate":
            target = resolve_target(request["url"])
            result = {"url": target.address.url}
        else:
            result = fetch_document(request["url"], request.get("locale", "en"), exchange,
                                    remaining_seconds=request.get("remaining_seconds", 30))
        emit({"event": "result", "result": result})
    except FetchFailure as error:
        emit({"event": "failure", "error_code": error.code, "terminal": error.terminal,
              "retry_after": error.retry_after, "cooldown_suppressed": error.cooldown_suppressed})
    except Exception:
        emit({"event": "failure", "error_code": "fetch_unavailable", "terminal": False})


if __name__ == "__main__":
    main()