Skip to content

Self-hosted AI workspace.

Python
9.2 KB
Raw Blame
"""Research navigation primitives.

Deep Research historically had its own narrow Search -> Fetch path.  This
module gives the research engine a small normalized surface for richer web
navigation while still reusing Odysseus' existing web tooling.
"""

from __future__ import annotations

import asyncio
import json
import logging
import urllib.parse
from dataclasses import dataclass
from typing import Any

from src.constants import MAX_OUTPUT_CHARS

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class ResearchAction:
    """A bounded navigation action the research planner can request."""

    tool: str
    args: dict[str, Any]


@dataclass
class ResearchPage:
    """Normalized readable page content for research extraction."""

    url: str
    title: str = ""
    content: str = ""
    og_image: str = ""
    success: bool = False
    retrieval: str = "fetch"
    error: str = ""


@dataclass(frozen=True)
class ResearchSourceAssessment:
    """Simple quality metadata for a gathered research source."""

    kind: str
    score: int
    reason: str


def parse_research_actions(text: str, *, allowed_tools: set[str] | None = None) -> list[ResearchAction]:
    """Parse a model reply into bounded research actions.

    Accepts either a JSON array directly or an object with an ``actions`` array.
    This is intentionally small and strict so a later model-planned research
    loop can be added without giving the research model arbitrary tool access.
    """
    allowed = allowed_tools or {
        "web_search",
        "web_fetch",
        "browser_open",
        "browser_read",
        "browser_snapshot",
        "private_browser",
    }
    raw = (text or "").strip()
    if raw.startswith("```"):
        raw = raw.removeprefix("```json").removeprefix("```").strip()
        if raw.endswith("```"):
            raw = raw[:-3].strip()
    try:
        parsed = json.loads(raw)
    except json.JSONDecodeError:
        return []
    if isinstance(parsed, dict):
        parsed = parsed.get("actions")
    if not isinstance(parsed, list):
        return []
    actions: list[ResearchAction] = []
    for item in parsed:
        if not isinstance(item, dict):
            continue
        has_tool_field = "tool" in item
        tool = str(item.get("tool") or item.get("action") or "").strip()
        if tool not in allowed:
            continue
        args = item.get("args")
        if not isinstance(args, dict):
            excluded = {"tool"} if has_tool_field else {"action"}
            args = {k: v for k, v in item.items() if k not in excluded}
        actions.append(ResearchAction(tool=tool, args=args))
    return actions


def assess_source(url: str, *, title: str = "", retrieval: str = "", summary: str = "") -> ResearchSourceAssessment:
    """Estimate source usefulness for planning and reporting.

    This is deliberately coarse. The model still judges evidence content; this
    only gives the planner a compact map of whether it has official/primary
    sources or mostly secondary/search-result material.
    """
    parsed = urllib.parse.urlparse(str(url or ""))
    host = (parsed.netloc or parsed.path.split("/", 1)[0]).lower().removeprefix("www.")
    path = (parsed.path or "").lower()
    text = " ".join([host, path, str(title or ""), str(summary or "")]).lower()
    retrieval = (retrieval or "fetch").lower()

    kind = "secondary"
    score = 55
    reason = "secondary web source"

    if host.endswith((".gov", ".edu", ".ac.uk")) or ".gov." in host:
        kind, score, reason = "official", 90, "government/academic domain"
    elif any(part in host for part in ("github.com", "huggingface.co", "docs.", "developer.", "support.")):
        kind, score, reason = "primary", 82, "primary project/vendor source"
    elif any(token in text for token in ("official", "documentation", "docs", "release notes", "press release")):
        kind, score, reason = "primary", 78, "primary-source wording"
    elif any(part in host for part in ("reddit.com", "quora.com", "stackoverflow.com", "news.ycombinator.com")):
        kind, score, reason = "community", 45, "community/forum source"
    elif any(token in text for token in ("affiliate", "coupon", "best-", "top-", "review")):
        kind, score, reason = "commercial", 40, "commercial/listicle source"

    if retrieval == "browser":
        score = min(100, score + 5)
        reason += "; browser-read"
    if not str(summary or "").strip():
        score = max(10, score - 20)
        reason += "; weak extraction"

    return ResearchSourceAssessment(kind=kind, score=score, reason=reason)


class ResearchNavigator:
    """Small web-navigation facade used by Deep Research.

    It delegates to the same search/fetch/browser implementations the chat
    agent uses, but normalizes results for the research extraction pipeline.
    """

    def __init__(
        self,
        *,
        progress_callback=None,
        session_id: str = "",
        search_provider: str | None = None,
    ) -> None:
        self._progress = progress_callback
        self.session_id = session_id
        self.search_provider = (search_provider or "").strip()
        self.providers_used: list[str] = []
        self.last_search_error = ""
        self.browser_fetches = 0

    async def search(self, query: str, *, count: int = 10) -> list[dict[str, Any]]:
        """Run a provider-chain web search and return structured results."""
        try:
            from src.search.providers import _get_search_settings
            from src.search.core import _build_provider_chain, _call_provider

            settings = _get_search_settings()
            provider = self.search_provider or (settings.get("research_search_provider") or "").strip()
            if not provider:
                provider = settings.get("search_provider", "searxng")
            if provider == "disabled":
                logger.info("Search is disabled for research")
                return []

            chain = _build_provider_chain(provider)
            raised = False
            for prov in chain:
                try:
                    results = await asyncio.to_thread(_call_provider, prov, query, count)
                    if results:
                        if prov not in self.providers_used:
                            self.providers_used.append(prov)
                        return results
                except Exception as e:
                    raised = True
                    logger.warning("Research search provider %s failed: %s", prov, e)
                    self.last_search_error = f"{prov}: {e}"
            if not raised:
                self.last_search_error = (
                    "no results from search provider(s): "
                    f"{', '.join(chain) if chain else provider}"
                )
            return []
        except Exception as e:
            logger.error("Research search failed for %r: %s", query, e)
            self.last_search_error = str(e)
            return []

    async def fetch(self, url: str, *, timeout: int = 10, max_bytes: int | None = None) -> ResearchPage:
        """Fetch readable text from a URL using Odysseus' web fetcher."""
        try:
            from src.search.content import fetch_webpage_content

            kwargs: dict[str, Any] = {"timeout": timeout}
            if max_bytes is not None:
                kwargs["max_bytes"] = max_bytes
            page = await asyncio.to_thread(fetch_webpage_content, url, **kwargs)
        except Exception as e:
            return ResearchPage(url=url, success=False, error=str(e))
        return self._normalize_fetch_page(url, page)

    async def browser_read(self, url: str, *, timeout: int = 45) -> ResearchPage:
        """Read a JS-heavy page through the private browser tool."""
        from src.agent_tools.web_tools import PrivateBrowserTool

        if self._progress:
            self._progress({"phase": "navigating", "url": url, "title": url})
        tool = PrivateBrowserTool()
        result = await tool.execute(
            json.dumps({"action": "read", "url": url, "timeout_ms": int(timeout * 1000)}),
            {"session_id": self.session_id or "research"},
        )
        output = str(result.get("output") or "").strip()
        if result.get("exit_code") != 0 or not output:
            return ResearchPage(
                url=url,
                success=False,
                retrieval="browser",
                error=str(result.get("error") or output or "browser returned no readable content"),
            )
        self.browser_fetches += 1
        return ResearchPage(
            url=url,
            title=url,
            content=output[:MAX_OUTPUT_CHARS],
            success=True,
            retrieval="browser",
        )

    @staticmethod
    def _normalize_fetch_page(url: str, page: Any) -> ResearchPage:
        if not isinstance(page, dict):
            return ResearchPage(url=url, success=False, error="fetch returned non-object result")
        content = str(page.get("content") or "").strip()
        return ResearchPage(
            url=url,
            title=str(page.get("title") or ""),
            content=content,
            og_image=str(page.get("og_image") or ""),
            success=bool(page.get("success") and content),
            retrieval="fetch",
            error=str(page.get("error") or ""),
        )