Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

30.4 KB
Raw Blame
# src/local_deep_research/search_system.py
from typing import Any, Callable, Dict

from langchain_core.language_models import BaseChatModel
from loguru import logger

from .advanced_search_system.findings.repository import FindingsRepository
from .advanced_search_system.questions.standard_question import (
    StandardQuestionGenerator,
)
from .advanced_search_system.strategies.followup.enhanced_contextual_followup import (
    EnhancedContextualFollowUpStrategy,
)

from .utilities.type_utils import unwrap_setting
from .citation_handler import CitationHandler
from .web_search_engines.search_engine_base import BaseSearchEngine
from .web_search_engines.engine_availability import SearchFailure
from .constants import DEFAULT_SEARCH_TOOL


def ensure_snapshot_username(settings_snapshot, username):
    """Ensure the run's settings snapshot carries the username under the
    ``_username`` key that snapshot-driven consumers look it up by.

    The LangGraph agent re-instantiates the search engine *per tool call*
    from this snapshot (``_make_web_search_tool`` -> ``create_search_engine``),
    and registering a user's document collections needs the username
    (``get_user_db_session``), which collection registration reads only from
    ``settings_snapshot["_username"]``. The snapshot is built without it (the
    user lives under ``system.user``), so without this a collection/library
    primary fails inside the agent with "Unknown search engine 'collection_…'".

    Injected here in ``AdvancedSearchSystem.__init__`` — the narrowest common
    consumer of the strategy-running paths — so the web run and the
    programmatic API are repaired in one place. (No-ops when ``username`` is
    absent, e.g. benchmarks/news, which don't use per-user collections anyway.)
    Mirrors the convention ``get_search()`` already applies to its own local
    snapshot copy.

    Returns the snapshot unchanged when there is no username, it is not a dict,
    or ``_username`` is already set (never overwrites an explicit value);
    otherwise a shallow copy with ``_username`` added (no mutation).
    """
    if (
        username
        and isinstance(settings_snapshot, dict)
        and not settings_snapshot.get("_username")
    ):
        return {**settings_snapshot, "_username": username}
    return settings_snapshot


def username_from_snapshot(settings_snapshot):
    """Return the run owner's username carried in a settings snapshot.

    Centralizes the ``settings_snapshot.get("_username")`` read (the mirror of
    :func:`ensure_snapshot_username`'s write) behind a single ``isinstance``
    guard so every consumer treats a missing / non-dict snapshot identically —
    returning ``None`` rather than raising or applying an inconsistent guard.
    ``None`` resolves the shared/built-in namespace in the per-user registries.
    """
    return (
        settings_snapshot.get("_username")
        if isinstance(settings_snapshot, dict)
        else None
    )


class AdvancedSearchSystem:
    """
    Advanced search system that coordinates different search strategies.
    """

    def __init__(
        self,
        llm: BaseChatModel,
        search: BaseSearchEngine,
        strategy_name: str = "source-based",  # Default to comprehensive research strategy
        include_text_content: bool | None = None,
        use_cross_engine_filter: bool = True,
        max_iterations: int | None = None,
        questions_per_iteration: int | None = None,
        use_atomic_facts: bool = False,
        username: str | None = None,
        settings_snapshot: dict | None = None,
        research_id: str | None = None,
        research_context: dict | None = None,
        programmatic_mode: bool = False,
        search_original_query: bool = True,
        initial_search_failure: SearchFailure | None = None,
    ):
        """Initialize the advanced search system.

        Args:
            llm: LLM to use for the search strategy.
            search: Search engine to use for queries.
            initial_search_failure: Failed primary initialization to carry into
                the agent's research-scoped availability tracker.
            strategy_name: The name of the search strategy to use. Options:
                - "standard": Basic iterative search strategy
                - "iterdrag": Iterative Dense Retrieval Augmented Generation
                - "source-based": Focuses on finding and extracting from sources
                - "parallel": Runs multiple search queries in parallel
                - "rapid": Quick single-pass search
                - "recursive": Recursive decomposition of complex queries
                - "iterative": Loop-based reasoning with persistent knowledge
                - "adaptive": Adaptive step-by-step reasoning
                - "smart": Automatically chooses best strategy based on query
                - "browsecomp": Optimized for BrowseComp-style puzzle queries
                - "evidence": Enhanced evidence-based verification with improved candidate discovery
                - "constrained": Progressive constraint-based search that narrows candidates step by step
                - "parallel-constrained": Parallel constraint-based search with combined constraint execution
                - "early-stop-constrained": Parallel constraint search with immediate evaluation and early stopping at 99% confidence
                - "dual-confidence": Dual confidence scoring with positive/negative/uncertainty
                - "dual-confidence-with-rejection": Dual confidence with early rejection of poor candidates
                - "concurrent-dual-confidence": Concurrent search & evaluation with progressive constraint relaxation
                - "modular": Modular architecture using constraint checking and candidate exploration modules
                - "browsecomp-entity": Entity-focused search for BrowseComp questions with knowledge graph building
                - "iterative-refinement": Iteratively refines results using LLM evaluation and follow-up queries
            include_text_content: If False, only includes metadata and links in search results
            use_cross_engine_filter: Whether to filter results across search
                engines.
            max_iterations: The maximum number of search iterations to
                perform. Will be read from the settings if not specified.
            questions_per_iteration: The number of questions to include in
                each iteration. Will be read from the settings if not specified.
            use_atomic_facts: Whether to use atomic fact decomposition for
                complex queries when using the source-based strategy.
            programmatic_mode: If True, disables database operations and metrics tracking.
                This is useful for running searches without database dependencies.
            search_original_query: Whether to include the original query in the first iteration
                of search. Set to False for news searches to avoid sending long subscription
                prompts to search engines.

        """
        # strategy_name can come from settings or from a replayed queue row
        # read back from JSON, so it may hold any type. The router path
        # always passes research_context as a dict, but a programmatic
        # caller can pass any type.
        if not isinstance(strategy_name, str):
            logger.warning(
                f"Ignoring non-string strategy_name "
                f"{type(strategy_name).__name__}; using source-based"
            )
            strategy_name = "source-based"
        if research_context is not None and not isinstance(
            research_context, dict
        ):
            logger.warning(
                f"Ignoring non-dict research_context "
                f"{type(research_context).__name__}"
            )
            research_context = None

        # Store research context for strategies
        self.research_id = research_id
        self.research_context = research_context
        self.username = username

        # Store required components
        self.model = llm
        self.search = search

        # Store settings snapshot. Inject the username under "_username" so
        # snapshot-driven consumers (notably the LangGraph agent's per-call
        # search-engine creation, which must register the user's document
        # collections) can find it. See ensure_snapshot_username.
        self.settings_snapshot = ensure_snapshot_username(
            settings_snapshot or {}, username
        )

        # Store programmatic mode
        self.programmatic_mode = programmatic_mode

        # Store search original query setting
        self.search_original_query = search_original_query

        # Log if running in programmatic mode
        if self.programmatic_mode:
            logger.warning(
                "Running in programmatic mode - database operations and metrics tracking disabled. "
                "Rate limiting, search metrics, and persistence features will not be available."
            )

        # Get iterations setting
        self.max_iterations = max_iterations
        if self.max_iterations is None:
            # Use settings from snapshot
            if "search.iterations" in self.settings_snapshot:
                self.max_iterations = unwrap_setting(
                    self.settings_snapshot["search.iterations"]
                )
            else:
                self.max_iterations = 1  # Default

        self.questions_per_iteration = questions_per_iteration
        if self.questions_per_iteration is None:
            # Use settings from snapshot
            if "search.questions_per_iteration" in self.settings_snapshot:
                self.questions_per_iteration = unwrap_setting(
                    self.settings_snapshot["search.questions_per_iteration"]
                )
            else:
                self.questions_per_iteration = 3  # Default

        # Single shared content cache for the ENTIRE research run: the
        # initial quick-report phase, every detailed-report subsection and
        # all parallel subagents resolve webpage / RAG-chunk fetches through
        # this one instance instead of re-fetching on the spot.
        try:
            from .utilities.research_content_cache import (
                ResearchContentCache,
                bind_cache_to_research,
            )

            self.content_cache = ResearchContentCache()
        except Exception:  # pragma: no cover - defensive
            self.content_cache = None  # type: ignore[assignment]

        # Log the strategy name that's being used
        logger.info(
            f"Initializing AdvancedSearchSystem with strategy_name='{strategy_name}'"
        )

        # Initialize components
        self.citation_handler = CitationHandler(
            self.model, settings_snapshot=self.settings_snapshot
        )
        self.question_generator = StandardQuestionGenerator(self.model)
        self.findings_repository = FindingsRepository(self.model)
        # For backward compatibility
        self.questions_by_iteration: dict[Any, Any] = {}
        self.progress_callback = lambda _1, _2, _3: None
        self.all_links_of_system: list[dict[Any, Any]] = []

        strategy_health: dict[str, Any] = {}
        if initial_search_failure is not None:
            strategy_health["initial_search_failure"] = initial_search_failure

        # Initialize strategy using factory
        from .search_system_factory import create_strategy

        # Special handling for follow-up strategy which needs different logic
        if strategy_name.lower() in [
            "enhanced-contextual-followup",
            "enhanced_contextual_followup",
            "contextual-followup",
            "contextual_followup",
        ]:
            logger.info("Creating EnhancedContextualFollowUpStrategy instance")
            # Get delegate strategy from research context
            # This should be the user's preferred strategy from settings
            delegate_strategy_name = (
                self.research_context.get("delegate_strategy", "source-based")
                if self.research_context
                else "source-based"
            )
            # research_context can be read back from JSON, so the value may
            # be any JSON type; only a name can select a strategy.
            if not isinstance(delegate_strategy_name, str):
                logger.warning(
                    f"Ignoring non-string delegate_strategy "
                    f"{type(delegate_strategy_name).__name__}; using source-based"
                )
                delegate_strategy_name = "source-based"

            delegate = create_strategy(
                strategy_name=delegate_strategy_name,
                model=self.model,
                search=self.search,
                all_links_of_system=[],
                settings_snapshot=self.settings_snapshot,
                knowledge_accumulation_mode=True,
                search_original_query=self.search_original_query,
                content_cache=getattr(self, "content_cache", None),
                max_iterations=self.max_iterations,
                questions_per_iteration=self.questions_per_iteration,
                **strategy_health,
            )

            # Create the contextual follow-up strategy with the delegate
            self.strategy = EnhancedContextualFollowUpStrategy(
                model=self.model,
                search=self.search,
                delegate_strategy=delegate,
                all_links_of_system=self.all_links_of_system,
                settings_snapshot=self.settings_snapshot,
                research_context=self.research_context,
                content_cache=getattr(self, "content_cache", None),
            )
            # The delegate owns the actual fetch path — make sure it shares
            # the single research-scoped cache even when the factory drops
            # unknown kwargs for that strategy type.
            if getattr(self, "content_cache", None) is not None:
                try:
                    delegate.content_cache = self.content_cache
                except Exception:  # allow: silent-exception
                    pass
        else:
            # Use factory for all other strategies
            logger.info(f"Creating {strategy_name} strategy using factory")
            self.strategy = create_strategy(
                strategy_name=strategy_name,
                model=self.model,
                search=self.search,
                all_links_of_system=self.all_links_of_system,
                settings_snapshot=self.settings_snapshot,
                # Pass strategy-specific parameters
                include_text_content=include_text_content,
                use_cross_engine_filter=use_cross_engine_filter,
                use_atomic_facts=use_atomic_facts,
                max_iterations=self.max_iterations,
                questions_per_iteration=self.questions_per_iteration,
                # Special parameters for iterative strategy
                search_iterations_per_round=self.max_iterations or 1,
                questions_per_search=self.questions_per_iteration,
                # Special parameters for adaptive strategy
                max_steps=self.max_iterations,
                source_questions_per_iteration=self.questions_per_iteration,
                # Special parameters for evidence and constrained strategies
                max_search_iterations=self.max_iterations,
                # Special parameters for focused iteration
                use_browsecomp_optimization=True,
                # Pass search original query parameter
                search_original_query=self.search_original_query,
                # Forwarded so strategies that create engines per tool call
                # (e.g. langgraph-agent) can match the system's mode.
                programmatic_mode=self.programmatic_mode,
                content_cache=getattr(self, "content_cache", None),
                **strategy_health,
            )

        # Share the single research-scoped cache with the strategy (covers
        # strategies whose constructors do not accept content_cache, and
        # factory paths that drop unknown kwargs).
        if getattr(self, "content_cache", None) is not None:
            try:
                self.strategy.content_cache = self.content_cache
            except Exception:  # allow: silent-exception
                pass
        # Also thread it into the long-lived search engine when present so
        # non-agent strategies (source-based / focused-iteration) share it
        # for their direct self.search.run() calls.
        if (
            getattr(self, "content_cache", None) is not None
            and getattr(self, "search", None) is not None
        ):
            try:
                self.search.content_cache = self.content_cache
            except Exception:  # allow: silent-exception
                pass

        # Log the actual strategy class
        logger.info(f"Created strategy of type: {type(self.strategy).__name__}")

        # Configure the strategy with our attributes
        if (
            hasattr(self, "progress_callback")
            and self.progress_callback is not None
        ):
            self.strategy.set_progress_callback(self.progress_callback)

        # Publish the binding only after construction succeeds. A failed
        # strategy has no system for the caller to close.
        if getattr(self, "content_cache", None) is not None:
            try:
                _rid = research_id or (
                    research_context.get("research_id")
                    if isinstance(research_context, dict)
                    else None
                )
                if _rid is not None and not isinstance(_rid, str):
                    _rid = str(_rid)
                if isinstance(_rid, str) and _rid:
                    bind_cache_to_research(_rid, self.content_cache)
            except Exception:  # allow: silent-exception
                pass

    def close(self):
        """Close resources held by the search system.

        Cascades close to the strategy, which may hold persistent
        ThreadPoolExecutor instances (e.g. ConstraintParallelStrategy holds
        search_executor and evaluation_executor,
        ConcurrentDualConfidenceStrategy holds evaluation_executor).

        NOTE: Does NOT close self.search (the search engine) — the caller
        (run_research_process) manages search engine lifecycle separately
        because the search engine may be shared or reused.

        Most strategies (including the default source-based) use
        context-managed ThreadPoolExecutors that clean up automatically.
        The close() call here is a safety net for the two constraint-based
        strategies that hold persistent executors in __init__. Those
        strategies also shut down their executors in
        find_relevant_information()'s finally block, so this is a second
        line of defense for the edge case where the method is never called.
        """
        from .utilities.resource_utils import safe_close

        if hasattr(self, "strategy"):
            safe_close(self.strategy, "search strategy")
        # Release the research-scoped content cache binding. The cache
        # itself is in-memory only; unbinding prevents unbounded growth
        # of the registry across many programmatic calls.
        try:
            from .utilities.research_content_cache import (
                unbind_cache_from_research,
            )

            _rid = getattr(self, "research_id", None) or (
                self.research_context.get("research_id")
                if isinstance(self.research_context, dict)
                else None
            )
            if _rid is not None and not isinstance(_rid, str):
                _rid = str(_rid)
            cache = getattr(self, "content_cache", None)
            if isinstance(_rid, str) and _rid and cache is not None:
                unbind_cache_from_research(_rid, cache=cache)
        except Exception:
            logger.debug(
                "content cache unbind failed (non-critical)",
                exc_info=True,
            )

    def _progress_callback(
        self, message: str, progress: int, metadata: dict
    ) -> None:
        """Handle progress updates from the strategy."""
        logger.info(f"Progress: {progress}% - {message}")
        if hasattr(self, "progress_callback"):
            self.progress_callback(message, progress, metadata)

    def set_progress_callback(
        self, callback: Callable[[str, int, dict], None]
    ) -> None:
        """Set a callback function to receive progress updates."""
        self.progress_callback = callback
        if hasattr(self, "strategy"):
            self.strategy.set_progress_callback(callback)
        # Pipeline-strategy live path for bulk-fetch progress (#7193):
        # adapt the 3-arg UI callback to the 2-arg ``(done, total)`` hook
        # and attach it to the search engine so ``FullSearchResults``
        # bulk fetches emit throttled milestones + per-page cancellation
        # checks. Late-binds so engines created via ``get_search`` before
        # the callback existed still get progress; agents stay snippets-
        # only post-#7189, so their hook remains dormant groundwork.
        try:
            search = getattr(self, "search", None)
            if search is not None and callback is not None:
                from .web_search_engines.engines.full_search import (
                    make_fetch_progress_adapter,
                )

                hook = make_fetch_progress_adapter(callback)
                setter = getattr(search, "set_fetch_progress_callback", None)
                if callable(setter):
                    setter(hook)
                elif hasattr(search, "progress_callback"):
                    search.progress_callback = hook
        except Exception:
            logger.debug("Failed to wire fetch progress to search engine")

    def analyze_topic(
        self,
        query: str,
        is_user_search: bool = True,
        is_news_search: bool = False,
        user_id: str = "anonymous",
        search_id: str | None = None,
        **kwargs,
    ) -> Dict:
        """Analyze a topic using the current strategy.

        Args:
            query: The research query to analyze
            is_user_search: Whether this is a user-initiated search
            is_news_search: Whether this is a news search
            user_id: The user ID for tracking
            search_id: The search ID (auto-generated if not provided)
            **kwargs: Additional arguments
        """

        # Generate search ID if not provided
        if search_id is None:
            import uuid

            search_id = str(uuid.uuid4())

        # Defense-in-depth: arm the PEP-578 egress audit-hook backstop for this
        # run if no caller already armed it. The web worker arms it in
        # research_service before calling us; CLI, the news scheduler and the
        # programmatic API construct AdvancedSearchSystem directly and would
        # otherwise run the full pipeline with the secondary net inactive.
        _armed_egress = self._arm_egress_backstop()
        # Bind the single research-scoped content cache for the whole run
        # (quick phase + all detailed subsections + subagents). Deep funnels
        # (engines created per web_search call, ContentFetcher, pipeline,
        # library resolver) resolve it via the thread-local search context.
        try:
            from .utilities.research_content_cache import (
                bind_cache_to_research,
            )

            _rid = self.research_id or (
                self.research_context.get("research_id")
                if isinstance(self.research_context, dict)
                else None
            )
            if _rid is not None and not isinstance(_rid, str):
                _rid = str(_rid)
            if (
                isinstance(_rid, str)
                and _rid
                and getattr(self, "content_cache", None) is not None
            ):
                bind_cache_to_research(_rid, self.content_cache)
            # Re-share with strategy/engine in case they were constructed
            # without it (factory drops unknown kwargs on some paths).
            if getattr(self, "content_cache", None) is not None:
                try:
                    if getattr(self.strategy, "content_cache", None) is None:
                        self.strategy.content_cache = self.content_cache
                except Exception:  # allow: silent-exception
                    pass
                try:
                    if (
                        getattr(self, "search", None) is not None
                        and getattr(self.search, "content_cache", None) is None
                    ):
                        self.search.content_cache = self.content_cache
                except Exception:  # allow: silent-exception
                    pass
        except Exception:  # allow: silent-exception
            pass
        try:
            # Perform the search
            return self._perform_search(
                query, search_id, is_user_search, is_news_search, user_id
            )
        finally:
            if _armed_egress:
                # Only clear what WE armed, so a reused thread doesn't leak the
                # context to unrelated subsequent work (the web worker owns its
                # own teardown via @thread_cleanup).
                from .security.egress.audit_hook import clear_active_context

                clear_active_context()

    def _arm_egress_backstop(self) -> bool:
        """Arm the egress audit-hook context for this run when no caller has.

        Returns True only when THIS call armed the context (so ``analyze_topic``
        clears it on exit). Returns False when it was already armed (web worker)
        or could not be built. Never raises — a backstop failure must not break
        a research run; the explicit PEPs remain the primary enforcement.
        """
        try:
            from .security.egress.audit_hook import (
                get_active_context,
                set_active_context,
            )
            from .security.egress.policy import (
                PolicyDeniedError,
                context_from_snapshot,
                resolve_run_primary_engine,
            )
        except Exception:
            return False

        if get_active_context() is not None:
            return False  # already armed (e.g. by the web research worker)
        if not self.settings_snapshot:
            return False

        try:
            ctx = context_from_snapshot(
                self.settings_snapshot,
                resolve_run_primary_engine(self.settings_snapshot),
                username=self.username,
            )
        except (PolicyDeniedError, ValueError):
            # Corrupted scope: the explicit PEPs and the run-start
            # precheck handle these; don't arm a partial backstop.
            logger.bind(policy_audit=True).debug(
                "egress backstop not armed: policy unevaluable", exc_info=True
            )
            return False
        except Exception:
            return False

        set_active_context(ctx)
        return True

    def _perform_search(
        self,
        query: str,
        search_id: str,
        is_user_search: bool,
        is_news_search: bool,
        user_id: str,
    ) -> Dict:
        """Perform the actual search."""
        # Send progress message with LLM info
        # Get settings from snapshot if available
        llm_provider = "unknown"
        llm_model = "unknown"
        search_tool = "unknown"

        if self.settings_snapshot:
            # Extract values from settings snapshot
            provider_setting = self.settings_snapshot.get("llm.provider", {})
            llm_provider = (
                provider_setting.get("value", "unknown")
                if isinstance(provider_setting, dict)
                else provider_setting
            )

            model_setting = self.settings_snapshot.get("llm.model", {})
            llm_model = (
                model_setting.get("value", "unknown")
                if isinstance(model_setting, dict)
                else model_setting
            )

            tool_setting = self.settings_snapshot.get("search.tool", {})
            search_tool = (
                tool_setting.get("value", DEFAULT_SEARCH_TOOL)
                if isinstance(tool_setting, dict)
                else tool_setting
            )

        self.progress_callback(
            f"Using {llm_provider} model: {llm_model}",
            1,  # Low percentage to show this as an early step
            {
                "phase": "setup",
                "llm_info": {
                    "name": llm_model,
                    "provider": llm_provider,
                },
            },
        )
        # Send progress message with search strategy info
        self.progress_callback(
            f"Using search tool: {search_tool}",
            1.5,  # Between setup and processing steps
            {
                "phase": "setup",
                "search_info": {
                    "tool": search_tool,
                },
            },
        )

        # Use the strategy to analyze the topic
        result = self.strategy.analyze_topic(query)

        # Update our attributes for backward compatibility

        self.questions_by_iteration = (
            self.strategy.questions_by_iteration.copy()
        )
        # Send progress message with search info

        # Only extend if they're different objects in memory to avoid duplication
        # This check prevents doubling the list when they reference the same object
        # Fix for issue #301: "too many links in detailed report mode"
        if id(self.all_links_of_system) != id(
            self.strategy.all_links_of_system
        ):
            self.all_links_of_system.extend(self.strategy.all_links_of_system)

        # Include the search system instance for access to citations
        result["search_system"] = self
        result["all_links_of_system"] = self.all_links_of_system

        # Ensure query is included in the result
        if "query" not in result:
            result["query"] = query
        result["questions_by_iteration"] = self.questions_by_iteration

        # Call news callback
        try:
            from .news.core.search_integration import NewsSearchCallback

            callback = NewsSearchCallback()
            context = {
                "is_user_search": is_user_search,
                "is_news_search": is_news_search,
                "user_id": user_id,
                "search_id": search_id,
            }
            callback(query, result, context)
        except Exception:
            logger.exception("Error in news callback")

        return result