Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.1 KB
Raw Blame
"""Project-wide constants for Local Deep Research."""

from enum import StrEnum
from typing import Dict, List

from .__version__ import __version__

# Honest, identifying User-Agent for APIs that prefer/require identification
# (e.g., academic APIs like arXiv, PubMed, OpenAlex)
USER_AGENT = (
    f"Local-Deep-Research/{__version__} "
    "(Academic Research Tool; https://github.com/LearningCircuit/local-deep-research)"
)

# Browser-like User-Agent for sites that may block bot requests
# Use sparingly and only when necessary
BROWSER_USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/120.0.0.0 Safari/537.36"
)


# Code-side single source of truth for the default search engine, used by
# every reader that needs a fallback for a MISSING ``search.tool`` setting
# (partial snapshots from the programmatic API, un-bootstrapped settings
# DBs). Import THIS instead of hardcoding the string literal: scattered
# literals are how the old ``"auto"`` default lingered across ~30 sites and
# had to be hunted down one by one when the meta engines were removed.
# Must match the registered default in defaults/default_settings.json —
# pinned by tests/test_constants.py::test_default_search_tool_matches_registry.
# Mirrors the DEFAULT_EGRESS_SCOPE pattern in security/egress/policy.py.
DEFAULT_SEARCH_TOOL: str = "searxng"


# --- Research status values ---
# Frontend helpers: src/local_deep_research/web/static/js/config/constants.js
# Injected via:     src/local_deep_research/web/template_config.py (_LDRTemplates.TemplateResponse)
# Template:         src/local_deep_research/web/templates/base.html
# If you add/remove/rename a status here, the frontend picks it up automatically.
class ResearchStatus(StrEnum):
    """Status values for research records.

    Uses StrEnum so values compare equal to plain strings,
    e.g. ``ResearchStatus.COMPLETED == "completed"`` is True.

    Lifecycle::

        [*] ─┬─► QUEUED ─┬─► IN_PROGRESS ─┬─► COMPLETED
             │            │                ├─► FAILED
             │            └─► SUSPENDED    └─► SUSPENDED
             │   (concurrency limit)  (terminated while queued)
             │
             └─► IN_PROGRESS (slots available, skips queue)

    Notes:
        - PENDING is declared as a model default but no creation path
          actually sets it.  All routes use QUEUED or IN_PROGRESS.
        - ERROR is checked as a terminal state but never set by current
          code.  It predates FAILED and exists for backward compatibility
          with older database records.
        - CANCELLED is not used by the research workflow.  It is used by
          the benchmark subsystem (BenchmarkStatus, BenchmarkTaskStatus).
    """

    # --- Active lifecycle states ---
    PENDING = "pending"  # Model default; never set by any creation path
    QUEUED = "queued"  # Waiting for a worker slot
    IN_PROGRESS = "in_progress"  # Worker actively executing

    # --- Terminal states ---
    COMPLETED = "completed"  # Finished successfully
    SUSPENDED = "suspended"  # User terminated the research
    FAILED = "failed"  # Unrecoverable error during execution

    # --- Legacy / compatibility ---
    ERROR = "error"  # Never set; predates FAILED
    CANCELLED = "cancelled"  # Unused by research; for benchmarks


# --- Research library file_path sentinel values ---
FILE_PATH_METADATA_ONLY = "metadata_only"
FILE_PATH_TEXT_ONLY = "text_only_not_stored"
FILE_PATH_BLOB_DELETED = "blob_deleted"
FILE_PATH_SENTINELS = (
    FILE_PATH_METADATA_ONLY,
    FILE_PATH_TEXT_ONLY,
    FILE_PATH_BLOB_DELETED,
)

# --- Default RAG / Local Search settings ---
# Code-side single source of truth for local search / embedding defaults,
# used by every reader needing a fallback for missing settings.
# Must match the registered defaults in defaults/settings_local_search.json —
# pinned by tests/test_constants.py::test_default_local_search_settings_match_registry.
DEFAULT_LOCAL_SEARCH_PROVIDER: str = "sentence_transformers"
DEFAULT_LOCAL_SEARCH_MODEL: str = "Alibaba-NLP/gte-modernbert-base"
DEFAULT_LOCAL_SEARCH_CHUNK_SIZE: int = 1000
DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP: int = 200
DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE: str = "recursive"
DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS: List[str] = ["\n\n", "\n", ". ", " ", ""]
DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON = '["\\n\\n", "\\n", ". ", " ", ""]'
DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC: str = "cosine"
DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS: bool = True
DEFAULT_LOCAL_SEARCH_INDEX_TYPE: str = "flat"


# --- Snippet / truncation lengths ---
SNIPPET_LENGTH_SHORT = 250
SNIPPET_LENGTH_LONG = 500

# --- /history/logs/<id> pagination caps ---
# Default matches the frontend logpanel DOM cap (MAX_LOG_ENTRIES); the
# hard cap is the ceiling the route clamps to so a client cannot force
# an unbounded load. Shared with the frontend by the template constants
# injection (see web/template_config.py) → window.LDR_LOG_LIMITS (see
# base.html).
HISTORY_LOGS_DEFAULT_LIMIT = 500
HISTORY_LOGS_HARD_CAP = 5000

# --- Research history collection ---
RESEARCH_HISTORY_COLLECTION_NAME = "History"
RESEARCH_HISTORY_COLLECTION_DESCRIPTION = (
    "Your research history indexed for AI-powered semantic search. "
    "Indexing converts past research reports and their sources into "
    "searchable content, enabling natural-language queries across all "
    "your previous research. Used by the History page search when in "
    "AI or Hybrid mode."
)

# --- Available search strategies (UI-facing) ---
# Single source of truth for strategies shown in all UI dropdowns.
# create_strategy() in search_system_factory.py handles additional names
# (aliases, internal strategies such as "news_aggregation") — this list is
# purely for the UI.
AVAILABLE_STRATEGIES: List[Dict[str, str]] = [
    {
        "name": "source-based",
        "label": "Source-Based (Best for small <16,000 context window)",
        "description": "Comprehensive research with inline citations. Focuses on finding and extracting information from authoritative sources.",
    },
    {
        "name": "focused-iteration",
        "label": "Focused Iteration - Quick (Minimal text output)",
        "description": "Fast & precise Q&A with iterative search. Good for complex queries requiring specific answers.",
    },
    {
        "name": "focused-iteration-standard",
        "label": "Focused Iteration - Comprehensive (Needs >16,000 context window)",
        "description": "Detailed long-form output with citations. Uses standard citation handler for comprehensive answers.",
    },
    {
        "name": "topic-organization",
        "label": "Topic Organization (Clusters by topic)",
        "description": "Clusters sources into topics with lead texts. Organizes research by themes for structured output.",
    },
    {
        "name": "langgraph-agent",
        "label": "LangGraph Agent (Autonomous agentic research)",
        "description": "Agentic research where the LLM autonomously decides what to search, which engines to use, and when to synthesize. Supports all search engines as tools.",
    },
]

LANGGRAPH_STRATEGY_NAME = "langgraph-agent"
LANGGRAPH_STRATEGY_ALIASES = (
    "langgraph-agent",
    "langgraph_agent",
    "mcp",
    "agentic",
)


# --- Journal quality scoring thresholds ---
# Used by journal_quality.scoring.derive_quality_score and
# journal_quality.scoring.institution_score_from_h_index. Single source of
# truth so the build phase, the runtime filter, and the dashboard agree on
# what each h-index threshold means.
#
# Thresholds calibrated from real OpenAlex data:
#   - Nature h-index ≈ 1,442
#   - PLOS ONE h-index ≈ 467
#   - Only ~3 journals globally have h-index > 1,000
# h-index has field-dependent bias (math vs biomed); these are general-purpose.

# Journal h-index thresholds → quality scores
JOURNAL_HINDEX_ELITE = 150  # Nature/Science/NEJM tier
JOURNAL_HINDEX_STRONG = 75
JOURNAL_HINDEX_VERY_GOOD = 40
JOURNAL_HINDEX_GOOD = 20
JOURNAL_HINDEX_ACCEPTABLE = 10

# Journal quality scores (1–10 scale)
JOURNAL_QUALITY_PREDATORY = 1
JOURNAL_QUALITY_DEFAULT = 4
JOURNAL_QUALITY_ACCEPTABLE = 5
JOURNAL_QUALITY_GOOD = 6
JOURNAL_QUALITY_VERY_GOOD = 7
JOURNAL_QUALITY_STRONG = 8
JOURNAL_QUALITY_ELITE = 10

# The complete set of scores the scoring algorithm emits. Scores 2, 3, 9
# are deliberately never produced by the tiered scoring logic; LLM outputs
# outside this set are rejected as parse failures so prompt drift surfaces
# via the existing failure counter rather than silently snapping.
# INVARIANT: score 9 is intentionally NOT in this set. Tier 4 LLM prompts
# never produce it and Tier 1-3 thresholds skip directly from 8 (h>=75)
# to 10 (h>=150). Do not "add it for completeness" — downstream code in
# search_utilities._format_quality_tag has a defensive branch for 9 that
# is currently dead by design.
VALID_QUALITY_SCORES = frozenset(
    {
        JOURNAL_QUALITY_PREDATORY,
        JOURNAL_QUALITY_DEFAULT,
        JOURNAL_QUALITY_ACCEPTABLE,
        JOURNAL_QUALITY_GOOD,
        JOURNAL_QUALITY_VERY_GOOD,
        JOURNAL_QUALITY_STRONG,
        JOURNAL_QUALITY_ELITE,
    }
)

# DOAJ scoring. There used to be a higher DOAJ_QUALITY_WITH_SEAL = 8
# tier, but DOAJ retired the Seal in April 2025 and removed it from
# their metadata, so listing is now the only DOAJ signal:
# https://blog.doaj.org/2025/04/09/our-metadata-changes-are-live-and-the-seal-has-been-retired/
DOAJ_QUALITY_LISTED = 5
CONFERENCE_QUALITY_DEFAULT = (
    5  # Neutral; in CS top conferences are Q1-equivalent
)
# Preprint repositories (arXiv, bioRxiv, SSRN, PsyArXiv, ...) are not
# peer-reviewed — the venue itself carries no quality signal. Cap all
# repository-type sources at this score regardless of their h-index,
# which is inflated by aggregating thousands of highly-cited papers
# (arXiv has h=674 because of its authors, not because of venue rigor).
# Matches the conference default: "acceptable, but the venue doesn't
# vouch for the paper". The filter's Tier 3.5 institution salvage can
# lift this to 6 when the authors are at a strong institution.
REPOSITORY_QUALITY_DEFAULT = 5

# Predatory whitelist override threshold. A flagged journal is rescued
# if it's in DOAJ (evidence-based) OR has h-index strictly greater than
# this value (heuristic — `>`, not `>=`).
#
# Do not re-tune without literature support. The h-index is an impact
# metric, not an integrity signal. Reviews of predatory-vs-legitimate
# classification (Blacklists and Whitelists to Tackle Predatory
# Publishing, mBio 2019, and the PMC2020 review that followed) treat
# DOAJ indexing + COPE / OASPA membership as the evidence-based
# whitelist — NOT any specific h-index boundary. The value 10 and
# strict-> here are pragmatic defaults; tuning them only changes
# behavior at the boundary and has no published basis. If you want
# real improvement, ADD more signals (JCR listing, OASPA membership)
# rather than tweaking this number. (Investigated in PR #3081, 2026-04.)
PREDATORY_WHITELIST_HINDEX = 10

# Institution h-index thresholds → quality scores. Capped at
# INSTITUTION_QUALITY_TOP — institution salvage scoring never beats a real
# venue match.
INSTITUTION_HINDEX_TOP = 250  # Top-tier research universities
INSTITUTION_HINDEX_HIGH = 50
INSTITUTION_QUALITY_TOP = 6
INSTITUTION_QUALITY_HIGH = 5
INSTITUTION_QUALITY_DEFAULT = 4


# --- API timeouts ---
# OpenAlex DOI→source_id batch enrichment. Distinct from the OpenAlex search
# engine timeout (which uses the safe_requests default of 30s) because batch
# metadata lookups are lightweight and we'd rather fail fast than block the
# pre-enrichment layer.
OPENALEX_ENRICHMENT_API_TIMEOUT = 15


# --- Journal-quality dataset download ---
# Minimum free disk space required before starting a bulk download. The
# five sources uncompress to ~1 GB total intermediate working set; the 2
# GB floor gives headroom for the atomic temp file + compiled DB while
# leaving room for the user's other work.
JOURNAL_QUALITY_MIN_FREE_DISK_BYTES = 2 * 1024**3


def get_available_strategies() -> List[Dict[str, str]]:
    """Get the list of available research strategies shown in the UI.

    Returns:
        List of dictionaries with 'name', 'label', and 'description' keys.
    """
    return AVAILABLE_STRATEGIES.copy()