Self-hosted AI workspace.
Python
fix(search): bound and police the scholarly metadata lookups
Léoedc94a244ed3
diff --git a/services/search/core.py b/services/search/core.pyindex dc97bfb61..d6fce2846 100644--- a/services/search/core.py+++ b/services/search/core.py@@ -3,11 +3,19 @@import jsonimport loggingimport re+import timeimport xml.etree.ElementTree as ETfrom concurrent.futures import ThreadPoolExecutor, as_completed+from contextlib import contextmanager+from contextvars import ContextVarfrom datetime import datetime, timedeltafrom typing import Dict, Any, Optional, List, Setfrom urllib.parse import urlparse+from src.constants import (+ ARXIV_API_URL,+ OPENALEX_API_URL,+ SCHOLARLY_LOOKUP_TOTAL_BUDGET,+)from src.search_passages import search_excerptimport httpx@@ -489,22 +497,80 @@ def _result_strongly_matches_title(title: str, result: dict) -> bool:return overlap >= (1.0 if len(wanted) == 2 else 0.8)+def _scholarly_user_agent() -> str:+ """Identify this build to the scholarly APIs using the real app version."""+ from src.constants import APP_VERSION++ return f"Odysseus/{APP_VERSION} scholarly-title-resolver"+++_scholarly_deadline: ContextVar[Optional[float]] = ContextVar(+ "scholarly_deadline", default=None+)+++@contextmanager+def _scholarly_budget():+ """Open one wall-clock budget shared by every hop of a lookup chain."""+ token = _scholarly_deadline.set(+ time.monotonic() + SCHOLARLY_LOOKUP_TOTAL_BUDGET+ )+ try:+ yield+ finally:+ _scholarly_deadline.reset(token)+++def _scholarly_api_get(url: str, params: dict) -> Optional[httpx.Response]:+ """GET a scholarly metadata API under the shared outbound policy.++ Returns ``None`` when the URL fails the outbound check or the caller's+ budget is already spent, so callers degrade to their next source instead+ of raising. ``follow_redirects`` stays on because both APIs redirect to+ canonical paths, which is exactly why the destination needs checking.+ """+ from src.constants import SCHOLARLY_LOOKUP_TIMEOUT+ from src.url_safety import check_outbound_url++ ok, reason = check_outbound_url(url, block_private=True)+ if not ok:+ logger.warning("Scholarly lookup blocked for %s: %s", url, reason)+ return None++ timeout = SCHOLARLY_LOOKUP_TIMEOUT+ deadline = _scholarly_deadline.get()+ if deadline is not None:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ logger.info("Scholarly lookup budget exhausted before %s", url)+ return None+ timeout = min(timeout, remaining)++ response = httpx.get(+ url,+ params=params,+ headers={"User-Agent": _scholarly_user_agent()},+ timeout=timeout,+ follow_redirects=True,+ )+ response.raise_for_status()+ return response++def _arxiv_title_results(title: str, count: int = 3) -> list[dict]:"""Resolve a paper title through arXiv's public Atom API."""try:- response = httpx.get(- "https://export.arxiv.org/api/query",- params={+ response = _scholarly_api_get(+ ARXIV_API_URL,+ {"search_query": f'ti:"{title}"',"start": 0,"max_results": max(1, min(int(count), 5)),},- headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},- timeout=12.0,- follow_redirects=True,)- response.raise_for_status()+ if response is None:+ return []root = ET.fromstring(response.text)except Exception as exc:logger.info("arXiv title lookup failed for %r: %s", title, exc)@@ -541,20 +607,18 @@ def _openalex_title_results(title: str, count: int = 3) -> list[dict]:# OpenAlex treats a literal question mark as query syntax and returns# HTTP 400 for otherwise valid titles such as "How Far ... GPT-4V?".search_title = re.sub(r"[?]+", " ", str(title or "")).strip()- response = httpx.get(- "https://api.openalex.org/works",- params={+ response = _scholarly_api_get(+ OPENALEX_API_URL,+ {"search": search_title,"per-page": max(1, min(int(count), 5)),"select": ("display_name,doi,primary_location,publication_year,type"),},- headers={"User-Agent": "Odysseus/0.20 scholarly-title-resolver"},- timeout=12.0,- follow_redirects=True,)- response.raise_for_status()+ if response is None:+ return []payload = response.json()except Exception as exc:logger.info("OpenAlex title lookup failed for %r: %s", title, exc)@@ -597,8 +661,16 @@ def _openalex_title_results(title: str, count: int = 3) -> list[dict]:def _scholarly_title_results(title: str, count: int = 3) -> list[dict]:- """Retry a noisy scholarly query as a bare title, then use arXiv API."""+ """Retry a noisy scholarly query as a bare title, then use arXiv API.++ The three hops share one wall-clock budget so a slow upstream cannot hold a+ user-facing search open for the sum of every per-request timeout.+ """+ with _scholarly_budget():+ return _scholarly_title_results_inner(title, count)++def _scholarly_title_results_inner(title: str, count: int) -> list[dict]:try:simplified = searxng_search_api(title, count=max(3, count))except Exception as exc:@@ -621,10 +693,11 @@ def _direct_scholarly_title_results(title: str, count: int = 3) -> list[dict]:# OpenAlex typically resolves titles in under a second and often returns# the official arXiv landing page. The arXiv API remains the fallback.- openalex = _openalex_title_results(title, count)- if openalex:- return openalex- return _arxiv_title_results(title, count)+ with _scholarly_budget():+ openalex = _openalex_title_results(title, count)+ if openalex:+ return openalex+ return _arxiv_title_results(title, count)def _augment_scholarly_results(query: str, results: list[dict], count: int) -> list[dict]:diff --git a/src/constants.py b/src/constants.pyindex 2b4919d3d..d52249e25 100644--- a/src/constants.py+++ b/src/constants.py@@ -140,6 +140,18 @@ LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.stripOPENAI_API_KEY = os.getenv("OPENAI_API_KEY")SEARXNG_INSTANCE = os.getenv("SEARXNG_INSTANCE", "http://localhost:8080")+# Scholarly title resolution. These are the only third-party metadata APIs the+# search path calls directly, so they get named constants rather than literals+# repeated at each call site. The budget bounds the whole SearXNG -> OpenAlex ->+# arXiv chain: each hop used to get its own full timeout, so one scholarly query+# could stall a user-facing search for the sum of all three.+ARXIV_API_URL = "https://export.arxiv.org/api/query"+OPENALEX_API_URL = "https://api.openalex.org/works"+SCHOLARLY_LOOKUP_TIMEOUT = float(os.getenv("ODYSSEUS_SCHOLARLY_LOOKUP_TIMEOUT", "12"))+SCHOLARLY_LOOKUP_TOTAL_BUDGET = float(+ os.getenv("ODYSSEUS_SCHOLARLY_LOOKUP_BUDGET", "20")+)+# Cleanup configurationCLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "True").lower() == "true"CLEANUP_INTERVAL_HOURS = int(os.getenv("CLEANUP_INTERVAL_HOURS", "24"))