Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: close four silent-failure leaks (tags, FTS syntax, batch PDFs, cache-busting date) (#52)
Francois du Plessis771df98a6a31
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 945b74f..6860e3d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,13 @@## [Unreleased]+### Four silent-failure leaks closed (tags, FTS syntax, batch PDFs, cache-busting date)++- **Tag filtering:** frontmatter tags are now lowercased before the alias lookup and before storage, matching the lowercase alias keys and the lowercasing `SearchFilters` does on the query side — `--tag llm` now finds notes tagged `LLM`, including through aliases.+- **FTS queries:** stray unbalanced double-quotes in bare words are stripped instead of raising an FTS5 syntax error that the caller saw as zero results. Hyphenated words are left intact — they were already valid inside the emitted quoted phrase tokens.+- **Batch PDF fetches:** in `fetch_many`, a PDF whose direct download fails (parser failure, non-PDF body at a `.pdf` URL) now falls back to the browser path, mirroring single-fetch behaviour instead of being silently dropped. `fetch-batch` also falls back to per-URL fetches when a whole batch fails, and reports `failed_urls` in its output instead of losing them silently.+- **Prompt-cache-busting date:** the vault CLAUDE.md blurb no longer interpolates `Today is YYYY-MM-DD`, which busted Claude Code's prompt cache once per day.+## [0.9.0] - 2026-07-23### Coverage before elegance: the synthesizer stops trading substance for prosediff --git a/src/hyperresearch/cli/fetch_batch.py b/src/hyperresearch/cli/fetch_batch.pyindex 6c3db65..4103201 100644--- a/src/hyperresearch/cli/fetch_batch.py+++ b/src/hyperresearch/cli/fetch_batch.py@@ -94,6 +94,15 @@ def fetch_batch(visible_urls = [u for u in new_urls if _needs_visible(u)]results = []+ failed_urls: list[dict] = [] # surfaced in JSON output so callers see what was lost++ def _fetch_one(provider, url: str, kind: str) -> None:+ try:+ results.append(provider.fetch(url))+ except Exception as exc:+ failed_urls.append({"url": url, "error": str(exc), "phase": kind})+ if not json_output:+ console.print(f" [red]Failed ({kind}):[/] {url} — {exc}")# Batch fetch normal URLsif normal_urls:@@ -101,15 +110,19 @@ def fetch_batch(try:results.extend(prov.fetch_many(normal_urls))except Exception as e:+ # fetch_many can return zero results on a single bad URL+ # inside the batch comprehension. Fall back to per-URL so we+ # don't silently lose the entire wave — partial progress+ # beats none.if not json_output:- console.print(f"[red]Batch fetch failed:[/] {e}")+ console.print(+ f"[red]Batch fetch failed:[/] {e}. Falling back to per-URL fetches."+ )+ for url in normal_urls:+ _fetch_one(prov, url, "batch-fallback")else:for url in normal_urls:- try:- results.append(prov.fetch(url))- except Exception as e:- if not json_output:- console.print(f" [red]Failed:[/] {url} — {e}")+ _fetch_one(prov, url, "normal")# Fetch auth-aggressive URLs with visible browser (sequential)if visible_urls:@@ -122,11 +135,7 @@ def fetch_batch(gates=vault.config.junk,)for url in visible_urls:- try:- results.append(visible_prov.fetch(url))- except Exception as e:- if not json_output:- console.print(f" [red]Failed (visible):[/] {url} — {e}")+ _fetch_one(visible_prov, url, "visible")# Phase 1: Write all note files to disk (no sync yet)note_files = [] # (note_path, url, result, domain, content_hash)@@ -223,12 +232,16 @@ def fetch_batch("notes_created": created_notes,"total_fetched": len(created_notes),"skipped": len(all_urls) - len(new_urls),+ "failed_urls": failed_urls,}if json_output:output(success(data, count=len(created_notes), vault=str(vault.root)), json_mode=True)else:- console.print(+ msg = (f"\n[bold]Done:[/] {len(created_notes)} notes created, "- f"{len(all_urls) - len(new_urls)} skipped (already fetched)."+ f"{len(all_urls) - len(new_urls)} skipped (already fetched)")+ if failed_urls:+ msg += f", [red]{len(failed_urls)} failed[/]"+ console.print(msg + ".")diff --git a/src/hyperresearch/core/agent_docs.py b/src/hyperresearch/core/agent_docs.pyindex 92c1b08..36a5972 100644--- a/src/hyperresearch/core/agent_docs.py+++ b/src/hyperresearch/core/agent_docs.py@@ -17,7 +17,7 @@ HYPERRESEARCH_SECTION_END = "<!-- hyperresearch:end -->"HYPERRESEARCH_BLURB = """{marker}-## Research Base (hyperresearch) — Today is {today}+## Research Base (hyperresearch)**CLI path: `{hpr}`** — use this exact path for every hyperresearch command. It may not be on your system PATH.@@ -175,12 +175,12 @@ def inject_agent_docs(vault_root: Path) -> list[str]:hpr_path = _resolve_executable()# Use forward slashes — bash on Windows eats backslasheshpr_path = hpr_path.replace("\\", "/")- from datetime import date+ # No date interpolation here: a `Today is YYYY-MM-DD` line in the+ # cached prefix would bust Claude Code's prompt cache once per day.blurb = HYPERRESEARCH_BLURB.format(marker=HYPERRESEARCH_SECTION_MARKER,end_marker=HYPERRESEARCH_SECTION_END,hpr=hpr_path,- today=date.today().isoformat(),)modified: list[str] = []diff --git a/src/hyperresearch/core/sync.py b/src/hyperresearch/core/sync.pyindex 3187158..3a5f7c4 100644--- a/src/hyperresearch/core/sync.py+++ b/src/hyperresearch/core/sync.py@@ -276,7 +276,12 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Nonexcept Exception:passfor tag in meta.tags:- resolved = alias_map.get(tag, tag)+ # Lowercase the frontmatter tag for BOTH the lookup and the fallback:+ # alias keys and canonicals are stored lowercase (cli/tag.py), and+ # SearchFilters lowercases query-side tags, so the stored value must+ # be lowercase to match. Without this, a frontmatter tag like `LLM`+ # misses its alias mapping and is invisible to `--tag llm` filtering.+ resolved = alias_map.get(tag.lower(), tag.lower())conn.execute("INSERT OR IGNORE INTO tags (note_id, tag) VALUES (?, ?)", (meta.id, resolved))# Update aliasesdiff --git a/src/hyperresearch/search/fts.py b/src/hyperresearch/search/fts.pyindex 620503e..69dc16e 100644--- a/src/hyperresearch/search/fts.py+++ b/src/hyperresearch/search/fts.py@@ -48,7 +48,11 @@ def preprocess_query(raw: str) -> str:words = part.strip().split()for word in words:if word:- clean = re.sub(r'[*^():{}]', '', word)+ # Strip FTS5-special chars that would break syntax. The+ # double-quote is included because quoted phrases were+ # already extracted by the outer split — any quote left+ # here is an unbalanced fragment.+ clean = re.sub(r'[*^():{}"]', '', word)if not clean:continue# Split glued alphanumeric (mamba3 -> mamba + 3)diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.pyindex f4bbbb6..225dcf9 100644--- a/src/hyperresearch/web/crawl4ai_provider.py+++ b/src/hyperresearch/web/crawl4ai_provider.py@@ -418,17 +418,26 @@ class Crawl4AIProvider:web_results = []- # Fetch PDFs directly (no browser needed)+ # Fetch PDFs directly (no browser needed). When _fetch_pdf returns+ # None (parser failure, non-PDF body served at a .pdf URL) we fall+ # back to the browser path — mirrors the single-fetch behaviour in+ # fetch() so batch callers don't silently lose academic PDFs that+ # need JS-rendered landing pages.+ failed_pdf_urls = []for url in pdf_urls:pdf_result = _fetch_pdf(url, self._settings)if pdf_result is not None:web_results.append(pdf_result)+ else:+ failed_pdf_urls.append(url)++ browser_urls = html_urls + failed_pdf_urls# Fetch HTML pages with browser- if html_urls:+ if browser_urls:async with self._make_crawler() as crawler:- results = await crawler.arun_many(urls=html_urls, config=self._run_config)- for cr, url in zip(results, html_urls, strict=False):+ results = await crawler.arun_many(urls=browser_urls, config=self._run_config)+ for cr, url in zip(results, browser_urls, strict=False):if not cr.success:continuemetadata = cr.metadata or {}diff --git a/tests/test_cli/test_fetch_batch.py b/tests/test_cli/test_fetch_batch.pynew file mode 100644index 0000000..fc45d89--- /dev/null+++ b/tests/test_cli/test_fetch_batch.py@@ -0,0 +1,62 @@+"""fetch-batch surfaces per-URL failures in JSON output — driven offline with+a fake provider, no network."""++from __future__ import annotations++import json+import os+from pathlib import Path++import pytest+from typer.testing import CliRunner++from hyperresearch.cli import app+from hyperresearch.web.base import WebResult++runner = CliRunner()+++class _FakeProvider:+ """fetch_many raises to force the batch-fallback lane; fetch then succeeds+ for one URL and fails for the other."""++ name = "fake"++ def fetch_many(self, urls):+ raise RuntimeError("batch boom")++ def fetch(self, url):+ if "bad" in url:+ raise RuntimeError("per-url boom")+ return WebResult(url=url, title="Good Page", content="hello world from the good page")+++def vault_dir(tmp_path: Path) -> Path:+ result = runner.invoke(app, ["init", str(tmp_path / "kb"), "--name", "Batch Test"])+ assert result.exit_code == 0+ return tmp_path / "kb"+++def test_fetch_batch_reports_failed_urls(vault_dir: Path, monkeypatch):+ """A URL that fails inside the batch-fallback lane must appear in+ failed_urls with its phase — if it silently vanishes, a caller sees a+ short success list and never learns a source was lost."""+ os.chdir(vault_dir)+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())++ result = runner.invoke(+ app,+ ["fetch-batch", "http://good.example/ok", "http://bad.example/nope", "--json"],+ )+ assert result.exit_code == 0+ data = json.loads(result.output)++ assert data["ok"] is True+ assert data["data"]["total_fetched"] == 1 # the good URL became a note++ failed = data["data"]["failed_urls"]+ assert len(failed) == 1+ assert failed[0]["url"] == "http://bad.example/nope"+ assert failed[0]["phase"] == "batch-fallback"+ assert "per-url boom" in failed[0]["error"]diff --git a/tests/test_core/test_sync.py b/tests/test_core/test_sync.pyindex 09a4d45..e13fcf7 100644--- a/tests/test_core/test_sync.py+++ b/tests/test_core/test_sync.py@@ -75,6 +75,55 @@ def test_sync_populates_tags(seeded_vault):assert "concurrency" in tags+def test_sync_lowercases_tags(tmp_vault):+ """Frontmatter tags must be stored lowercase so SearchFilters (which+ lowercases the query side) can match them. Without this, `--tag llm`+ silently returns zero rows for any note tagged `LLM`."""+ from hyperresearch.core.note import write_note++ write_note(+ tmp_vault.notes_dir,+ "Mixed-case tags note",+ body="# Body\n",+ tags=["LLM", "Mamba", "rust"],+ )+ plan = compute_sync_plan(tmp_vault, force=True)+ execute_sync(tmp_vault, plan)++ rows = tmp_vault.db.execute(+ "SELECT tag FROM tags ORDER BY tag"+ ).fetchall()+ stored = sorted(r["tag"] for r in rows)+ assert stored == ["llm", "mamba", "rust"]+ # Crucially, no uppercase variant slipped through.+ assert "LLM" not in stored+ assert "Mamba" not in stored+++def test_sync_resolves_alias_for_uppercase_tag(tmp_vault):+ """Alias keys are stored lowercase (cli/tag.py), so the lookup must+ lowercase the frontmatter tag FIRST — `ML` must resolve through the+ `ml -> machine-learning` alias, not bypass it."""+ from hyperresearch.core.note import write_note++ tmp_vault.db.execute(+ "INSERT INTO tag_aliases (alias, canonical) VALUES (?, ?)",+ ("ml", "machine-learning"),+ )+ tmp_vault.db.commit()+ write_note(+ tmp_vault.notes_dir,+ "Aliased uppercase tag note",+ body="# Body\n",+ tags=["ML"],+ )+ plan = compute_sync_plan(tmp_vault, force=True)+ execute_sync(tmp_vault, plan)++ stored = [r["tag"] for r in tmp_vault.db.execute("SELECT tag FROM tags")]+ assert stored == ["machine-learning"]++def test_sync_populates_links(seeded_vault):rows = seeded_vault.db.execute("SELECT source_id, target_ref, target_id FROM links"diff --git a/tests/test_search/test_fts.py b/tests/test_search/test_fts.pyindex 0f5c65b..052eb79 100644--- a/tests/test_search/test_fts.py+++ b/tests/test_search/test_fts.py@@ -17,6 +17,28 @@ def test_preprocess_quoted_phrase():assert '"async await"' in result+def test_preprocess_hyphenated_word_stays_a_phrase():+ """Bare tokens are emitted inside double quotes, where FTS5 reads `-` as+ a plain character, not the NOT operator — so `foo-bar` is a valid phrase+ query as-is. It must NOT be split into AND'd tokens, which would loosen+ phrase matching."""+ result = preprocess_query("foo-bar")+ assert '"foo-bar"*' in result+++def test_preprocess_mixed_hyphen_query():+ result = preprocess_query("python 3-async")+ assert '"python"*' in result+ assert '"3-async"*' in result+++def test_preprocess_strips_stray_quote():+ """Quoted phrases survive whole, but a single unbalanced `"` inside a+ bare word would otherwise break FTS5 syntax."""+ result = preprocess_query('foo"bar')+ assert '"foobar"*' in result++def test_preprocess_passthrough_operators():query = "python AND async"assert preprocess_query(query) == querydiff --git a/tests/test_web/test_fetch_many_fallback.py b/tests/test_web/test_fetch_many_fallback.pynew file mode 100644index 0000000..375565f--- /dev/null+++ b/tests/test_web/test_fetch_many_fallback.py@@ -0,0 +1,110 @@+"""Batch-fetch PDF→browser fallback (the #52 fix) — driven entirely offline+with a fake crawler, no browser or network."""++from __future__ import annotations++import asyncio++import pytest++from hyperresearch.web.base import WebResult++provider = pytest.importorskip(+ "hyperresearch.web.crawl4ai_provider",+ reason="crawl4ai extra not installed",+)+++def _run(coro):+ """Run ``coro`` to completion in an isolated thread with its own loop.++ A prior test in the full suite can leave this thread's event-loop state as+ "running" (seen intermittently on Python 3.11 once crawl4ai's async+ machinery has been exercised), which makes a plain ``asyncio.run()`` here+ raise "cannot be called from a running event loop". A fresh thread has no+ running loop, so running the coroutine there is immune to that leaked state.+ """+ import threading++ box: dict = {}++ def target() -> None:+ try:+ box["result"] = asyncio.run(coro)+ except Exception as exc:+ box["error"] = exc++ thread = threading.Thread(target=target)+ thread.start()+ thread.join()+ if "error" in box:+ raise box["error"]+ return box["result"]+++class _FakeCR:+ """Minimal stand-in for a crawl4ai result the fallback loop reads."""++ def __init__(self, url: str):+ self.success = True+ self.url = url+ self.markdown = f"browser text for {url}" # str path -> content = markdown+ self.metadata = {"title": "Fallback Title"}+ self.media = {}+ self.screenshot = None+ self.html = "<html></html>"+++class _FakeCrawler:+ """Async-context-manager crawler that records the URLs it was handed."""++ def __init__(self, captured: list[str]):+ self._captured = captured++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ async def arun_many(self, urls, config):+ self._captured.extend(urls)+ return [_FakeCR(u) for u in urls]+++def _bare_provider(captured: list[str], monkeypatch) -> object:+ inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)+ inst._settings = provider.FetchSettings()+ inst._gates = provider.JunkGates()+ inst._run_config = object()+ monkeypatch.setattr(inst, "_make_crawler", lambda: _FakeCrawler(captured))+ return inst+++def test_failed_pdf_falls_back_to_browser_lane(monkeypatch):+ """A .pdf whose direct fetch returns None must be retried through the+ browser lane. If this fallback regresses, batch fetches silently drop+ academic PDFs that only render behind a JS landing page."""+ monkeypatch.setattr(provider, "_fetch_pdf", lambda url, settings=None: None)+ captured: list[str] = []+ inst = _bare_provider(captured, monkeypatch)++ results = _run(inst._fetch_many_async(["http://8.8.8.8/paper.pdf"]))++ assert captured == ["http://8.8.8.8/paper.pdf"]+ assert len(results) == 1+ assert results[0].content == "browser text for http://8.8.8.8/paper.pdf"+++def test_successful_pdf_does_not_reach_browser_lane(monkeypatch):+ """The mirror case: when the direct PDF fetch succeeds, the .pdf must NOT+ also be handed to the browser, or every academic PDF gets fetched twice."""+ real = WebResult(url="http://8.8.8.8/paper.pdf", title="Paper", content="pdf text")+ monkeypatch.setattr(provider, "_fetch_pdf", lambda url, settings=None: real)+ captured: list[str] = []+ inst = _bare_provider(captured, monkeypatch)++ results = _run(inst._fetch_many_async(["http://8.8.8.8/paper.pdf"]))++ assert captured == [] # browser lane never entered+ assert results == [real]