Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: close four silent-failure leaks (tags, FTS syntax, batch PDFs, cache-busting date) (#52)

Francois du Plessis771df98a6a31
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 945b74f..6860e3d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
## [Unreleased]
+### Four silent-failure leaks closed (tags, FTS syntax, batch PDFs, cache-busting date)
+
+- **Tag filtering:** frontmatter tags are now lowercased before the alias lookup and before storage, matching the lowercase alias keys and the lowercasing `SearchFilters` does on the query side — `--tag llm` now finds notes tagged `LLM`, including through aliases.
+- **FTS queries:** stray unbalanced double-quotes in bare words are stripped instead of raising an FTS5 syntax error that the caller saw as zero results. Hyphenated words are left intact — they were already valid inside the emitted quoted phrase tokens.
+- **Batch PDF fetches:** in `fetch_many`, a PDF whose direct download fails (parser failure, non-PDF body at a `.pdf` URL) now falls back to the browser path, mirroring single-fetch behaviour instead of being silently dropped. `fetch-batch` also falls back to per-URL fetches when a whole batch fails, and reports `failed_urls` in its output instead of losing them silently.
+- **Prompt-cache-busting date:** the vault CLAUDE.md blurb no longer interpolates `Today is YYYY-MM-DD`, which busted Claude Code's prompt cache once per day.
+
## [0.9.0] - 2026-07-23
### Coverage before elegance: the synthesizer stops trading substance for prose
diff --git a/src/hyperresearch/cli/fetch_batch.py b/src/hyperresearch/cli/fetch_batch.py
index 6c3db65..4103201 100644
--- a/src/hyperresearch/cli/fetch_batch.py
+++ b/src/hyperresearch/cli/fetch_batch.py
@@ -94,6 +94,15 @@ def fetch_batch(
visible_urls = [u for u in new_urls if _needs_visible(u)]
results = []
+ failed_urls: list[dict] = [] # surfaced in JSON output so callers see what was lost
+
+ def _fetch_one(provider, url: str, kind: str) -> None:
+ try:
+ results.append(provider.fetch(url))
+ except Exception as exc:
+ failed_urls.append({"url": url, "error": str(exc), "phase": kind})
+ if not json_output:
+ console.print(f" [red]Failed ({kind}):[/] {url} — {exc}")
# Batch fetch normal URLs
if normal_urls:
@@ -101,15 +110,19 @@ def fetch_batch(
try:
results.extend(prov.fetch_many(normal_urls))
except Exception as e:
+ # fetch_many can return zero results on a single bad URL
+ # inside the batch comprehension. Fall back to per-URL so we
+ # don't silently lose the entire wave — partial progress
+ # beats none.
if not json_output:
- console.print(f"[red]Batch fetch failed:[/] {e}")
+ console.print(
+ f"[red]Batch fetch failed:[/] {e}. Falling back to per-URL fetches."
+ )
+ for url in normal_urls:
+ _fetch_one(prov, url, "batch-fallback")
else:
for url in normal_urls:
- try:
- results.append(prov.fetch(url))
- except Exception as e:
- if not json_output:
- console.print(f" [red]Failed:[/] {url} — {e}")
+ _fetch_one(prov, url, "normal")
# Fetch auth-aggressive URLs with visible browser (sequential)
if visible_urls:
@@ -122,11 +135,7 @@ def fetch_batch(
gates=vault.config.junk,
)
for url in visible_urls:
- try:
- results.append(visible_prov.fetch(url))
- except Exception as e:
- if not json_output:
- console.print(f" [red]Failed (visible):[/] {url} — {e}")
+ _fetch_one(visible_prov, url, "visible")
# Phase 1: Write all note files to disk (no sync yet)
note_files = [] # (note_path, url, result, domain, content_hash)
@@ -223,12 +232,16 @@ def fetch_batch(
"notes_created": created_notes,
"total_fetched": len(created_notes),
"skipped": len(all_urls) - len(new_urls),
+ "failed_urls": failed_urls,
}
if json_output:
output(success(data, count=len(created_notes), vault=str(vault.root)), json_mode=True)
else:
- console.print(
+ msg = (
f"\n[bold]Done:[/] {len(created_notes)} notes created, "
- f"{len(all_urls) - len(new_urls)} skipped (already fetched)."
+ f"{len(all_urls) - len(new_urls)} skipped (already fetched)"
)
+ if failed_urls:
+ msg += f", [red]{len(failed_urls)} failed[/]"
+ console.print(msg + ".")
diff --git a/src/hyperresearch/core/agent_docs.py b/src/hyperresearch/core/agent_docs.py
index 92c1b08..36a5972 100644
--- a/src/hyperresearch/core/agent_docs.py
+++ b/src/hyperresearch/core/agent_docs.py
@@ -17,7 +17,7 @@ HYPERRESEARCH_SECTION_END = "<!-- hyperresearch:end -->"
HYPERRESEARCH_BLURB = """
{marker}
-## Research Base (hyperresearch) — Today is {today}
+## Research Base (hyperresearch)
**CLI path: `{hpr}`** — use this exact path for every hyperresearch command. It may not be on your system PATH.
@@ -175,12 +175,12 @@ def inject_agent_docs(vault_root: Path) -> list[str]:
hpr_path = _resolve_executable()
# Use forward slashes — bash on Windows eats backslashes
hpr_path = hpr_path.replace("\\", "/")
- from datetime import date
+ # No date interpolation here: a `Today is YYYY-MM-DD` line in the
+ # cached prefix would bust Claude Code's prompt cache once per day.
blurb = HYPERRESEARCH_BLURB.format(
marker=HYPERRESEARCH_SECTION_MARKER,
end_marker=HYPERRESEARCH_SECTION_END,
hpr=hpr_path,
- today=date.today().isoformat(),
)
modified: list[str] = []
diff --git a/src/hyperresearch/core/sync.py b/src/hyperresearch/core/sync.py
index 3187158..3a5f7c4 100644
--- a/src/hyperresearch/core/sync.py
+++ b/src/hyperresearch/core/sync.py
@@ -276,7 +276,12 @@ def _upsert_note_to_db(conn, note, synced_at: str, file_mtime: float = 0) -> Non
except Exception:
pass
for tag in meta.tags:
- resolved = alias_map.get(tag, tag)
+ # Lowercase the frontmatter tag for BOTH the lookup and the fallback:
+ # alias keys and canonicals are stored lowercase (cli/tag.py), and
+ # SearchFilters lowercases query-side tags, so the stored value must
+ # be lowercase to match. Without this, a frontmatter tag like `LLM`
+ # misses its alias mapping and is invisible to `--tag llm` filtering.
+ resolved = alias_map.get(tag.lower(), tag.lower())
conn.execute("INSERT OR IGNORE INTO tags (note_id, tag) VALUES (?, ?)", (meta.id, resolved))
# Update aliases
diff --git a/src/hyperresearch/search/fts.py b/src/hyperresearch/search/fts.py
index 620503e..69dc16e 100644
--- a/src/hyperresearch/search/fts.py
+++ b/src/hyperresearch/search/fts.py
@@ -48,7 +48,11 @@ def preprocess_query(raw: str) -> str:
words = part.strip().split()
for word in words:
if word:
- clean = re.sub(r'[*^():{}]', '', word)
+ # Strip FTS5-special chars that would break syntax. The
+ # double-quote is included because quoted phrases were
+ # already extracted by the outer split — any quote left
+ # here is an unbalanced fragment.
+ clean = re.sub(r'[*^():{}"]', '', word)
if not clean:
continue
# Split glued alphanumeric (mamba3 -> mamba + 3)
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index f4bbbb6..225dcf9 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -418,17 +418,26 @@ class Crawl4AIProvider:
web_results = []
- # Fetch PDFs directly (no browser needed)
+ # Fetch PDFs directly (no browser needed). When _fetch_pdf returns
+ # None (parser failure, non-PDF body served at a .pdf URL) we fall
+ # back to the browser path — mirrors the single-fetch behaviour in
+ # fetch() so batch callers don't silently lose academic PDFs that
+ # need JS-rendered landing pages.
+ failed_pdf_urls = []
for url in pdf_urls:
pdf_result = _fetch_pdf(url, self._settings)
if pdf_result is not None:
web_results.append(pdf_result)
+ else:
+ failed_pdf_urls.append(url)
+
+ browser_urls = html_urls + failed_pdf_urls
# Fetch HTML pages with browser
- if html_urls:
+ if browser_urls:
async with self._make_crawler() as crawler:
- results = await crawler.arun_many(urls=html_urls, config=self._run_config)
- for cr, url in zip(results, html_urls, strict=False):
+ results = await crawler.arun_many(urls=browser_urls, config=self._run_config)
+ for cr, url in zip(results, browser_urls, strict=False):
if not cr.success:
continue
metadata = cr.metadata or {}
diff --git a/tests/test_cli/test_fetch_batch.py b/tests/test_cli/test_fetch_batch.py
new file mode 100644
index 0000000..fc45d89
--- /dev/null
+++ b/tests/test_cli/test_fetch_batch.py
@@ -0,0 +1,62 @@
+"""fetch-batch surfaces per-URL failures in JSON output — driven offline with
+a fake provider, no network."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from hyperresearch.cli import app
+from hyperresearch.web.base import WebResult
+
+runner = CliRunner()
+
+
+class _FakeProvider:
+ """fetch_many raises to force the batch-fallback lane; fetch then succeeds
+ for one URL and fails for the other."""
+
+ name = "fake"
+
+ def fetch_many(self, urls):
+ raise RuntimeError("batch boom")
+
+ def fetch(self, url):
+ if "bad" in url:
+ raise RuntimeError("per-url boom")
+ return WebResult(url=url, title="Good Page", content="hello world from the good page")
+
+
+def vault_dir(tmp_path: Path) -> Path:
+ result = runner.invoke(app, ["init", str(tmp_path / "kb"), "--name", "Batch Test"])
+ assert result.exit_code == 0
+ return tmp_path / "kb"
+
+
+def test_fetch_batch_reports_failed_urls(vault_dir: Path, monkeypatch):
+ """A URL that fails inside the batch-fallback lane must appear in
+ failed_urls with its phase — if it silently vanishes, a caller sees a
+ short success list and never learns a source was lost."""
+ os.chdir(vault_dir)
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+
+ result = runner.invoke(
+ app,
+ ["fetch-batch", "http://good.example/ok", "http://bad.example/nope", "--json"],
+ )
+ assert result.exit_code == 0
+ data = json.loads(result.output)
+
+ assert data["ok"] is True
+ assert data["data"]["total_fetched"] == 1 # the good URL became a note
+
+ failed = data["data"]["failed_urls"]
+ assert len(failed) == 1
+ assert failed[0]["url"] == "http://bad.example/nope"
+ assert failed[0]["phase"] == "batch-fallback"
+ assert "per-url boom" in failed[0]["error"]
diff --git a/tests/test_core/test_sync.py b/tests/test_core/test_sync.py
index 09a4d45..e13fcf7 100644
--- a/tests/test_core/test_sync.py
+++ b/tests/test_core/test_sync.py
@@ -75,6 +75,55 @@ def test_sync_populates_tags(seeded_vault):
assert "concurrency" in tags
+def test_sync_lowercases_tags(tmp_vault):
+ """Frontmatter tags must be stored lowercase so SearchFilters (which
+ lowercases the query side) can match them. Without this, `--tag llm`
+ silently returns zero rows for any note tagged `LLM`."""
+ from hyperresearch.core.note import write_note
+
+ write_note(
+ tmp_vault.notes_dir,
+ "Mixed-case tags note",
+ body="# Body\n",
+ tags=["LLM", "Mamba", "rust"],
+ )
+ plan = compute_sync_plan(tmp_vault, force=True)
+ execute_sync(tmp_vault, plan)
+
+ rows = tmp_vault.db.execute(
+ "SELECT tag FROM tags ORDER BY tag"
+ ).fetchall()
+ stored = sorted(r["tag"] for r in rows)
+ assert stored == ["llm", "mamba", "rust"]
+ # Crucially, no uppercase variant slipped through.
+ assert "LLM" not in stored
+ assert "Mamba" not in stored
+
+
+def test_sync_resolves_alias_for_uppercase_tag(tmp_vault):
+ """Alias keys are stored lowercase (cli/tag.py), so the lookup must
+ lowercase the frontmatter tag FIRST — `ML` must resolve through the
+ `ml -> machine-learning` alias, not bypass it."""
+ from hyperresearch.core.note import write_note
+
+ tmp_vault.db.execute(
+ "INSERT INTO tag_aliases (alias, canonical) VALUES (?, ?)",
+ ("ml", "machine-learning"),
+ )
+ tmp_vault.db.commit()
+ write_note(
+ tmp_vault.notes_dir,
+ "Aliased uppercase tag note",
+ body="# Body\n",
+ tags=["ML"],
+ )
+ plan = compute_sync_plan(tmp_vault, force=True)
+ execute_sync(tmp_vault, plan)
+
+ stored = [r["tag"] for r in tmp_vault.db.execute("SELECT tag FROM tags")]
+ assert stored == ["machine-learning"]
+
+
def test_sync_populates_links(seeded_vault):
rows = seeded_vault.db.execute(
"SELECT source_id, target_ref, target_id FROM links"
diff --git a/tests/test_search/test_fts.py b/tests/test_search/test_fts.py
index 0f5c65b..052eb79 100644
--- a/tests/test_search/test_fts.py
+++ b/tests/test_search/test_fts.py
@@ -17,6 +17,28 @@ def test_preprocess_quoted_phrase():
assert '"async await"' in result
+def test_preprocess_hyphenated_word_stays_a_phrase():
+ """Bare tokens are emitted inside double quotes, where FTS5 reads `-` as
+ a plain character, not the NOT operator — so `foo-bar` is a valid phrase
+ query as-is. It must NOT be split into AND'd tokens, which would loosen
+ phrase matching."""
+ result = preprocess_query("foo-bar")
+ assert '"foo-bar"*' in result
+
+
+def test_preprocess_mixed_hyphen_query():
+ result = preprocess_query("python 3-async")
+ assert '"python"*' in result
+ assert '"3-async"*' in result
+
+
+def test_preprocess_strips_stray_quote():
+ """Quoted phrases survive whole, but a single unbalanced `"` inside a
+ bare word would otherwise break FTS5 syntax."""
+ result = preprocess_query('foo"bar')
+ assert '"foobar"*' in result
+
+
def test_preprocess_passthrough_operators():
query = "python AND async"
assert preprocess_query(query) == query
diff --git a/tests/test_web/test_fetch_many_fallback.py b/tests/test_web/test_fetch_many_fallback.py
new file mode 100644
index 0000000..375565f
--- /dev/null
+++ b/tests/test_web/test_fetch_many_fallback.py
@@ -0,0 +1,110 @@
+"""Batch-fetch PDF→browser fallback (the #52 fix) — driven entirely offline
+with a fake crawler, no browser or network."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from hyperresearch.web.base import WebResult
+
+provider = pytest.importorskip(
+ "hyperresearch.web.crawl4ai_provider",
+ reason="crawl4ai extra not installed",
+)
+
+
+def _run(coro):
+ """Run ``coro`` to completion in an isolated thread with its own loop.
+
+ A prior test in the full suite can leave this thread's event-loop state as
+ "running" (seen intermittently on Python 3.11 once crawl4ai's async
+ machinery has been exercised), which makes a plain ``asyncio.run()`` here
+ raise "cannot be called from a running event loop". A fresh thread has no
+ running loop, so running the coroutine there is immune to that leaked state.
+ """
+ import threading
+
+ box: dict = {}
+
+ def target() -> None:
+ try:
+ box["result"] = asyncio.run(coro)
+ except Exception as exc:
+ box["error"] = exc
+
+ thread = threading.Thread(target=target)
+ thread.start()
+ thread.join()
+ if "error" in box:
+ raise box["error"]
+ return box["result"]
+
+
+class _FakeCR:
+ """Minimal stand-in for a crawl4ai result the fallback loop reads."""
+
+ def __init__(self, url: str):
+ self.success = True
+ self.url = url
+ self.markdown = f"browser text for {url}" # str path -> content = markdown
+ self.metadata = {"title": "Fallback Title"}
+ self.media = {}
+ self.screenshot = None
+ self.html = "<html></html>"
+
+
+class _FakeCrawler:
+ """Async-context-manager crawler that records the URLs it was handed."""
+
+ def __init__(self, captured: list[str]):
+ self._captured = captured
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ async def arun_many(self, urls, config):
+ self._captured.extend(urls)
+ return [_FakeCR(u) for u in urls]
+
+
+def _bare_provider(captured: list[str], monkeypatch) -> object:
+ inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
+ inst._settings = provider.FetchSettings()
+ inst._gates = provider.JunkGates()
+ inst._run_config = object()
+ monkeypatch.setattr(inst, "_make_crawler", lambda: _FakeCrawler(captured))
+ return inst
+
+
+def test_failed_pdf_falls_back_to_browser_lane(monkeypatch):
+ """A .pdf whose direct fetch returns None must be retried through the
+ browser lane. If this fallback regresses, batch fetches silently drop
+ academic PDFs that only render behind a JS landing page."""
+ monkeypatch.setattr(provider, "_fetch_pdf", lambda url, settings=None: None)
+ captured: list[str] = []
+ inst = _bare_provider(captured, monkeypatch)
+
+ results = _run(inst._fetch_many_async(["http://8.8.8.8/paper.pdf"]))
+
+ assert captured == ["http://8.8.8.8/paper.pdf"]
+ assert len(results) == 1
+ assert results[0].content == "browser text for http://8.8.8.8/paper.pdf"
+
+
+def test_successful_pdf_does_not_reach_browser_lane(monkeypatch):
+ """The mirror case: when the direct PDF fetch succeeds, the .pdf must NOT
+ also be handed to the browser, or every academic PDF gets fetched twice."""
+ real = WebResult(url="http://8.8.8.8/paper.pdf", title="Paper", content="pdf text")
+ monkeypatch.setattr(provider, "_fetch_pdf", lambda url, settings=None: real)
+ captured: list[str] = []
+ inst = _bare_provider(captured, monkeypatch)
+
+ results = _run(inst._fetch_many_async(["http://8.8.8.8/paper.pdf"]))
+
+ assert captured == [] # browser lane never entered
+ assert results == [real]