Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: stop discarding non-English pages and silently swallowing search errors (#41)

synqing79b92f9bce3c
diff --git a/src/hyperresearch/cli/search.py b/src/hyperresearch/cli/search.py
index cf0fd0f..a358d9a 100644
--- a/src/hyperresearch/cli/search.py
+++ b/src/hyperresearch/cli/search.py
@@ -36,7 +36,7 @@ def search(
from hyperresearch.core.vault import Vault, VaultError
from hyperresearch.models.note import ContentType, Tier
from hyperresearch.search.filters import SearchFilters
- from hyperresearch.search.fts import search_fts
+ from hyperresearch.search.fts import SearchQueryError, search_fts
# Validate enums up-front
if tier is not None:
@@ -97,7 +97,16 @@ def search(
"penalize_deprecated": vault.config.search_penalize_deprecated,
"penalize_stale": vault.config.search_penalize_stale,
}
- results = search_fts(vault.db, query, filters=filters, limit=limit, offset=offset, ranking=ranking)
+ try:
+ results = search_fts(
+ vault.db, query, filters=filters, limit=limit, offset=offset, ranking=ranking
+ )
+ except SearchQueryError as e:
+ if json_output:
+ output(error(str(e), "BAD_QUERY"), json_mode=True)
+ else:
+ console.print(f"[red]Error:[/] {e}")
+ raise typer.Exit(2) from e
# Auto-include body in JSON mode (agents almost always need it)
want_body = include_body or (json_output and not no_body)
diff --git a/src/hyperresearch/mcp/server.py b/src/hyperresearch/mcp/server.py
index 3fb392f..7aa39ce 100644
--- a/src/hyperresearch/mcp/server.py
+++ b/src/hyperresearch/mcp/server.py
@@ -43,7 +43,7 @@ def search_notes(query: str, tag: str = "", status: str = "", parent: str = "",
vault = _get_vault()
vault.auto_sync()
from hyperresearch.search.filters import SearchFilters
- from hyperresearch.search.fts import search_fts
+ from hyperresearch.search.fts import SearchQueryError, search_fts
tags = [t.strip() for t in tag.split(",") if t.strip()] or None
filters = SearchFilters(tags=tags, status=status or None, parent=parent or None)
ranking = {
@@ -55,7 +55,10 @@ def search_notes(query: str, tag: str = "", status: str = "", parent: str = "",
"penalize_deprecated": vault.config.search_penalize_deprecated,
"penalize_stale": vault.config.search_penalize_stale,
}
- results = search_fts(vault.db, query, filters=filters, limit=limit, ranking=ranking)
+ try:
+ results = search_fts(vault.db, query, filters=filters, limit=limit, ranking=ranking)
+ except SearchQueryError as e:
+ return f"Invalid search query: {e}"
for r in results:
row = vault.db.execute("SELECT body FROM note_content WHERE note_id = ?", (r["id"],)).fetchone()
r["body"] = row["body"] if row else ""
diff --git a/src/hyperresearch/search/fts.py b/src/hyperresearch/search/fts.py
index ae48eed..1457978 100644
--- a/src/hyperresearch/search/fts.py
+++ b/src/hyperresearch/search/fts.py
@@ -8,6 +8,14 @@ import sqlite3
from hyperresearch.search.filters import SearchFilters
+class SearchQueryError(ValueError):
+ """The search query could not be turned into a valid FTS5 query.
+
+ Raised instead of returning an empty result set, so a malformed query is
+ never indistinguishable from a topic with no matching notes.
+ """
+
+
def _split_alphanum(word: str) -> list[str]:
"""Split words where letters meet digits: 'mamba3' -> ['mamba', '3'],
'gpt4o' -> ['gpt', '4', 'o'], 'llama3.1' -> ['llama', '3', '1'].
@@ -61,9 +69,22 @@ def search_fts(
include_index: bool = False,
ranking: dict | None = None,
) -> list[dict]:
- """Execute a full-text search against the notes_fts table."""
+ """Execute a full-text search against the notes_fts table.
+
+ Raises:
+ SearchQueryError: the query is empty or has no searchable terms.
+ sqlite3.OperationalError: the FTS index is missing or unreadable. This is
+ deliberately not swallowed — a broken index must not look like a
+ topic with no results.
+ """
fts_query = preprocess_query(query)
+ if not fts_query.strip():
+ raise SearchQueryError(
+ f"Search query {query!r} contains no searchable terms. "
+ "Provide at least one word or quoted phrase."
+ )
+
filter_clause = ""
filter_params: list = []
if filters:
@@ -101,8 +122,13 @@ def search_fts(
try:
rows = conn.execute(sql, params).fetchall()
- except sqlite3.OperationalError:
- return []
+ except sqlite3.OperationalError as exc:
+ msg = str(exc)
+ if "syntax error" in msg or "malformed MATCH" in msg or "unterminated" in msg:
+ raise SearchQueryError(f"Invalid search query {query!r}: {msg}") from exc
+ # Missing table, corrupt index, locked database — surface it. Returning []
+ # here previously made a broken vault indistinguishable from an empty one.
+ raise
results = []
for row in rows:
diff --git a/src/hyperresearch/serve/server.py b/src/hyperresearch/serve/server.py
index 46b2707..e81b4c9 100644
--- a/src/hyperresearch/serve/server.py
+++ b/src/hyperresearch/serve/server.py
@@ -508,8 +508,13 @@ class HyperresearchHandler(BaseHTTPRequestHandler):
self._send(200, body, "Search")
return
- from hyperresearch.search.fts import search_fts
- results = search_fts(self.db, query, limit=50)
+ from hyperresearch.search.fts import SearchQueryError, search_fts
+ try:
+ results = search_fts(self.db, query, limit=50)
+ except SearchQueryError as e:
+ body += f"<p>{html_mod.escape(str(e))}</p>"
+ self._send(200, body, "Search")
+ return
body += f"<p>{len(results)} results</p>\n<div class='results'>\n"
for r in results:
snippet = r["snippet"].replace(">>>", "<mark>").replace("<<<", "</mark>")
diff --git a/src/hyperresearch/web/base.py b/src/hyperresearch/web/base.py
index fb0c0d5..a518043 100644
--- a/src/hyperresearch/web/base.py
+++ b/src/hyperresearch/web/base.py
@@ -6,6 +6,32 @@ from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Protocol, runtime_checkable
+# Whitespace that is legitimate in extracted text.
+_TEXT_WHITESPACE = "\t\n\r\f\v"
+
+
+def is_binary_garbage_char(c: str) -> bool:
+ """True if `c` indicates binary or mis-decoded content rather than real text.
+
+ Deliberately NOT `ord(c) > 127`. Treating all non-ASCII as binary rejects
+ valid CJK, Arabic, Cyrillic, Greek, Hebrew, Thai, and accented Latin text —
+ i.e. most of the non-English web. Only genuine markers of binary data or a
+ failed decode count here.
+ """
+ o = ord(c)
+ if o < 0x20 and c not in _TEXT_WHITESPACE:
+ return True # C0 control characters
+ if c == "�":
+ return True # replacement character — decoding already failed
+ return 0x80 <= o <= 0x9F # C1 control characters
+
+
+def binary_garbage_ratio(text: str) -> float:
+ """Fraction of `text` that looks like binary or mis-decoded content."""
+ if not text:
+ return 0.0
+ return sum(1 for c in text if is_binary_garbage_char(c)) / len(text)
+
@dataclass
class WebResult:
@@ -102,8 +128,7 @@ class WebResult:
if any(m in sample for m in pdf_binary_signals):
return "Binary PDF garbage in content"
- non_printable = sum(1 for c in sample if ord(c) > 127 or c in '\x00\x01\x02\x03\x04\x05')
- if non_printable > len(sample) * 0.15:
+ if binary_garbage_ratio(sample) > 0.05:
return "High ratio of binary/non-printable content"
# Cookie consent / boilerplate pages (short with mostly nav/cookie text)
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index 6ee3623..3a9a74b 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -17,7 +17,7 @@ from datetime import UTC, datetime
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator
from crawl4ai.content_filter_strategy import PruningContentFilter
-from hyperresearch.web.base import WebResult
+from hyperresearch.web.base import WebResult, binary_garbage_ratio
# Fix Windows encoding before crawl4ai's managed browser tries to log Unicode
if sys.platform == "win32":
@@ -56,15 +56,10 @@ def _looks_like_binary(text: str) -> bool:
pdf_markers = ("endstream", "endobj", "/Filter", "/FlateDecode", "stream\nx", "%PDF-")
if any(m in sample for m in pdf_markers):
return True
- # High ratio of actual binary/control chars - do NOT flag ord(c) > 127, as that would
- # falsely reject valid CJK, Arabic, Cyrillic, and other non-ASCII text.
- # Only flag: true control chars (< 0x20, excluding normal whitespace), the Unicode
- # replacement character (U+FFFD from bad decoding), and C1 controls (0x80-0x9F).
- def _is_garbage(c: str) -> bool:
- o = ord(c)
- return (o < 0x20 and c not in '\t\n\r\f\v') or c == '\ufffd' or (0x80 <= o <= 0x9F)
- garbage = sum(1 for c in sample if _is_garbage(c))
- return garbage > len(sample) * 0.05
+ # Shared with WebResult.looks_like_junk() \u2014 keep one implementation, since these
+ # two gates drifting apart is what let `ord(c) > 127` survive in base.py and
+ # silently discard every non-English page.
+ return binary_garbage_ratio(sample) > 0.05
def _fetch_pdf(url: str) -> WebResult | None:
diff --git a/tests/test_search/test_fts.py b/tests/test_search/test_fts.py
index dd1afc5..0f5c65b 100644
--- a/tests/test_search/test_fts.py
+++ b/tests/test_search/test_fts.py
@@ -1,7 +1,11 @@
"""Tests for full-text search."""
+import sqlite3
+
+import pytest
+
from hyperresearch.search.filters import SearchFilters
-from hyperresearch.search.fts import preprocess_query, search_fts
+from hyperresearch.search.fts import SearchQueryError, preprocess_query, search_fts
def test_preprocess_simple_query():
@@ -56,3 +60,27 @@ def test_filter_by_path_glob(seeded_vault):
filters = SearchFilters(path_glob="notes/python/*")
results = search_fts(seeded_vault.db, "python", filters=filters)
assert all("python" in r["path"] for r in results)
+
+
+# --- Degenerate queries must not masquerade as "no results" -------------------
+# Previously every sqlite3.OperationalError was swallowed and [] returned, so an
+# invalid query and a corrupt index both looked identical to an empty topic.
+
+
[email protected]("query", ["", " ", "***", "()", "^^^", "{}"])
+def test_degenerate_query_raises_rather_than_returning_empty(seeded_vault, query):
+ """A query with no searchable terms is an error, not zero results."""
+ with pytest.raises(SearchQueryError):
+ search_fts(seeded_vault.db, query)
+
+
+def test_no_results_is_still_empty_not_an_error(seeded_vault):
+ """A valid query that matches nothing must stay a normal empty result."""
+ assert search_fts(seeded_vault.db, "zzzznonexistenttermzzzz") == []
+
+
+def test_broken_index_surfaces_instead_of_returning_empty(seeded_vault):
+ """A missing FTS table must raise, not look like a topic with no notes."""
+ seeded_vault.db.execute("DROP TABLE IF EXISTS notes_fts")
+ with pytest.raises(sqlite3.OperationalError, match="notes_fts"):
+ search_fts(seeded_vault.db, "python")
diff --git a/tests/test_web/test_junk_detection.py b/tests/test_web/test_junk_detection.py
new file mode 100644
index 0000000..cc87b51
--- /dev/null
+++ b/tests/test_web/test_junk_detection.py
@@ -0,0 +1,111 @@
+"""Junk-detection regression tests.
+
+`looks_like_junk()` previously counted every `ord(c) > 127` character as
+"binary/non-printable", so any page in a non-Latin script — and plenty of
+accented Latin text — was silently discarded as binary garbage.
+"""
+
+# ruff: noqa: RUF001
+# RUF001/RUF003 flag fullwidth punctuation as "ambiguous" ASCII lookalikes.
+# Here it is the correct punctuation for the language being tested, and real
+# non-ASCII text is the entire point of this file — substituting ASCII commas
+# would defeat the regression these fixtures exist to catch.
+
+from __future__ import annotations
+
+import pytest
+
+from hyperresearch.web.base import WebResult, binary_garbage_ratio
+
+# Real prose, not placeholder shapes — a filter that passes on `"x" * 500`
+# but discards actual Chinese text would still be broken.
+CHINESE = (
+ "锂离子电池的能量密度持续提升,但热失控风险仍然是电动汽车安全的核心问题。"
+ "研究人员正在开发固态电解质,以降低易燃性并提高循环寿命。"
+ "本文综述了近年来固态电池在材料、界面工程和规模化生产方面的进展,"
+ "并讨论了当前技术路线在成本与产能爬坡方面面临的主要挑战。"
+) * 3
+
+JAPANESE = (
+ "半導体産業におけるサプライチェーンの再編は、地政学的な要因によって加速している。"
+ "各国政府は国内での生産能力の拡大を支援するために大規模な補助金を導入しており、"
+ "製造装置メーカーは需要の急増に対応するため生産体制の見直しを迫られている。"
+) * 3
+
+ARABIC = (
+ "تشهد صناعة الطاقة المتجددة نموًا سريعًا في منطقة الشرق الأوسط، "
+ "حيث تستثمر الحكومات مبالغ كبيرة في مشاريع الطاقة الشمسية وطاقة الرياح "
+ "بهدف تنويع مصادر الدخل وتقليل الاعتماد على الوقود الأحفوري."
+) * 3
+
+RUSSIAN = (
+ "Исследования в области квантовых вычислений продвигаются быстрее, чем ожидалось. "
+ "Основной проблемой остаётся коррекция ошибок, требующая значительного числа "
+ "физических кубитов для кодирования одного логического кубита."
+) * 3
+
+ACCENTED_LATIN = (
+ "L'évolution des systèmes énergétiques européens dépend fortement des "
+ "investissements dans les réseaux électriques transfrontaliers. "
+ "Les décideurs doivent équilibrer sécurité d'approvisionnement, coûts et "
+ "objectifs climatiques — un arbitrage particulièrement délicat."
+) * 3
+
+
+ "name,text",
+ [
+ ("chinese", CHINESE),
+ ("japanese", JAPANESE),
+ ("arabic", ARABIC),
+ ("russian", RUSSIAN),
+ ("accented_latin", ACCENTED_LATIN),
+ ],
+)
+def test_non_english_pages_are_not_junk(name: str, text: str):
+ """Non-Latin and accented text is real content, not binary garbage."""
+ result = WebResult(url=f"https://example.com/{name}", title=name, content=text)
+ assert result.looks_like_junk() is None, (
+ f"{name} page was rejected as junk: {result.looks_like_junk()}"
+ )
+
+
+def test_binary_content_is_still_rejected():
+ """The filter must still catch genuinely binary content."""
+ binary = "".join(chr(i % 32) for i in range(2000))
+ result = WebResult(url="https://example.com/bin", title="bin", content=binary)
+ assert result.looks_like_junk() == "High ratio of binary/non-printable content"
+
+
+def test_mojibake_is_still_rejected():
+ """U+FFFD means decoding already failed — that content is not trustworthy."""
+ broken = "�" * 400 + "some readable text " * 20
+ result = WebResult(url="https://example.com/bad", title="bad", content=broken)
+ assert result.looks_like_junk() == "High ratio of binary/non-printable content"
+
+
+def test_pdf_structure_markers_still_rejected():
+ """Raw PDF internals must not be saved as article text."""
+ pdf_junk = "%PDF-1.4\n" + "endobj endstream /FlateDecode " * 100
+ result = WebResult(url="https://example.com/x.pdf", title="x", content=pdf_junk)
+ assert result.looks_like_junk() == "Binary PDF garbage in content"
+
+
+def test_binary_garbage_ratio_ignores_script():
+ """The ratio must not depend on whether text happens to be ASCII."""
+ assert binary_garbage_ratio(CHINESE) == 0.0
+ assert binary_garbage_ratio("plain ascii text") == 0.0
+ assert binary_garbage_ratio("\x00\x01\x02\x03") == 1.0
+
+
+def test_tabs_and_newlines_are_not_garbage():
+ """Ordinary whitespace is legitimate in extracted text."""
+ assert binary_garbage_ratio("a\tb\nc\r\nd\f\ve") == 0.0
+
+
+def test_single_shared_implementation():
+ """crawl4ai's binary check must not drift from the one in base.py again."""
+ from hyperresearch.web.crawl4ai_provider import _looks_like_binary
+
+ assert _looks_like_binary(CHINESE) is False
+ assert _looks_like_binary("\x00\x01\x02" * 700) is True