Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

Incidentals: tier/content_type filtering, gitignore, crawl4ai polish

Jordane3140d3ec8cd
diff --git a/.gitignore b/.gitignore
index 8ecd429..31c9acd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,16 @@ build/
.env
bench/
codex_bench/
+
+# Agent install outputs — generated per-user by `hyperresearch install`.
+# These files contain absolute paths resolved to the installing user's
+# Python environment. NEVER commit them. The authoritative versions live
+# in src/hyperresearch/skills/ (skills) and src/hyperresearch/core/hooks.py
+# (agent templates).
+.claude/
+CLAUDE.md
+AGENTS.md
+GEMINI.md
+
+# Per-vault research output
+research/
diff --git a/src/hyperresearch/cli/search.py b/src/hyperresearch/cli/search.py
index 66ec6e1..cf0fd0f 100644
--- a/src/hyperresearch/cli/search.py
+++ b/src/hyperresearch/cli/search.py
@@ -13,6 +13,8 @@ def search(
tag: list[str] = typer.Option([], "--tag", "-t", help="Filter by tag (AND logic)"),
status: str | None = typer.Option(None, "--status", "-s", help="Filter by status"),
note_type: str | None = typer.Option(None, "--type", help="Filter by type"),
+ tier: str | None = typer.Option(None, "--tier", help="Filter by epistemic tier: ground_truth|institutional|practitioner|commentary|unknown"),
+ content_type: str | None = typer.Option(None, "--content-type", help="Filter by artifact kind: paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown"),
parent: str | None = typer.Option(None, "--parent", "-p", help="Filter by parent topic"),
after: str | None = typer.Option(None, "--after", help="Created after date (YYYY-MM-DD)"),
before: str | None = typer.Option(None, "--before", help="Created before date (YYYY-MM-DD)"),
@@ -32,9 +34,32 @@ def search(
) -> None:
"""Full-text search across all notes."""
from hyperresearch.core.vault import Vault, VaultError
+ from hyperresearch.models.note import ContentType, Tier
from hyperresearch.search.filters import SearchFilters
from hyperresearch.search.fts import search_fts
+ # Validate enums up-front
+ if tier is not None:
+ try:
+ Tier(tier)
+ except ValueError:
+ valid = ", ".join(t.value for t in Tier)
+ if json_output:
+ output(error(f"Invalid --tier '{tier}'. Must be one of: {valid}", "INVALID_TIER"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --tier '{tier}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
+ if content_type is not None:
+ try:
+ ContentType(content_type)
+ except ValueError:
+ valid = ", ".join(c.value for c in ContentType)
+ if json_output:
+ output(error(f"Invalid --content-type '{content_type}'. Must be one of: {valid}", "INVALID_CONTENT_TYPE"), json_mode=True)
+ else:
+ console.print(f"[red]Invalid --content-type '{content_type}'.[/] Must be one of: {valid}")
+ raise typer.Exit(1)
+
try:
vault = Vault.discover()
except VaultError as e:
@@ -49,6 +74,8 @@ def search(
tags=tag or None,
status=status,
note_type=note_type,
+ tier=tier,
+ content_type=content_type,
parent=parent,
after=after,
before=before,
diff --git a/src/hyperresearch/core/vault.py b/src/hyperresearch/core/vault.py
index 785ac15..33c9afa 100644
--- a/src/hyperresearch/core/vault.py
+++ b/src/hyperresearch/core/vault.py
@@ -38,6 +38,10 @@ class Vault:
def db(self) -> sqlite3.Connection:
if self._conn is None:
self._conn = get_connection(self.db_path)
+ # Ensure schema is up to date — init_schema is idempotent and runs
+ # any pending migrations. This fires on every vault open, which is
+ # cheap when there's nothing to migrate.
+ init_schema(self._conn)
return self._conn
@property
diff --git a/src/hyperresearch/search/filters.py b/src/hyperresearch/search/filters.py
index 675a4b5..dfa3d2d 100644
--- a/src/hyperresearch/search/filters.py
+++ b/src/hyperresearch/search/filters.py
@@ -10,6 +10,8 @@ class SearchFilters:
tags: list[str] | None = None
status: str | None = None
note_type: str | None = None
+ tier: str | None = None # Epistemic tier filter
+ content_type: str | None = None # Artifact kind filter
after: str | None = None
before: str | None = None
path_glob: str | None = None
@@ -42,6 +44,14 @@ class SearchFilters:
clauses.append(f"{table_alias}.type = ?")
params.append(self.note_type)
+ if self.tier:
+ clauses.append(f"{table_alias}.tier = ?")
+ params.append(self.tier)
+
+ if self.content_type:
+ clauses.append(f"{table_alias}.content_type = ?")
+ params.append(self.content_type)
+
if self.after:
clauses.append(f"{table_alias}.created >= ?")
params.append(self.after)
diff --git a/src/hyperresearch/search/fts.py b/src/hyperresearch/search/fts.py
index abb2fb6..c5602a9 100644
--- a/src/hyperresearch/search/fts.py
+++ b/src/hyperresearch/search/fts.py
@@ -83,7 +83,7 @@ def search_fts(
sql = f"""
SELECT
- n.id, n.title, n.path, n.status, n.type,
+ n.id, n.title, n.path, n.status, n.type, n.tier, n.content_type,
n.created, n.updated, n.word_count, n.summary,
snippet(notes_fts, 2, '>>>', '<<<', '...', 64) as snippet,
bm25(notes_fts, 0.0, {tw}, {bw}, {tgw}, {aw}) as score,
@@ -113,6 +113,8 @@ def search_fts(
"path": row["path"],
"status": row["status"],
"type": row["type"],
+ "tier": row["tier"] if "tier" in row.keys() else None,
+ "content_type": row["content_type"] if "content_type" in row.keys() else None,
"tags": tag_list,
"created": row["created"],
"updated": row["updated"],
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index af1345d..fca0fe2 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -58,9 +58,15 @@ def _looks_like_binary(text: str) -> bool:
pdf_markers = ("endstream", "endobj", "/Filter", "/FlateDecode", "stream\nx", "%PDF-")
if any(m in sample for m in pdf_markers):
return True
- # High ratio of non-printable or replacement chars
- non_printable = sum(1 for c in sample if ord(c) > 127 or c in '\x00\x01\x02\x03\x04\x05')
- return non_printable > len(sample) * 0.15
+ # High ratio of actual binary/control chars — do NOT flag ord(c) > 127, as that would
+ # falsely reject valid CJK, Arabic, Cyrillic, and other non-ASCII text.
+ # Only flag: true control chars (< 0x20, excluding normal whitespace), the Unicode
+ # replacement character (U+FFFD from bad decoding), and C1 controls (0x80–0x9F).
+ def _is_garbage(c: str) -> bool:
+ o = ord(c)
+ return (o < 0x20 and c not in '\t\n\r\f\v') or c == '\ufffd' or (0x80 <= o <= 0x9F)
+ garbage = sum(1 for c in sample if _is_garbage(c))
+ return garbage > len(sample) * 0.05
def _fetch_pdf(url: str) -> WebResult | None: