Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: new notes skip indexed ids, and fetch records only indexed notes (#160)

Francois du Plessis327498be58f4
diff --git a/src/hyperresearch/cli/escalation_cmd.py b/src/hyperresearch/cli/escalation_cmd.py
index 66ba4bc..3798070 100644
--- a/src/hyperresearch/cli/escalation_cmd.py
+++ b/src/hyperresearch/cli/escalation_cmd.py
@@ -15,6 +15,7 @@ from pathlib import Path
from urllib.parse import urlparse
import typer
+from rich.markup import escape
from hyperresearch.cli._output import console, output
from hyperresearch.models.output import error, success
@@ -126,7 +127,8 @@ def escalation_ingest(
Notes carry `fetch_provider: chrome` — browser-lane provenance is visible.
"""
from hyperresearch.core.escalation import EscalationError, list_items, resolve
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.fetcher import discard_unindexed_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.scholar import extract_doi
from hyperresearch.core.sync import compute_sync_plan, execute_sync
@@ -169,6 +171,7 @@ def escalation_ingest(
note_path = write_note(
vault.notes_dir, title=title, body=body, tags=all_tags,
status="draft", source=url, extra_frontmatter=extra_meta,
+ taken_ids=indexed_note_ids(conn),
)
note_id = note_path.stem
@@ -176,6 +179,16 @@ def escalation_ingest(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
+ # A note sync did not index under its own id gets no source row (it would
+ # point at whichever file holds that id), and the item stays claimed.
+ refused = discard_unindexed_note(vault, note_path)
+ if refused:
+ if json_output:
+ output(error(refused, "NOTE_NOT_INDEXED"), json_mode=True)
+ else:
+ console.print(f"[red]Error:[/] {escape(refused)}")
+ raise typer.Exit(1)
+
content_hash = hashlib.sha256(body.encode("utf-8")).hexdigest()[:16]
conn.execute(
"""INSERT OR IGNORE INTO sources (url, note_id, domain, fetched_at, provider, content_hash)
diff --git a/src/hyperresearch/cli/fetch.py b/src/hyperresearch/cli/fetch.py
index 90c4fde..0e7c0f2 100644
--- a/src/hyperresearch/cli/fetch.py
+++ b/src/hyperresearch/cli/fetch.py
@@ -9,10 +9,15 @@ from pathlib import Path
from urllib.parse import urlparse
import typer
+from rich.markup import escape
from hyperresearch.cli._output import console, err_console, output
from hyperresearch.core.config import AssetSettings, FetchSettings
-from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
+from hyperresearch.core.fetcher import (
+ discard_unindexed_note,
+ existing_live_note_for_url,
+ reclaim_orphaned_source_row,
+)
from hyperresearch.models.output import error, success
app = typer.Typer()
@@ -226,7 +231,7 @@ def fetch(
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
"""Fetch a URL and save its content as a research note."""
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.sync import compute_sync_plan, execute_sync
from hyperresearch.core.vault import Vault, VaultError
from hyperresearch.web.base import get_provider
@@ -493,6 +498,7 @@ def fetch(
tier=detected_tier,
content_type=detected_content_type,
extra_frontmatter=extra_meta,
+ taken_ids=indexed_note_ids(conn),
)
# Save raw file (PDF, etc.) if present
@@ -535,6 +541,16 @@ def fetch(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
+ # The id is free when write_note picks it, but another file can still take
+ # it before this sync. Recording the source then would wire it to that note.
+ refused = discard_unindexed_note(vault, note_path, raw_file_path)
+ if refused:
+ if json_output:
+ output(error(refused, "NOTE_NOT_INDEXED"), json_mode=True)
+ else:
+ console.print(f"[red]Not indexed:[/] {escape(refused)}")
+ raise typer.Exit(1)
+
# Record source in DB. Use INSERT OR IGNORE to survive a duplicate-URL
# race: two parallel fetches on the same URL both pass the earlier
# duplicate-check SELECT, both try to INSERT, and without IGNORE the
diff --git a/src/hyperresearch/cli/fetch_batch.py b/src/hyperresearch/cli/fetch_batch.py
index a1c9f58..ae9b127 100644
--- a/src/hyperresearch/cli/fetch_batch.py
+++ b/src/hyperresearch/cli/fetch_batch.py
@@ -7,9 +7,14 @@ import sys
from urllib.parse import urlparse
import typer
+from rich.markup import escape
from hyperresearch.cli._output import console, output
-from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
+from hyperresearch.core.fetcher import (
+ discard_unindexed_note,
+ existing_live_note_for_url,
+ reclaim_orphaned_source_row,
+)
from hyperresearch.models.output import error, success
@@ -24,7 +29,7 @@ def fetch_batch(
) -> None:
"""Fetch multiple URLs and save each as a research note. Batched sync for speed."""
from hyperresearch.core.enrich import enrich_note_file
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.oa import (
oa_frontmatter,
recover_full_text,
@@ -187,6 +192,8 @@ def fetch_batch(
# Phase 1: Write all note files to disk (no sync yet)
note_files = [] # (note_path, url, result, domain, content_hash, oa_location, rescue_reason)
+ # Ids written in this batch exist on disk, which write_note checks itself.
+ taken_ids = indexed_note_ids(conn)
for url, result, oa_location, rescue_reason in pending:
# Open-access recovery. This path writes its own notes rather than
# going through cli.fetch, so the DOI capture and the OA swap both have
@@ -240,6 +247,7 @@ def fetch_batch(
source=url,
parent=parent,
extra_frontmatter=extra_meta,
+ taken_ids=taken_ids,
)
# Auto-enrich before sync
@@ -268,6 +276,20 @@ def fetch_batch(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
+ # A note sync did not index under its own id gets no source row: the row
+ # would point at whichever file holds that id. Checked before Phase 3,
+ # whose inserts open a transaction the clean-up's re-sync cannot nest in.
+ indexed_files = []
+ for entry in note_files:
+ refused = discard_unindexed_note(vault, entry[0])
+ if refused is None:
+ indexed_files.append(entry)
+ continue
+ failed_urls.append({"url": entry[1], "error": refused, "phase": "index"})
+ if not json_output:
+ console.print(f" [red]Not indexed:[/] {escape(refused)}")
+ note_files = indexed_files
+
# Phase 3: Bulk-insert source records
created_notes = []
for note_path, url, result, domain, content_hash, oa_location, rescue_reason in note_files:
diff --git a/src/hyperresearch/cli/graph.py b/src/hyperresearch/cli/graph.py
index a2870a5..bc82f53 100644
--- a/src/hyperresearch/cli/graph.py
+++ b/src/hyperresearch/cli/graph.py
@@ -191,7 +191,7 @@ def graph_stub(
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
"""Create stub notes for all broken [[links]]."""
- from hyperresearch.core.note import stub_summary, write_note
+ from hyperresearch.core.note import indexed_note_ids, stub_summary, write_note
from hyperresearch.core.vault import Vault
vault = Vault.discover()
@@ -232,6 +232,9 @@ def graph_stub(
note_id=target,
status="draft",
summary=stub_summary(target),
+ # The resolver never matches a ref by its slug, so a broken
+ # [[foo bar]] can slugify to an id another note already holds.
+ taken_ids=indexed_note_ids(vault.db),
)
created.append({"id": target, "title": title, "path": path.relative_to(vault.root).as_posix()})
diff --git a/src/hyperresearch/cli/note.py b/src/hyperresearch/cli/note.py
index 3b652a7..bca0cbe 100644
--- a/src/hyperresearch/cli/note.py
+++ b/src/hyperresearch/cli/note.py
@@ -38,7 +38,7 @@ def note_new(
import sys
from pathlib import Path as P
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.vault import Vault
from hyperresearch.models.note import ContentType, Tier, slugify
@@ -116,11 +116,13 @@ def note_new(
console.print(f"[yellow]Template '{template}' not found, using default.[/]")
path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,
note_type=note_type, parent=parent, summary=summary,
- source=source, tier=tier, content_type=content_type)
+ source=source, tier=tier, content_type=content_type,
+ taken_ids=indexed_note_ids(vault.db))
else:
path = write_note(vault.notes_dir, title, body=body, tags=tags, status=status,
note_type=note_type, parent=parent, summary=summary,
- source=source, tier=tier, content_type=content_type)
+ source=source, tier=tier, content_type=content_type,
+ taken_ids=indexed_note_ids(vault.db))
# Sync the new file into the DB so type/tags are indexed immediately
from hyperresearch.core.sync import compute_sync_plan, execute_sync
diff --git a/src/hyperresearch/cli/repair.py b/src/hyperresearch/cli/repair.py
index a08f8bb..8daf3aa 100644
--- a/src/hyperresearch/cli/repair.py
+++ b/src/hyperresearch/cli/repair.py
@@ -53,7 +53,7 @@ def repair(
if stub_broken:
if not json_output:
console.print("[bold]2/6 Stubbing broken links...[/]")
- from hyperresearch.core.note import stub_summary, write_note
+ from hyperresearch.core.note import indexed_note_ids, stub_summary, write_note
rows = vault.db.execute(
"SELECT DISTINCT target_ref FROM links WHERE target_id IS NULL"
).fetchall()
@@ -67,6 +67,7 @@ def repair(
body=f"# {title}\n\n*Stub — created to resolve a broken link. Expand this note.*\n",
note_id=target, status="draft",
summary=stub_summary(target),
+ taken_ids=indexed_note_ids(vault.db),
)
stubs_created += 1
except Exception as exc:
diff --git a/src/hyperresearch/cli/research.py b/src/hyperresearch/cli/research.py
index e3b874d..5956972 100644
--- a/src/hyperresearch/cli/research.py
+++ b/src/hyperresearch/cli/research.py
@@ -8,9 +8,14 @@ from datetime import UTC, datetime
from urllib.parse import urlparse
import typer
-
-from hyperresearch.cli._output import console, output
-from hyperresearch.core.fetcher import existing_live_note_for_url, reclaim_orphaned_source_row
+from rich.markup import escape
+
+from hyperresearch.cli._output import console, err_console, output
+from hyperresearch.core.fetcher import (
+ discard_unindexed_note,
+ existing_live_note_for_url,
+ reclaim_orphaned_source_row,
+)
from hyperresearch.models.output import error, success
@@ -25,7 +30,7 @@ def research(
json_output: bool = typer.Option(False, "--json", "-j", help="JSON output"),
) -> None:
"""Deep research: search the web, fetch results, save as linked notes, generate synthesis."""
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.sync import compute_sync_plan, execute_sync
from hyperresearch.core.vault import Vault, VaultError
from hyperresearch.models.note import slugify
@@ -160,7 +165,10 @@ def research(
note_type="moc",
parent=parent,
summary=f"Research synthesis: {topic} ({len(created_notes)} sources)",
+ taken_ids=indexed_note_ids(conn),
)
+ # The id write_note chose: a second run on the same topic gets "-2".
+ moc_id = moc_path.stem
# Final sync to pick up the MOC
plan = compute_sync_plan(vault)
@@ -228,7 +236,7 @@ def research(
def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
"""Save a single WebResult as a note. Returns note data dict or None if skipped."""
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
url = result.url
@@ -260,6 +268,7 @@ def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
source=url,
parent=parent,
extra_frontmatter=extra_meta,
+ taken_ids=indexed_note_ids(conn),
)
# Auto-enrich: add suggested tags and summary before sync
@@ -275,6 +284,13 @@ def _save_result(vault, conn, prov, result, tags, parent) -> dict | None:
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
+ # A note sync did not index under its own id is skipped: its source row
+ # would point at whichever file holds that id.
+ refused = discard_unindexed_note(vault, note_path)
+ if refused:
+ err_console.print(f"[yellow]Skipped:[/] {escape(refused)}")
+ return None
+
conn.execute(
"""INSERT OR IGNORE INTO sources (url, note_id, domain, fetched_at, provider, content_hash)
VALUES (?, ?, ?, ?, ?, ?)""",
diff --git a/src/hyperresearch/core/fetcher.py b/src/hyperresearch/core/fetcher.py
index b8f5312..69222cc 100644
--- a/src/hyperresearch/core/fetcher.py
+++ b/src/hyperresearch/core/fetcher.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import sqlite3
+from pathlib import Path
from urllib.parse import urlparse
@@ -50,6 +51,38 @@ def reclaim_orphaned_source_row(
)
+def discard_unindexed_note(vault, note_path: Path, raw_file: str | None = None) -> str | None:
+ """Check, after sync, that the notes row for ``note_path``'s id is this file.
+
+ The fetch paths record ``sources`` and ``assets`` under ``note_path.stem``.
+ When sync refused the file (another file already holds that id), those rows
+ would land on the other note: the duplicate-url check then answers
+ "already fetched" with the wrong note, and the fetched text is never
+ indexed. Returns None when the file is indexed under its id. Otherwise,
+ like the duplicate-url race path in ``cli/fetch.py``, removes the file and
+ its raw artifact (``raw_file``, relative to ``research/``), re-syncs, and
+ returns an error naming the file and the id's owner.
+ """
+ from hyperresearch.core.sync import compute_sync_plan, execute_sync
+
+ note_id = note_path.stem
+ rel = note_path.relative_to(vault.root).as_posix()
+ row = vault.db.execute("SELECT path FROM notes WHERE id = ?", (note_id,)).fetchone()
+ if row is not None and row["path"] == rel:
+ return None
+ note_path.unlink(missing_ok=True)
+ if raw_file:
+ (vault.root / "research" / raw_file).unlink(missing_ok=True)
+ plan = compute_sync_plan(vault)
+ if plan.to_delete:
+ execute_sync(vault, plan)
+ owner = f"belongs to {row['path']}" if row is not None else "is not in the index"
+ return (
+ f"{rel} was not indexed: its id '{note_id}' {owner}. "
+ "The file was removed and no source was recorded, so the fetch can be retried."
+ )
+
+
def fetch_and_save(
vault,
url: str,
@@ -64,9 +97,9 @@ def fetch_and_save(
Raises:
ValueError: If URL is already fetched.
- RuntimeError: If fetch fails.
+ RuntimeError: If fetch fails, or sync does not index the new note.
"""
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.sync import compute_sync_plan, execute_sync
from hyperresearch.web.base import get_provider
@@ -159,6 +192,7 @@ def fetch_and_save(
source=url,
parent=parent,
extra_frontmatter=extra_meta,
+ taken_ids=indexed_note_ids(conn),
)
# Save raw file (PDF, etc.) if present
@@ -201,6 +235,10 @@ def fetch_and_save(
if plan.to_add or plan.to_update:
execute_sync(vault, plan)
+ refused = discard_unindexed_note(vault, note_path, raw_file_path)
+ if refused:
+ raise RuntimeError(refused)
+
# Record source (upsert: an orphaned row for this url may already exist)
content_hash = hashlib.sha256(result.content.encode("utf-8")).hexdigest()[:16]
conn.execute(
diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.py
index 5d5c9c4..03cd802 100644
--- a/src/hyperresearch/core/note.py
+++ b/src/hyperresearch/core/note.py
@@ -82,6 +82,11 @@ def _collision_id(base: str, counter: int) -> str:
return slugify(f"{trimmed.rstrip('-')}{suffix}")
+def indexed_note_ids(conn) -> set[str]:
+ """Every id the notes table holds, for ``write_note(taken_ids=...)``."""
+ return {row["id"] for row in conn.execute("SELECT id FROM notes")}
+
+
def write_note(
notes_dir: Path,
title: str,
@@ -97,6 +102,7 @@ def write_note(
tier: str | None = None,
content_type: str | None = None,
extra_frontmatter: dict | None = None,
+ taken_ids: set[str] | None = None,
) -> Path:
"""Create a new note file on disk. Returns the file path.
@@ -105,6 +111,11 @@ def write_note(
tier: Epistemic role — ground_truth|institutional|practitioner|commentary|unknown.
content_type: Artifact kind — paper|docs|article|blog|forum|dataset|policy|code|book|transcript|review|unknown.
extra_frontmatter: Additional fields to set on NoteMeta (e.g. source_domain, fetched_at).
+ taken_ids: Ids to skip even when no file of that name exists, such as
+ the ids already in the notes table. A note keeps its id when its
+ file is renamed (`note mv`), so `<id>.md` can be free on disk while
+ the id is not; a note written there is refused by sync as an id
+ collision.
"""
# The id must be a fixed point of the slugifier. NoteMeta.ensure_slug
# already slugifies the FRONTMATTER id, so an unslugified note_id here
@@ -142,7 +153,8 @@ def write_note(
file_path = target_dir / f"{nid}.md"
counter = 2
- while file_path.exists():
+ taken = taken_ids or set()
+ while file_path.exists() or meta.id in taken:
candidate = _collision_id(nid, counter)
file_path = target_dir / f"{candidate}.md"
meta.id = candidate
diff --git a/src/hyperresearch/mcp/server.py b/src/hyperresearch/mcp/server.py
index 8ef18fa..7f13ea3 100644
--- a/src/hyperresearch/mcp/server.py
+++ b/src/hyperresearch/mcp/server.py
@@ -334,7 +334,7 @@ def create_note(title: str, body: str, tags: str = "", source: str = "", summary
summary: One-line summary (auto-generated if empty)
"""
from hyperresearch.core.enrich import enrich_note_file
- from hyperresearch.core.note import write_note
+ from hyperresearch.core.note import indexed_note_ids, write_note
from hyperresearch.core.sync import compute_sync_plan, execute_sync
vault = _get_vault()
@@ -354,6 +354,7 @@ def create_note(title: str, body: str, tags: str = "", source: str = "", summary
source=source or None,
summary=summary or None,
extra_frontmatter=extra if extra else None,
+ taken_ids=indexed_note_ids(vault.db),
)
enrich_note_file(note_path, vault.db, tag_list)
diff --git a/tests/test_cli/test_note_id_taken.py b/tests/test_cli/test_note_id_taken.py
new file mode 100644
index 0000000..71c4321
--- /dev/null
+++ b/tests/test_cli/test_note_id_taken.py
@@ -0,0 +1,405 @@
+"""A new note must not be written under an id another file already holds.
+
+`note mv` keeps a note's id while renaming its file, so after
+`note mv topic topic-moved` the filename `topic.md` is free on disk but the id
+`topic` is not. write_note only checked for an existing FILE, so the next note
+titled "Topic" was written to `topic.md` and sync refused it as an id collision.
+Each fetch path then recorded the URL (and its assets) under `note_id = "topic"`,
+the moved note: the duplicate-url check answered "already fetched" with that
+note from then on, and the fetched text was never indexed. `note new`, MCP
+`create_note` and the research map-of-content reported the moved note's id for
+a file that was never indexed.
+
+The callers now pass the indexed ids to write_note, and the fetch paths, after
+sync, record nothing unless the notes row for the id is the new file (the race
+in which another file takes the id between write_note and sync). The fetch
+paths are covered at all five call sites: `fetch`, `fetch-batch`, research's
+`_save_result`, `escalation ingest` and `fetch_and_save`; the other callers
+below them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from hyperresearch.cli import app
+from hyperresearch.core.vault import Vault
+from hyperresearch.web.base import WebResult
+
+runner = CliRunner()
+
+FIRST_URL = "https://a.example.org/x"
+URL = "https://b.example.org/y"
+
+
+class _FakeProvider:
+ """Every page has the same title, so every note's id starts as `topic`."""
+
+ name = "fake"
+
+ def fetch(self, url: str) -> WebResult:
+ return WebResult(
+ url=url,
+ title="Topic",
+ content="enough content to clear the junk gate. " * 10,
+ screenshot=b"screenshot of " + url.encode(),
+ raw_bytes=b"%PDF-1.4 stand-in for " + url.encode(),
+ raw_content_type="application/pdf",
+ )
+
+ def fetch_many(self, urls):
+ return [self.fetch(u) for u in urls]
+
+
+def vault_dir(tmp_path: Path, monkeypatch) -> Path:
+ result = runner.invoke(app, ["init", str(tmp_path / "kb"), "--name", "Id Taken Test"])
+ assert result.exit_code == 0
+ os.chdir(tmp_path / "kb")
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _FakeProvider())
+ return tmp_path / "kb"
+
+
+def moved_topic(vault_dir: Path) -> Path:
+ """Fetch a page titled "Topic", then `note mv` it: the id `topic` stays
+ taken while the filename `topic.md` is free again."""
+ first = runner.invoke(app, ["fetch", FIRST_URL, "--json"])
+ assert first.exit_code == 0, first.output
+ assert json.loads(first.stdout)["data"]["note_id"] == "topic"
+ moved = runner.invoke(app, ["note", "mv", "topic", "topic-moved", "--json"])
+ assert moved.exit_code == 0, moved.output
+ assert not (vault_dir / "research" / "notes" / "topic.md").exists()
+ assert _note_path(vault_dir, "topic") == "research/notes/topic-moved.md"
+ return vault_dir
+
+
+def id_taken_before_sync(monkeypatch):
+ """Force the race: right after write_note returns, another file claiming
+ the new note's id is written and indexed (alone: the new note stays
+ unindexed), before the caller's own sync."""
+ import hyperresearch.core.note as note_mod
+ from hyperresearch.core.sync import SyncPlan, execute_sync
+
+ real_write_note = note_mod.write_note
+ raced: list[Path] = []
+
+ def write_note_then_race(notes_dir, *args, **kwargs):
+ path = real_write_note(notes_dir, *args, **kwargs)
+ if not raced:
+ rival = notes_dir / "rival.md"
+ rival.write_text(
+ f"---\ntitle: Rival\nid: {path.stem}\n---\n\nRival body.\n", encoding="utf-8"
+ )
+ with Vault.discover(notes_dir) as other:
+ execute_sync(other, SyncPlan(to_add=[rival]))
+ raced.append(path)
+ return path
+
+ monkeypatch.setattr(note_mod, "write_note", write_note_then_race)
+ return raced
+
+
+def _db(vault_dir: Path):
+ return Vault.discover(vault_dir).db
+
+
+def _note_path(vault_dir: Path, note_id: str) -> str | None:
+ row = _db(vault_dir).execute("SELECT path FROM notes WHERE id = ?", (note_id,)).fetchone()
+ return row["path"] if row else None
+
+
+def _source_note_id(vault_dir: Path, url: str):
+ row = _db(vault_dir).execute("SELECT note_id FROM sources WHERE url = ?", (url,)).fetchone()
+ return row["note_id"] if row else "<no row>"
+
+
+def _assert_recorded_under_own_note(vault_dir: Path, note_id: str) -> None:
+ """The url's source row names the new note, and that id's notes row is the
+ new note's own file rather than the moved one."""
+ assert note_id != "topic"
+ assert _note_path(vault_dir, note_id) == f"research/notes/{note_id}.md"
+ assert _source_note_id(vault_dir, URL) == note_id
+ assert _note_path(vault_dir, "topic") == "research/notes/topic-moved.md"
+ sync = runner.invoke(app, ["sync", "--json"])
+ assert json.loads(sync.stdout)["data"]["errors"] == [] # no file left unindexed
+
+
+def _assert_nothing_recorded(vault_dir: Path, raced: list[Path]) -> None:
+ """Refused by the guard: no source row, no assets, the stray file removed,
+ and the id still belongs to the file that took it."""
+ assert raced, "the race fixture never ran"
+ stray = raced[0]
+ assert not stray.exists()
+ assert _source_note_id(vault_dir, URL) == "<no row>"
+ assert _note_path(vault_dir, stray.stem) == "research/notes/rival.md"
+ assets = _db(vault_dir).execute("SELECT COUNT(*) FROM assets").fetchone()[0]
+ assert assets == 0
+
+
+# --- the ordinary case: an id held by a moved note is skipped ---------------
+
+
+def test_fetch_after_note_mv_records_the_new_note(moved_topic: Path):
+ result = runner.invoke(app, ["fetch", URL, "--save-assets", "--json"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.stdout)["data"]
+ _assert_recorded_under_own_note(moved_topic, data["note_id"])
+
+ asset_ids = {r["note_id"] for r in _db(moved_topic).execute("SELECT note_id FROM assets")}
+ assert asset_ids == {data["note_id"]} # the screenshot is not filed under the moved note
+
+ again = runner.invoke(app, ["fetch", URL, "--json"])
+ assert again.exit_code == 1
+ assert json.loads(again.stdout)["error"] == f"URL already fetched as note '{data['note_id']}'"
+
+
+def test_fetch_batch_after_note_mv_records_the_new_note(moved_topic: Path):
+ result = runner.invoke(app, ["fetch-batch", URL, "--json"])
+ assert result.exit_code == 0, result.output
+ created = json.loads(result.stdout)["data"]["notes_created"]
+ assert len(created) == 1
+ _assert_recorded_under_own_note(moved_topic, created[0]["note_id"])
+
+
+def test_save_result_after_note_mv_records_the_new_note(moved_topic: Path):
+ from hyperresearch.cli.research import _save_result
+
+ vault = Vault.discover(moved_topic)
+ prov = _FakeProvider()
+ data = _save_result(vault, vault.db, prov, prov.fetch(URL), [], None)
+ assert data is not None
+ _assert_recorded_under_own_note(moved_topic, data["note_id"])
+
+
+def _claimed_escalation_item(vault_dir: Path) -> tuple[int, Path]:
+ added = runner.invoke(
+ app, ["escalation", "add", URL, "--reason", "interactive_needed", "--json"]
+ )
+ assert added.exit_code == 0, added.output
+ item_id = json.loads(added.stdout)["data"]["id"]
+ assert runner.invoke(app, ["escalation", "claim", "--json"]).exit_code == 0
+ body = vault_dir / "scratch-body.md"
+ body.write_text("Extracted page content with plenty of real words in it.", encoding="utf-8")
+ return item_id, body
+
+
+def test_escalation_ingest_after_note_mv_records_the_new_note(moved_topic: Path):
+ item_id, body = _claimed_escalation_item(moved_topic)
+ result = runner.invoke(
+ app,
+ ["escalation", "ingest", str(item_id), "--title", "Topic", "--body-file", str(body), "--json"],
+ )
+ assert result.exit_code == 0, result.output
+ _assert_recorded_under_own_note(moved_topic, json.loads(result.stdout)["data"]["note_id"])
+
+
+def test_fetch_and_save_after_note_mv_records_the_new_note(moved_topic: Path):
+ from hyperresearch.core.fetcher import fetch_and_save
+
+ data = fetch_and_save(Vault.discover(moved_topic), URL)
+ _assert_recorded_under_own_note(moved_topic, data["note_id"])
+
+
+# --- the race: another file takes the id between write_note and sync --------
+
+
+def test_fetch_records_nothing_when_the_id_is_taken_before_sync(
+ vault_dir: Path, id_taken_before_sync
+):
+ result = runner.invoke(app, ["fetch", URL, "--save-assets", "--json"])
+ assert result.exit_code == 1, result.output
+ payload = json.loads(result.stdout)
+ assert payload["error_code"] == "NOTE_NOT_INDEXED"
+ assert "research/notes/topic.md" in payload["error"] # the refused file
+ assert "research/notes/rival.md" in payload["error"] # the id's owner
+ _assert_nothing_recorded(vault_dir, id_taken_before_sync)
+ assert not (vault_dir / "research" / "raw" / "topic.pdf").exists() # its raw artifact too
+
+ retry = runner.invoke(app, ["fetch", URL, "--json"]) # the race does not repeat
+ assert retry.exit_code == 0, retry.output
+ data = json.loads(retry.stdout)["data"]
+ assert _source_note_id(vault_dir, URL) == data["note_id"] != "topic"
+ assert data["raw_file"] == f"raw/{data['note_id']}.pdf"
+
+
+def test_fetch_batch_records_nothing_when_the_id_is_taken_before_sync(
+ vault_dir: Path, id_taken_before_sync
+):
+ result = runner.invoke(app, ["fetch-batch", URL, "--save-assets", "--json"])
+ assert result.exit_code == 0, result.output
+ data = json.loads(result.stdout)["data"]
+ assert data["notes_created"] == []
+ assert [(f["url"], f["phase"]) for f in data["failed_urls"]] == [(URL, "index")]
+ assert "research/notes/rival.md" in data["failed_urls"][0]["error"]
+ _assert_nothing_recorded(vault_dir, id_taken_before_sync)
+
+
+def test_save_result_records_nothing_when_the_id_is_taken_before_sync(
+ vault_dir: Path, id_taken_before_sync
+):
+ from hyperresearch.cli.research import _save_result
+
+ vault = Vault.discover(vault_dir)
+ prov = _FakeProvider()
+ assert _save_result(vault, vault.db, prov, prov.fetch(URL), [], None) is None
+ _assert_nothing_recorded(vault_dir, id_taken_before_sync)
+
+
+def test_escalation_ingest_records_nothing_when_the_id_is_taken_before_sync(
+ vault_dir: Path, id_taken_before_sync
+):
+ from hyperresearch.core.escalation import list_items
+
+ item_id, body = _claimed_escalation_item(vault_dir)
+ result = runner.invoke(
+ app,
+ ["escalation", "ingest", str(item_id), "--title", "Topic", "--body-file", str(body), "--json"],
+ )
+ assert result.exit_code == 1, result.output
+ assert json.loads(result.stdout)["error_code"] == "NOTE_NOT_INDEXED"
+ _assert_nothing_recorded(vault_dir, id_taken_before_sync)
+ assert list_items(_db(vault_dir))[0]["status"] == "in_progress" # still claimed
+
+ again = runner.invoke(
+ app,
+ ["escalation", "ingest", str(item_id), "--title", "Topic", "--body-file", str(body), "--json"],
+ )
+ assert again.exit_code == 0, again.output
+ assert _source_note_id(vault_dir, URL) == json.loads(again.stdout)["data"]["note_id"] != "topic"
+
+
+def test_fetch_and_save_records_nothing_when_the_id_is_taken_before_sync(
+ vault_dir: Path, id_taken_before_sync
+):
+ from hyperresearch.core.fetcher import fetch_and_save
+
+ with pytest.raises(RuntimeError, match="was not indexed"):
+ fetch_and_save(Vault.discover(vault_dir), URL, save_assets=True)
+ _assert_nothing_recorded(vault_dir, id_taken_before_sync)
+ assert not (vault_dir / "research" / "raw" / "topic.pdf").exists()
+
+
+# --- the other callers that mint a new note under a derived id ---------------
+
+
+def _sync_errors() -> list:
+ return json.loads(runner.invoke(app, ["sync", "--json"]).stdout)["data"]["errors"]
+
+
+def moved_note(vault_dir: Path) -> Path:
+ """`note new` "Topic", then `note mv` it to topic-moved.md."""
+ assert runner.invoke(app, ["note", "new", "Topic", "--json"]).exit_code == 0
+ moved = runner.invoke(app, ["note", "mv", "topic", "topic-moved", "--json"])
+ assert moved.exit_code == 0, moved.output
+ return vault_dir
+
+
+def _assert_new_note_indexed(vault_dir: Path, note_id: str) -> None:
+ assert note_id != "topic"
+ assert _note_path(vault_dir, note_id) == f"research/notes/{note_id}.md"
+ assert _note_path(vault_dir, "topic") == "research/notes/topic-moved.md"
+ assert _sync_errors() == []
+
+
+def test_note_new_after_note_mv_gets_a_free_id(moved_note: Path):
+ result = runner.invoke(app, ["note", "new", "Topic", "--json"])
+ assert result.exit_code == 0, result.output
+ _assert_new_note_indexed(moved_note, json.loads(result.stdout)["data"]["id"])
+
+
+def test_mcp_create_note_after_note_mv_gets_a_free_id(moved_note: Path):
+ mcp = pytest.importorskip("hyperresearch.mcp.server")
+ mcp._vault = None # module-global cache; force rediscovery under tmp_path
+ try:
+ out = json.loads(mcp.create_note("Topic", "Body of the new note."))
+ finally:
+ mcp._vault = None
+ assert out["ok"] is True
+ _assert_new_note_indexed(moved_note, out["data"]["note_id"])
+
+
+def broken_ref_on_a_taken_slug(vault_dir: Path) -> Path:
+ """[[foo bar]] is broken (the resolver matches ids, aliases and titles,
+ never slugs), but its slug `foo-bar` is another note's id."""
+ notes = vault_dir / "research" / "notes"
+ (notes / "foo-bar.md").write_text(
+ "---\ntitle: Something Else\nid: foo-bar\n---\n\nBody.\n", encoding="utf-8"
+ )
+ (notes / "linker.md").write_text(
+ "---\ntitle: Linker\nid: linker\n---\n\nSee [[foo bar]].\n", encoding="utf-8"
+ )
+ assert _sync_errors() == []
+ assert _link_target(vault_dir, "foo bar") is None
+ return vault_dir
+
+
+def _link_target(vault_dir: Path, ref: str):
+ row = _db(vault_dir).execute(
+ "SELECT target_id FROM links WHERE target_ref = ?", (ref,)
+ ).fetchone()
+ return row["target_id"]
+
+
+def _assert_stub_resolves(vault_dir: Path) -> None:
+ stub_id = _link_target(vault_dir, "foo bar")
+ assert stub_id not in (None, "foo-bar")
+ assert _note_path(vault_dir, stub_id) == f"research/temp/{stub_id}.md"
+ assert _note_path(vault_dir, "foo-bar") == "research/notes/foo-bar.md"
+ assert _sync_errors() == []
+
+
+def test_graph_stub_for_a_broken_ref_whose_slug_is_taken(broken_ref_on_a_taken_slug: Path):
+ result = runner.invoke(app, ["graph", "stub", "--json"])
+ assert result.exit_code == 0, result.output
+ _assert_stub_resolves(broken_ref_on_a_taken_slug)
+
+
+def test_repair_stub_for_a_broken_ref_whose_slug_is_taken(broken_ref_on_a_taken_slug: Path):
+ result = runner.invoke(app, ["repair", "--json"])
+ assert result.exit_code == 0, result.output
+ assert json.loads(result.stdout)["data"]["stubs"] == 1
+ _assert_stub_resolves(broken_ref_on_a_taken_slug)
+
+
+class _SearchProvider(_FakeProvider):
+ """Two fresh results per search, so a second `research` run creates notes
+ (and therefore a map-of-content) again."""
+
+ searches = 0
+
+ def search(self, topic: str, max_results: int = 5):
+ type(self).searches += 1
+ n = self.searches
+ return [
+ WebResult(url=f"https://s{n}.example.org/{i}", title=f"Result {n} {i}",
+ content="enough content to clear the junk gate. " * 10)
+ for i in range(2)
+ ]
+
+
+def test_research_map_of_content_after_note_mv_gets_a_free_id(vault_dir: Path, monkeypatch):
+ monkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _SearchProvider())
+ assert runner.invoke(app, ["research", "topic", "--json"]).exit_code == 0
+ moved = runner.invoke(app, ["note", "mv", "research-topic", "moc-moved", "--json"])
+ assert moved.exit_code == 0, moved.output
+
+ result = runner.invoke(app, ["research", "topic", "--json"])
+ assert result.exit_code == 0, result.output
+ created = json.loads(result.stdout)["data"]["notes_created"]
+ moc = [n for n in created if n.get("type") == "moc"]
+ assert len(moc) == 1
+ moc_id = moc[0]["note_id"]
+ assert moc_id != "research-topic" # the moved map-of-content keeps that id
+ assert _note_path(vault_dir, moc_id) == f"research/notes/{moc_id}.md"
+ assert Path(moc[0]["path"]).as_posix() == f"research/notes/{moc_id}.md" # OS separators
+ assert _note_path(vault_dir, "research-topic") == "research/notes/moc-moved.md"
+ assert _sync_errors() == []
diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.py
index cc16c23..272f98b 100644
--- a/tests/test_core/test_note.py
+++ b/tests/test_core/test_note.py
@@ -141,6 +141,22 @@ def test_write_collision_id_survives_reslugify_bytecap(tmp_vault):
assert len(p2.stem.encode("utf-8")) <= 200
+def test_write_skips_taken_ids_with_no_file(tmp_vault):
+ """`note mv` keeps a note's id while renaming its file, so an id can be
+ taken while `<id>.md` is free. taken_ids skips it through the same
+ fixed-point collision ids as a file collision."""
+ p = write_note(tmp_vault.notes_dir, "Same Title", taken_ids={"same-title", "same-title-2"})
+ assert p.name == "same-title-3.md"
+ assert read_note(p, tmp_vault.root).meta.id == "same-title-3"
+
+ long_title = "x" * 120 # slug hits the 80-char cap
+ p2 = write_note(tmp_vault.notes_dir, long_title, taken_ids={slugify(long_title)})
+ n2 = read_note(p2, tmp_vault.root)
+ assert p2.stem == n2.meta.id != slugify(long_title)
+ assert slugify(n2.meta.id) == n2.meta.id
+ assert len(p2.stem) <= 80
+
+
def test_write_with_parent(tmp_vault):
"""`parent:` is frontmatter metadata (DB-indexed), NOT a filesystem dir."""
path = write_note(