Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: collision note ids survive re-slugify (orphan-note FK crash) (#66)

Francois du Plessiscc45cff66aec
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c15c28f..3783200 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,17 @@
## [Unreleased]
+- **Collision note ids survive the frontmatter re-parse** (an orphan-note
+ foreign-key crash). `write_note()` appended `-2` to a slug already sitting on
+ `slugify()`'s 80-char cap, producing an 82-char id whose next parse
+ re-slugified it back under the cap: the suffix fell off, the second note
+ collapsed into the base id, sync refused the duplicate, and the `sources`
+ insert died with `IntegrityError: FOREIGN KEY constraint failed`, leaving the
+ `.md` file on disk but permanently unregistered. Collision ids are now built
+ by trimming the base so base+suffix fits both the 80-char and 200-byte caps,
+ then normalized once, so the disk id equals its own re-parse; short-title
+ collisions keep their existing `-2` ids, so existing vaults do not shift.
+
### Two safety fixes in the viewer and the installer
- **Stored XSS in `hpr serve` (#72, reported by @letospace).** `_serve_search` escaped every interpolated value except the FTS snippet, and the snippet is note body text — remote page content, for a fetched note. `strip_markdown` was not a defense: its tag regex needs a closing `>`, so an unterminated `<img src=x onerror=...` passed into `body_plain` intact, and the `>` of the `</mark>` the search page injects finished the tag. The snippet is escaped before the markers are substituted now, so the `<mark>` tags are the only markup that survives. Bounded by the server binding `127.0.0.1` and being read-only, but the script ran same-origin with the wiki and could read every note the viewer could reach. Link and image URLs in the renderer also got a scheme allowlist, so a note body can no longer render a `javascript:` or `data:` link.
diff --git a/src/hyperresearch/core/note.py b/src/hyperresearch/core/note.py
index 12fcb81..c790717 100644
--- a/src/hyperresearch/core/note.py
+++ b/src/hyperresearch/core/note.py
@@ -51,6 +51,24 @@ def read_note(file_path: Path, vault_root: Path) -> Note:
)
+def _collision_id(base: str, counter: int) -> str:
+ """Collision suffix that keeps the id a fixed point of slugify().
+
+ Appending "-<n>" to a slug that already sits on a slugify() length cap
+ makes an id the next frontmatter parse re-slugifies back UNDER the cap:
+ the suffix falls off, the note collapses into the base id, sync refuses
+ the duplicate id, and the sources insert dies on its foreign key,
+ leaving an orphan .md file (verbumetecclesia fetches, 2026-07-23).
+ Trim the base so base+suffix fits both the 80-char and 200-byte caps,
+ then slugify once so the disk id equals its own re-parse.
+ """
+ suffix = f"-{counter}"
+ trimmed = base[: 80 - len(suffix)]
+ while len(trimmed.encode("utf-8")) + len(suffix.encode("utf-8")) > 200:
+ trimmed = trimmed[:-1]
+ return slugify(f"{trimmed.rstrip('-')}{suffix}")
+
+
def write_note(
notes_dir: Path,
title: str,
@@ -104,8 +122,9 @@ def write_note(
file_path = target_dir / f"{nid}.md"
counter = 2
while file_path.exists():
- file_path = target_dir / f"{nid}-{counter}.md"
- meta.id = f"{nid}-{counter}"
+ candidate = _collision_id(nid, counter)
+ file_path = target_dir / f"{candidate}.md"
+ meta.id = candidate
counter += 1
content = render_note(meta, body)
diff --git a/tests/test_core/test_note.py b/tests/test_core/test_note.py
index 3a3a65f..b5aa2bb 100644
--- a/tests/test_core/test_note.py
+++ b/tests/test_core/test_note.py
@@ -111,6 +111,36 @@ def test_write_avoids_collision(tmp_vault):
assert p2.exists()
+# --- Collision ids must survive the frontmatter re-parse ---------------------
+# slugify() caps ids at 80 chars / 200 bytes. Appending "-2" to a capped slug
+# used to produce an over-cap id that the next parse chopped back to the base
+# id: sync refused the duplicate, the sources insert failed its foreign key,
+# and the .md file was left orphaned (verbumetecclesia fetches, 2026-07-23).
+
+
+def test_write_collision_id_survives_reslugify(tmp_vault):
+ long_title = "x" * 120 # slug hits the 80-char cap
+ p1 = write_note(tmp_vault.notes_dir, long_title)
+ p2 = write_note(tmp_vault.notes_dir, long_title)
+ n1 = read_note(p1, tmp_vault.root)
+ n2 = read_note(p2, tmp_vault.root)
+ assert p1.stem == n1.meta.id
+ assert p2.stem == n2.meta.id # used to collapse back to the base id
+ assert n1.meta.id != n2.meta.id
+ assert slugify(n2.meta.id) == n2.meta.id
+ assert len(p2.stem) <= 80
+
+
+def test_write_collision_id_survives_reslugify_bytecap(tmp_vault):
+ long_cjk = "液" * 90 # 3 UTF-8 bytes per char: the 200-byte cap bites
+ write_note(tmp_vault.notes_dir, long_cjk)
+ p2 = write_note(tmp_vault.notes_dir, long_cjk)
+ n2 = read_note(p2, tmp_vault.root)
+ assert p2.stem == n2.meta.id
+ assert slugify(n2.meta.id) == n2.meta.id
+ assert len(p2.stem.encode("utf-8")) <= 200
+
+
def test_write_with_parent(tmp_vault):
"""`parent:` is frontmatter metadata (DB-indexed), NOT a filesystem dir."""
path = write_note(
diff --git a/tests/test_core/test_sync.py b/tests/test_core/test_sync.py
index e13fcf7..f09fe59 100644
--- a/tests/test_core/test_sync.py
+++ b/tests/test_core/test_sync.py
@@ -307,3 +307,21 @@ def test_sync_cleans_up_stale_collided_row(tmp_vault):
assert result.errors == []
row = tmp_vault.db.execute("SELECT path FROM notes WHERE id = ?", ("foo",)).fetchone()
assert row["path"] == "research/notes/foo.md"
+
+def test_sync_registers_capped_collision_notes_separately(tmp_vault):
+ """Two notes whose titles collide on the slug length cap must land as two
+ DB rows — the orphan-note bug collapsed the second into the first id and
+ sync then refused it, so the note existed on disk but never in the DB."""
+ from hyperresearch.core.note import write_note
+
+ long_title = "y" * 120
+ p1 = write_note(tmp_vault.notes_dir, long_title, body="Eerste.")
+ p2 = write_note(tmp_vault.notes_dir, long_title, body="Tweede.")
+
+ plan = compute_sync_plan(tmp_vault)
+ result = execute_sync(tmp_vault, plan)
+ assert result.errors == []
+
+ rows = tmp_vault.db.execute("SELECT id FROM notes ORDER BY id").fetchall()
+ ids = {r["id"] for r in rows}
+ assert ids == {p1.stem, p2.stem}