Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(episodes): keep a session after its transcript is pruned, and search episode bodies (#1331)

Raghav Chamadiya873b820c390f
diff --git a/packages/core/src/repowise/core/fts_query.py b/packages/core/src/repowise/core/fts_query.py
new file mode 100644
index 000000000..5b08897b0
--- /dev/null
+++ b/packages/core/src/repowise/core/fts_query.py
@@ -0,0 +1,190 @@
+"""Turning a natural-language question into a SQLite FTS5 MATCH expression.
+
+Lifted out of :mod:`repowise.core.persistence.search` when a second store
+needed the same expression. It sits at the top of ``repowise.core`` rather than
+beside its first caller because the second one — the episode store — is
+stdlib-only by test (``tests/unit/precedent/test_structural_perf.py``) and
+importing ``repowise.core.persistence`` would pull SQLAlchemy onto a hook path
+whose budget was fought down from 965 ms to 155 ms by deleting three
+module-level imports.
+
+The escaping in here is the part worth sharing rather than rewriting:
+:func:`meaningful_terms` reduces a query to ``[a-zA-Z0-9_]+`` tokens, so every
+character that could change the shape of an FTS5 expression is gone before a
+term is built, and the one path that quotes raw text doubles its quotes.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Callable
+
+#: Shortest term that still gets a prefix wildcard. See :func:`match_term`.
+PREFIX_MIN_CHARS = 4
+
+#: A term matching more than this share of the corpus is dropped from the query
+#: expression: it cannot discriminate between documents, and every one of them
+#: drags thousands of candidates in for BM25 to sort out afterwards.
+#: Env-tunable because the right ceiling is a property of a corpus's
+#: vocabulary, and the only honest way to set it is to measure recall on the
+#: corpus in question.
+DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))
+
+#: Never let the ceiling empty a query. A question written entirely out of
+#: common words keeps its rarest terms — a thin match beats no match.
+MIN_KEPT_TERMS = 3
+
+_TOKEN_RE = re.compile(r"[a-zA-Z0-9_]+")
+
+#: Common English stop words to strip from FTS queries.
+STOP_WORDS = frozenset(
+ {
+ "a",
+ "an",
+ "the",
+ "is",
+ "are",
+ "was",
+ "were",
+ "be",
+ "been",
+ "being",
+ "have",
+ "has",
+ "had",
+ "do",
+ "does",
+ "did",
+ "will",
+ "would",
+ "shall",
+ "should",
+ "may",
+ "might",
+ "must",
+ "can",
+ "could",
+ "am",
+ "to",
+ "of",
+ "in",
+ "for",
+ "on",
+ "with",
+ "at",
+ "by",
+ "from",
+ "as",
+ "into",
+ "about",
+ "it",
+ "its",
+ "this",
+ "that",
+ "these",
+ "those",
+ "i",
+ "we",
+ "you",
+ "he",
+ "she",
+ "they",
+ "me",
+ "him",
+ "her",
+ "us",
+ "them",
+ "my",
+ "your",
+ "his",
+ "our",
+ "their",
+ "what",
+ "which",
+ "who",
+ "whom",
+ "how",
+ "when",
+ "where",
+ "why",
+ "not",
+ "no",
+ "so",
+ "if",
+ "or",
+ "and",
+ "but",
+ "all",
+ "each",
+ "very",
+ "just",
+ "also",
+ "than",
+ "too",
+ "only",
+ }
+)
+
+
+def meaningful_terms(query: str) -> list[str]:
+ """Alphanumeric tokens of *query*, minus stop words and single characters."""
+ tokens = _TOKEN_RE.findall(query.lower())
+ return [t for t in tokens if t not in STOP_WORDS and len(t) > 1]
+
+
+def match_term(term: str) -> str:
+ """One FTS5 term, prefix-matched only when it is long enough to be specific.
+
+ A prefix wildcard is what earns morphological recall — ``invalidat*`` finds
+ "invalidates" and "invalidation" — and it is also what turns a short token
+ into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.
+ Below the cut the term is matched exactly, which costs the odd plural and
+ buys back a candidate set that BM25 was being asked to sort out afterwards.
+ """
+ return f'"{term}"*' if len(term) >= PREFIX_MIN_CHARS else f'"{term}"'
+
+
+def build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:
+ """Build an FTS5 MATCH expression from a natural-language query.
+
+ Terms are OR-ed, because a developer question is a paraphrase of the page
+ that answers it: its rarest words are frequently the asker's vocabulary
+ rather than the page's, so requiring all of them (AND) excludes the right
+ page far more often than it narrows usefully. Measured on a 3,678-page
+ corpus, AND over the meaningful terms returned nothing at all for 65 of 99
+ questions, and held the expected page for 25 of 99 against 99 of 99 for OR.
+
+ What OR needs is not a different operator but fewer junk terms. A question
+ carries words that are not stop words yet still match a large share of any
+ code corpus ("file", "does", "page", "index"), and each one drags in
+ thousands of pages that only BM25 tie-breaking then has to sort out. Given
+ a *document_frequency* callable, terms matching more than :data:`DF_CEILING`
+ of the corpus are dropped from the expression; the same measurement puts
+ the median candidate set at 20% of the corpus rather than 65%. Without the
+ callable the expression keeps every term, which is the prior behaviour.
+
+ At least :data:`MIN_KEPT_TERMS` terms always survive — the rarest ones — so
+ a question written entirely from common words still matches something.
+ """
+ meaningful = meaningful_terms(query)
+
+ if not meaningful:
+ # All stop words — fall back to exact phrase. The one place raw text
+ # reaches the expression, so it is the one place quotes are doubled.
+ safe = query.replace('"', '""')
+ return f'"{safe}"'
+
+ kept = meaningful
+ if document_frequency is not None:
+ total = document_frequency("") # corpus size, by convention
+ if total > 0:
+ ceiling = DF_CEILING * total
+ selective = [t for t in meaningful if document_frequency(t) <= ceiling]
+ if len(selective) < MIN_KEPT_TERMS:
+ # Every term is common. Keep the rarest few rather than none:
+ # a thin match beats an empty one.
+ selective = sorted(meaningful, key=document_frequency)[:MIN_KEPT_TERMS]
+ kept = selective
+
+ return " OR ".join(match_term(t) for t in kept)
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index 79f264c04..b7af3f267 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -17,15 +17,20 @@ Usage::
from __future__ import annotations
import logging
-import os
from collections import OrderedDict
-from collections.abc import Callable
from dataclasses import dataclass
from typing import Literal
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.sql import text
+from repowise.core.fts_query import DF_CEILING as _DF_CEILING
+from repowise.core.fts_query import MIN_KEPT_TERMS as _MIN_KEPT_TERMS
+from repowise.core.fts_query import PREFIX_MIN_CHARS as _PREFIX_MIN_CHARS
+from repowise.core.fts_query import build_fts5_query as _build_fts5_query
+from repowise.core.fts_query import match_term as _match_term
+from repowise.core.fts_query import meaningful_terms as _meaningful_terms
+
from .information_floor import information_floor, meets_information_floor, substantive_text
@@ -83,21 +88,6 @@ PG_FTS_EXPRESSION = (
"|| COALESCE(summary,'') || ' ' || COALESCE(target_path,''))"
)
-# Shortest term that still gets a prefix wildcard. See :func:`_match_term`.
-_PREFIX_MIN_CHARS = 4
-
-# A term matching more than this share of the corpus is dropped from the query
-# expression: it cannot discriminate between pages, and every one of them drags
-# thousands of candidates in for BM25 to sort out afterwards. Env-tunable
-# because the right ceiling is a property of a corpus's vocabulary, and the only
-# honest way to set it is to measure recall on the corpus in question.
-_DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))
-
-# Never let the ceiling empty a query. A question written entirely out of common
-# words keeps its rarest terms — a thin match beats no match, especially since
-# vector retrieval is answering the same question alongside this.
-_MIN_KEPT_TERMS = 3
-
# How many distinct terms' document frequencies one FullTextSearch keeps. Term
# frequency changes only when the corpus is rewritten, and questions reuse
# vocabulary heavily, so a small cache removes almost all of the counting.
@@ -109,115 +99,6 @@ _SCORE_EPSILON = 1e-6
_log = logging.getLogger(__name__)
-# Common English stop words to strip from FTS queries
-_STOP_WORDS = frozenset(
- {
- "a",
- "an",
- "the",
- "is",
- "are",
- "was",
- "were",
- "be",
- "been",
- "being",
- "have",
- "has",
- "had",
- "do",
- "does",
- "did",
- "will",
- "would",
- "shall",
- "should",
- "may",
- "might",
- "must",
- "can",
- "could",
- "am",
- "to",
- "of",
- "in",
- "for",
- "on",
- "with",
- "at",
- "by",
- "from",
- "as",
- "into",
- "about",
- "it",
- "its",
- "this",
- "that",
- "these",
- "those",
- "i",
- "we",
- "you",
- "he",
- "she",
- "they",
- "me",
- "him",
- "her",
- "us",
- "them",
- "my",
- "your",
- "his",
- "our",
- "their",
- "what",
- "which",
- "who",
- "whom",
- "how",
- "when",
- "where",
- "why",
- "not",
- "no",
- "so",
- "if",
- "or",
- "and",
- "but",
- "all",
- "each",
- "very",
- "just",
- "also",
- "than",
- "too",
- "only",
- }
-)
-
-
-def _meaningful_terms(query: str) -> list[str]:
- """Alphanumeric tokens of *query*, minus stop words and single characters."""
- import re
-
- tokens = re.findall(r"[a-zA-Z0-9_]+", query.lower())
- return [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]
-
-
-def _match_term(term: str) -> str:
- """One FTS5 term, prefix-matched only when it is long enough to be specific.
-
- A prefix wildcard is what earns morphological recall — ``invalidat*`` finds
- "invalidates" and "invalidation" — and it is also what turns a short token
- into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.
- Below the cut the term is matched exactly, which costs the odd plural and
- buys back a candidate set that BM25 was being asked to sort out afterwards.
- """
- return f'"{term}"*' if len(term) >= _PREFIX_MIN_CHARS else f'"{term}"'
-
def _pg_term(term: str) -> str:
"""One ``to_tsquery`` lexeme, prefix-matched on the same rule as FTS5.
@@ -229,52 +110,6 @@ def _pg_term(term: str) -> str:
return f"{term}:*" if len(term) >= _PREFIX_MIN_CHARS else term
-def _build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:
- """Build an FTS5 MATCH expression from a natural-language query.
-
- Terms are OR-ed, because a developer question is a paraphrase of the page
- that answers it: its rarest words are frequently the asker's vocabulary
- rather than the page's, so requiring all of them (AND) excludes the right
- page far more often than it narrows usefully. Measured on a 3,678-page
- corpus, AND over the meaningful terms returned nothing at all for 65 of 99
- questions, and held the expected page for 25 of 99 against 99 of 99 for OR.
-
- What OR needs is not a different operator but fewer junk terms. A question
- carries words that are not stop words yet still match a large share of any
- code corpus ("file", "does", "page", "index"), and each one drags in
- thousands of pages that only BM25 tie-breaking then has to sort out. Given
- a *document_frequency* callable, terms matching more than
- :data:`_DF_CEILING` of the corpus are dropped from the expression; the same
- measurement puts the median candidate set at 20% of the corpus rather than
- 65%. Without the callable the expression keeps every term, which is the
- prior behaviour.
-
- At least :data:`_MIN_KEPT_TERMS` terms always survive — the rarest ones —
- so a question written entirely from common words still matches something.
- """
- meaningful = _meaningful_terms(query)
-
- if not meaningful:
- # All stop words — fall back to exact phrase
- safe = query.replace('"', '""')
- return f'"{safe}"'
-
- kept = meaningful
- if document_frequency is not None:
- total = document_frequency("") # corpus size, by convention
- if total > 0:
- ceiling = _DF_CEILING * total
- selective = [t for t in meaningful if document_frequency(t) <= ceiling]
- if len(selective) < _MIN_KEPT_TERMS:
- # Every term is common. Keep the rarest few rather than none:
- # a thin match beats an empty one, and the vector arm is still
- # answering alongside this.
- selective = sorted(meaningful, key=document_frequency)[:_MIN_KEPT_TERMS]
- kept = selective
-
- return " OR ".join(_match_term(t) for t in kept)
-
-
def snippet_around(content: str, query: str, length: int = _SNIPPET_LEN) -> str:
"""A window of *content* centred on the first term of *query* it carries.
diff --git a/packages/core/src/repowise/core/precedent/store.py b/packages/core/src/repowise/core/precedent/store.py
index 6e874ba20..6328a5c7a 100644
--- a/packages/core/src/repowise/core/precedent/store.py
+++ b/packages/core/src/repowise/core/precedent/store.py
@@ -20,12 +20,17 @@ from __future__ import annotations
import hashlib
import json
+import logging
import sqlite3
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from pathlib import Path
+from repowise.core.fts_query import build_fts5_query, meaningful_terms
+
+_log = logging.getLogger(__name__)
+
EPISODES_DIRNAME = "episodes"
EPISODES_DB_FILENAME = "episodes.db"
@@ -53,6 +58,17 @@ DEFAULT_MAX_ROWS = 5000
#: chunk rather than one variable past whatever SQLite was compiled with.
_IN_CHUNK = 500
+#: How much wider than the caller's limit the BM25 window is drawn, so the
+#: rerank has something to reorder, and the hard ceiling on it, because every
+#: row in the window is a body this store then scans.
+_RERANK_FACTOR = 5
+_RERANK_WINDOW_MAX = 50
+
+#: Appended to an episode's evidence the first time a run finds the source it
+#: was derived from gone. The episode itself stays: the body is the content and
+#: the pointer was only ever provenance.
+SOURCE_GONE_NOTE = " (source no longer on disk)"
+
_SCHEMA = """
CREATE TABLE IF NOT EXISTS episodes (
id TEXT PRIMARY KEY,
@@ -70,6 +86,31 @@ CREATE INDEX IF NOT EXISTS idx_episodes_tier_kind ON episodes(tier, kind);
CREATE INDEX IF NOT EXISTS idx_episodes_last_seen ON episodes(last_seen_at);
"""
+#: The searchable projection of an episode. External-content rather than a
+#: table of its own: ``body`` is already a column and this corpus is 11 MB on
+#: a real machine, so a standalone index would keep a second copy of all of it
+#: for nothing. The columns are named after the ones they shadow, which is what
+#: lets SQLite read them back out of ``episodes`` on a hit.
+_FTS_SCHEMA = """
+CREATE VIRTUAL TABLE IF NOT EXISTS episode_fts USING fts5(
+ subject, body, content='episodes', content_rowid='rowid'
+);
+CREATE TRIGGER IF NOT EXISTS episodes_fts_ai AFTER INSERT ON episodes BEGIN
+ INSERT INTO episode_fts(rowid, subject, body)
+ VALUES (new.rowid, new.subject, new.body);
+END;
+CREATE TRIGGER IF NOT EXISTS episodes_fts_ad AFTER DELETE ON episodes BEGIN
+ INSERT INTO episode_fts(episode_fts, rowid, subject, body)
+ VALUES ('delete', old.rowid, old.subject, old.body);
+END;
+CREATE TRIGGER IF NOT EXISTS episodes_fts_au AFTER UPDATE ON episodes BEGIN
+ INSERT INTO episode_fts(episode_fts, rowid, subject, body)
+ VALUES ('delete', old.rowid, old.subject, old.body);
+ INSERT INTO episode_fts(rowid, subject, body)
+ VALUES (new.rowid, new.subject, new.body);
+END;
+"""
+
def default_store_path(repo_path: Path | str) -> Path:
"""Path to *repo_path*'s episode database (not created here)."""
@@ -137,6 +178,47 @@ class EpisodeStore:
self._conn.execute("PRAGMA busy_timeout=5000")
self._conn.executescript(_SCHEMA)
self._conn.commit()
+ self.fts_enabled = self._ensure_fts()
+
+ def _ensure_fts(self) -> bool:
+ """Create the search index and refill it if it is behind. Never raises.
+
+ Modelled on ``persistence/search.py:_upgrade_sqlite_schema``, whose one
+ binding rule is that it must not raise: every command opens the store
+ through it, so an exception over an index would take down a product
+ that is still perfectly usable without one. Here the stake is smaller
+ and the rule is the same — a store that cannot build an index still
+ answers every read the guard and the hooks make of it.
+
+ Refill is one statement rather than a re-derivation. The rows are the
+ system of record and ``body`` is a column of them, so FTS5's own
+ ``rebuild`` reads what a hand-written refill would have selected.
+
+ Drift is detected by which objects are missing, not by comparing row
+ counts: an external-content table reads through to ``episodes`` on a
+ scan, so ``count(*)`` answers with the content table's size and reports
+ an index holding nothing as full. Presence is the honest signal, and it
+ is enough — once the triggers exist no drift can accumulate behind them.
+ """
+ try:
+ existing = {
+ row[0]
+ for row in self._conn.execute(
+ "SELECT name FROM sqlite_master WHERE name IN "
+ "('episode_fts', 'episodes_fts_ai', 'episodes_fts_ad', 'episodes_fts_au')"
+ )
+ }
+ self._conn.executescript(_FTS_SCHEMA)
+ if len(existing) < 4:
+ # A store written before this index existed, or by a build
+ # without FTS5, or one whose objects somebody dropped.
+ self._conn.execute("INSERT INTO episode_fts(episode_fts) VALUES ('rebuild')")
+ self._conn.commit()
+ return True
+ except sqlite3.Error:
+ self._conn.rollback()
+ _log.debug("episode search index unavailable", exc_info=True)
+ return False
@classmethod
def open_for_repo(cls, repo_path: Path | str) -> EpisodeStore:
@@ -234,7 +316,7 @@ class EpisodeStore:
self.prune(now=stamp)
return dropped
- def sync_tier(
+ def accumulate_tier(
self,
*,
tier: str,
@@ -243,50 +325,94 @@ class EpisodeStore:
present_subjects: Sequence[str],
now: float | None = None,
) -> int:
- """Accumulate into *tier*, bounded by what still exists.
+ """Accumulate into *tier*, outliving the sources the episodes came from.
The third lifecycle, and the reason it is not one of the other two:
* :meth:`replace_kinds` says *this run derived the whole kind*, which
- is false here — a run reads only the transcript bytes appended since
- the last one, so a sweep would delete every session it did not
- revisit;
+ is false here — a run reads only the bytes appended since the last
+ one, so a sweep would delete every source it did not revisit;
* :meth:`append_tier` says *this run covered a window and everything in
- it is re-observed*, which is also false — the pass has no window, and
- nothing about reading today's session says last March's still exists.
-
- What is true, and true of no other tier, is that this one's membership
- can be **enumerated without being read**: the sources are files on
- disk, so a run knows which episodes still have something behind them
- for the cost of the directory listing it already did. That is the
- vouch. *present_subjects* are marked re-observed whether or not they
- were read this run — which is what stops a live episode dying of the
- TTL simply because its session ended — and the rest of *kind* is
- dropped, because its source is gone rather than merely quiet.
-
- Returns rows dropped for having no source left.
+ it is re-observed*, which is also false — the pass has no window.
+
+ What replaces both is that an episode is **self-contained**: the body
+ holds the prose, so the source pointer is provenance and not content.
+ The source is therefore free to disappear, and it will. The harness
+ this tier reads from prunes its transcripts at 30 days by default,
+ counted here at 1,509 files whose age distribution stops dead at 30 —
+ so an earlier version of this method, which read "the file is gone" as
+ "the episode is gone", capped the tier at the harness's retention and
+ could never support the one claim the tier exists to make. Being the
+ only durable copy is the strongest thing the store has to say.
+
+ What bounds the tier instead is what bounds the other accumulating one:
+ the TTL and the per-tier row cap. Every run marks the whole kind
+ re-observed, because a run of the index is what vouches that this
+ repository is still live; the TTL then evicts a tier whose repository
+ has stopped being indexed altogether, which is what it is for.
+
+ *present_subjects* survives the change with a smaller job. It is still
+ the cheap enumeration — the sources are files on disk, so a listing
+ answers it without a read — but a missing source is now recorded as a
+ **note on the row** rather than acted on, so a reader can say the
+ quotation outlived what it was quoted from instead of following a
+ pointer into nothing. An empty *present_subjects* is no vouch at all
+ (a run on a machine that never had the sources looks identical to one
+ whose sources all vanished), so it annotates nothing.
+
+ Returns rows newly marked as having lost their source.
"""
stamp = time.time() if now is None else now
- ids = [episode_id(tier, kind, s) for s in present_subjects]
- with self._conn: # one transaction: never half-synced
+ with self._conn: # one transaction: never half-written
self._upsert(episodes, stamp)
- # Chunked: a heavy machine's transcript directory can outrun
- # SQLite's per-statement variable limit, and a partial IN list
- # would read as "these are gone" and delete live episodes.
- for start in range(0, len(ids), _IN_CHUNK):
- batch = ids[start : start + _IN_CHUNK]
- placeholders = ",".join("?" * len(batch))
- self._conn.execute(
- f"UPDATE episodes SET last_seen_at = ? WHERE id IN ({placeholders})",
- (stamp, *batch),
- )
- cur = self._conn.execute(
- "DELETE FROM episodes WHERE tier = ? AND kind = ? AND last_seen_at < ?",
- (tier, kind, stamp),
+ self._conn.execute(
+ "UPDATE episodes SET last_seen_at = ? WHERE tier = ? AND kind = ?",
+ (stamp, tier, kind),
)
- dropped = cur.rowcount or 0
+ noted = self._note_missing_sources(tier, kind, present_subjects)
self.prune(now=stamp)
- return dropped
+ return noted
+
+ def _note_missing_sources(
+ self, tier: str, kind: str, present_subjects: Sequence[str]
+ ) -> int:
+ """Make the note match what is on disk, in both directions. In-transaction.
+
+ A temporary table rather than a ``NOT IN`` list, because this is a
+ negation and a negation cannot be chunked: a partial list would read as
+ "everything else is gone" and annotate live rows. The same reasoning
+ that made the positive form chunk-safe makes this form all-or-nothing.
+
+ The note is **cleared as well as written**, and the asymmetry it fixes
+ is not hypothetical. Clearing it only on re-derivation would leave the
+ note stuck forever on the ordinary case: a source has to be *read* to
+ be re-derived, an old session has nothing new to read, and one
+ directory listing that misses a file — a transient failure, no deletion
+ needed — would then mark it gone permanently while the file sat there.
+ Presence is a fact about now, so the row is made to agree with now.
+ """
+ if not present_subjects:
+ return 0
+ ids = [(episode_id(tier, kind, s),) for s in present_subjects]
+ self._conn.execute("CREATE TEMP TABLE IF NOT EXISTS present_ids (id TEXT PRIMARY KEY)")
+ self._conn.execute("DELETE FROM present_ids")
+ self._conn.executemany("INSERT OR IGNORE INTO present_ids (id) VALUES (?)", ids)
+ cur = self._conn.execute(
+ # ``instr`` rather than LIKE: the marker is a literal, and a LIKE
+ # pattern is a thing to escape even when it is a constant today.
+ "UPDATE episodes SET evidence = evidence || ? "
+ "WHERE tier = ? AND kind = ? AND instr(evidence, ?) = 0 "
+ "AND id NOT IN (SELECT id FROM present_ids)",
+ (SOURCE_GONE_NOTE, tier, kind, SOURCE_GONE_NOTE),
+ )
+ noted = cur.rowcount or 0
+ self._conn.execute(
+ "UPDATE episodes SET evidence = replace(evidence, ?, '') "
+ "WHERE tier = ? AND kind = ? AND instr(evidence, ?) > 0 "
+ "AND id IN (SELECT id FROM present_ids)",
+ (SOURCE_GONE_NOTE, tier, kind, SOURCE_GONE_NOTE),
+ )
+ return noted
def _upsert(self, episodes: Iterable[Episode], stamp: float) -> None:
"""Insert or refresh *episodes*. Caller owns the transaction.
@@ -438,6 +564,96 @@ class EpisodeStore:
sql += " ORDER BY birth_at DESC, id ASC"
return [_row_to_dict(row) for row in self._conn.execute(sql, params)]
+ def search(
+ self,
+ query: str,
+ *,
+ tiers: Sequence[str] | None = None,
+ limit: int = 10,
+ ) -> list[dict]:
+ """Episodes matching *query*, best first, each with its BM25 ``score``.
+
+ Ranked rather than filtered, which is the difference between this and
+ :meth:`list_episodes`: a question is a paraphrase of the episode that
+ answers it, so the expression is built by the same shared builder the
+ wiki index uses (:mod:`repowise.core.fts_query`) — it is where the
+ reduction to ``[a-zA-Z0-9_]+`` lives, and reimplementing that escaping
+ is how a store learns about FTS5's grammar the expensive way.
+
+ BM25 orders the candidates and a second pass reorders them by how many
+ **distinct** question terms the body carries, BM25 breaking the ties.
+ Measured on this repository's own store against a grep-established
+ answer key: 74% to 84% hit@3, 58% to 68% hit@1. The reason is the shape
+ of the document — a session is long and repeats itself, so a body
+ mentioning one term of the question forty times outscores one that
+ answers all four, and term frequency is exactly what BM25 rewards.
+ Counting distinct terms fits no constant to this corpus; the signal is
+ the question's own vocabulary.
+
+ The opposite lever was tried first and is recorded because it looked
+ obvious: passing the wiki index's document-frequency ceiling made this
+ *worse* (74% to 68%), since a term in a fifth of the sessions is
+ ordinary here rather than uninformative.
+
+ Coverage counts a term as present by substring, so ``cat`` is found
+ inside ``concatenate``. That is loose, and it is what the numbers above
+ were measured with: tightening it to word boundaries is a change to the
+ thing being measured, not a free correctness fix, so it belongs with a
+ re-run rather than with a tidy-up.
+
+ Returns nothing rather than raising when there is no index, and the
+ same for a query FTS5 will not parse: a search that cannot run is a
+ feature that says nothing, never a caller that has to handle it.
+ """
+ if not self.fts_enabled or not query.strip():
+ return []
+ if tiers is not None and not tiers:
+ return []
+ # Deliberately without the document-frequency ceiling the wiki index
+ # passes here. Measured on this store: applying it moved r1's
+ # grep-grounded regression from 74% to 68% hit@3, and pushed the
+ # episode holding every rare term of a question from rank 9 to 11.
+ # A session body is long and mentions many things, so a term in a fifth
+ # of the sessions is ordinary rather than uninformative, and matching
+ # more of a question's vocabulary discriminates better than matching
+ # only its rarest words. The ceiling is a property of the wiki corpus's
+ # vocabulary, not a general truth, and it does not transfer.
+ expression = build_fts5_query(query)
+ sql = (
+ "SELECT e.id, e.tier, e.kind, e.subject, e.body, e.evidence, e.nodes, "
+ "e.birth_commit, e.birth_at, e.last_seen_at, bm25(episode_fts) AS score "
+ "FROM episode_fts JOIN episodes e ON e.rowid = episode_fts.rowid "
+ "WHERE episode_fts MATCH ?"
+ )
+ params: list[object] = [expression]
+ if tiers is not None:
+ sql += f" AND e.tier IN ({','.join('?' * len(tiers))})"
+ params.extend(tiers)
+ # BM25 is negative and more negative is better, so plain ascending
+ # order is best-first; the sign is flipped on the way out.
+ sql += " ORDER BY score LIMIT ?"
+ # Reranking can only reorder what BM25 hands it, so the window is wider
+ # than the caller asked for — and bounded, because every row in it is a
+ # body this method then scans.
+ window = max(1, limit) * _RERANK_FACTOR
+ params.append(min(window, _RERANK_WINDOW_MAX))
+ try:
+ rows = self._conn.execute(sql, params).fetchall()
+ except sqlite3.Error:
+ _log.debug("episode search failed for %r", query, exc_info=True)
+ return []
+
+ hits = [{**_row_to_dict(row), "score": -(row[10] or 0.0)} for row in rows]
+ terms = set(meaningful_terms(query))
+ if terms:
+ hits.sort(
+ key=lambda h: (
+ -sum(1 for t in terms if t in h["body"].casefold()),
+ -h["score"],
+ )
+ )
+ return hits[: max(1, limit)]
+
def count(self) -> int:
(rows,) = self._conn.execute("SELECT COUNT(*) FROM episodes").fetchone()
return int(rows)
diff --git a/packages/core/src/repowise/core/precedent/transcript_episodes.py b/packages/core/src/repowise/core/precedent/transcript_episodes.py
index a2de8beab..0e94a1fb2 100644
--- a/packages/core/src/repowise/core/precedent/transcript_episodes.py
+++ b/packages/core/src/repowise/core/precedent/transcript_episodes.py
@@ -123,7 +123,7 @@ class TranscriptEpisodeRecorder:
Presence is registered when a transcript is *shown* to the recorder, not
when it yields events: a session already fully consumed by the cursor
yields nothing and must still count as present, or the next write would
- read its silence as deletion.
+ read its silence as a source that had gone away.
"""
def __init__(self, repo_root: Path) -> None:
@@ -134,11 +134,11 @@ class TranscriptEpisodeRecorder:
def note_present(self, paths: Iterable[Path]) -> None:
"""Register transcripts as existing without reading them.
- Presence and reading are different claims, and conflating them deletes
- data: a run may stop reading early on a large first sweep, but every
- transcript it *listed* still has a session behind it. Called with the
- whole discovery result, so a partial sweep never reads its own
- unfinished work as sessions that have gone away.
+ Presence and reading are different claims: a run may stop reading early
+ on a large first sweep, but every transcript it *listed* still has a
+ session behind it. Called with the whole discovery result, so a partial
+ sweep never records its own unfinished work as sources that have gone
+ away.
"""
for path in paths:
self._present.append(self._subject(path))
@@ -354,13 +354,13 @@ def record_transcript_episodes(repo_path: Path | str, recorder: TranscriptEpisod
pending = recorder.pending()
if not subjects and not pending:
# Nothing was even discovered: this run cannot vouch for the tier's
- # membership, so it must not sweep it. Same rule as a git run that
+ # membership, so it must not annotate it. Same rule as a git run that
# walked no window.
return 0
with EpisodeStore.open_for_repo(root) as store:
prior = _prior_rows(store, [f.subject for f in pending])
episodes = derive_transcript_episodes(recorder, root, prior)
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind=KIND_SESSION,
episodes=episodes,
diff --git a/packages/core/src/repowise/core/sessions/miners/decisions.py b/packages/core/src/repowise/core/sessions/miners/decisions.py
index 88e534e61..f217dec05 100644
--- a/packages/core/src/repowise/core/sessions/miners/decisions.py
+++ b/packages/core/src/repowise/core/sessions/miners/decisions.py
@@ -706,7 +706,8 @@ async def mine_session_decisions(
deferred = 0
discovered = adapter.discover(repo_root, projects_root=projects_root)
# Every discovered transcript is present whether or not this run gets
- # to read it; the episode writer treats absence as deletion.
+ # to read it; the episode writer notes absence on the row it can no
+ # longer point at, and keeps the episode.
recorder.note_present(discovered)
for index, path in enumerate(discovered):
if time.monotonic() > deadline:
diff --git a/tests/unit/precedent/test_episode_store.py b/tests/unit/precedent/test_episode_store.py
index 1be70f33c..210c9ee7a 100644
--- a/tests/unit/precedent/test_episode_store.py
+++ b/tests/unit/precedent/test_episode_store.py
@@ -8,6 +8,7 @@ import pytest
from repowise.core.precedent.store import (
SHAREABLE_TIERS,
+ SOURCE_GONE_NOTE,
TIER_GIT,
TIER_STRUCTURAL,
TIER_TRANSCRIPT,
@@ -220,11 +221,13 @@ class TestPaths:
assert not path.exists() # resolving must not create anything
-class TestSyncTier:
- """The third lifecycle: accumulate, bounded by what still exists.
+class TestAccumulateTier:
+ """The third lifecycle: accumulate, and outlive the sources.
- Its whole reason to exist is that this tier's membership can be enumerated
- without being read, which is true of no other tier.
+ The tier's sources are pruned by a harness on a schedule nobody sets, so an
+ episode that dies with its source can only ever be as old as that schedule.
+ The body is the content and the pointer is provenance, so the episode
+ stays and the missing source becomes a note.
"""
def _session(self, subject: str, body: str = "b", birth_at: float = 10.0) -> Episode:
@@ -241,7 +244,7 @@ class TestSyncTier:
def test_a_present_but_unread_member_is_kept_and_re_observed(self, tmp_path: Path) -> None:
"""The case append_tier cannot express and replace_kinds would delete."""
with _store(tmp_path) as store:
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[self._session("one"), self._session("two")],
@@ -249,7 +252,7 @@ class TestSyncTier:
now=1000.0,
)
# A later run reads nothing new but both sources are still there.
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[],
@@ -262,30 +265,124 @@ class TestSyncTier:
# Re-observation must not reset the birth.
assert {r["birth_at"] for r in rows} == {10.0}
- def test_a_member_whose_source_is_gone_is_dropped(self, tmp_path: Path) -> None:
+ def test_a_member_whose_source_is_gone_is_kept_and_noted(self, tmp_path: Path) -> None:
+ """The retention fix, stated as a test: the episode outlives its source."""
with _store(tmp_path) as store:
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[self._session("one"), self._session("two")],
present_subjects=["one", "two"],
now=1000.0,
)
- store.sync_tier(
+ noted = store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[],
present_subjects=["one"],
now=2000.0,
)
- assert [r["subject"] for r in store.list_episodes(tier=TIER_TRANSCRIPT)] == ["one"]
+ assert noted == 1
+ rows = {r["subject"]: r for r in store.list_episodes(tier=TIER_TRANSCRIPT)}
+ assert set(rows) == {"one", "two"}
+ assert rows["two"]["evidence"].endswith(SOURCE_GONE_NOTE)
+ assert SOURCE_GONE_NOTE not in rows["one"]["evidence"]
+ # The body — the thing the episode is actually made of — is intact.
+ assert rows["two"]["body"] == "b"
+
+ def test_the_note_is_written_once_however_often_it_is_missed(self, tmp_path: Path) -> None:
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[self._session("one"), self._session("two")],
+ present_subjects=["one", "two"],
+ now=1000.0,
+ )
+ noted = [
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[],
+ present_subjects=["one"],
+ now=stamp,
+ )
+ for stamp in (2000.0, 3000.0, 4000.0)
+ ]
+ # Newly marked once, then nothing left to mark: the count is the
+ # contract, and a note appended every run would still read as one
+ # note if only the string were checked.
+ assert noted == [1, 0, 0]
+ (two,) = [
+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"
+ ]
+ assert two["evidence"].count(SOURCE_GONE_NOTE) == 1
+
+ def test_a_source_that_comes_back_loses_the_note(self, tmp_path: Path) -> None:
+ """A re-derivation rewrites the evidence, so the note is not sticky."""
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[self._session("one"), self._session("two")],
+ present_subjects=["one", "two"],
+ now=1000.0,
+ )
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[],
+ present_subjects=["one"],
+ now=2000.0,
+ )
+ # Assert the transition, not just the destination: without this the
+ # test passes on code that never writes a note at all.
+ (gone,) = [
+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"
+ ]
+ assert SOURCE_GONE_NOTE in gone["evidence"]
+
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[self._session("two")],
+ present_subjects=["one", "two"],
+ now=3000.0,
+ )
+ (two,) = [
+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"
+ ]
+ assert SOURCE_GONE_NOTE not in two["evidence"]
+
+ def test_a_run_that_enumerated_nothing_annotates_nothing(self, tmp_path: Path) -> None:
+ """No listing is "cannot tell", not "every source has gone"."""
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[self._session("one")],
+ present_subjects=["one"],
+ now=1000.0,
+ )
+ assert (
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[],
+ present_subjects=[],
+ now=2000.0,
+ )
+ == 0
+ )
+ (row,) = store.list_episodes(tier=TIER_TRANSCRIPT)
+ assert SOURCE_GONE_NOTE not in row["evidence"]
def test_it_leaves_other_tiers_and_kinds_alone(self, tmp_path: Path) -> None:
with _store(tmp_path) as store:
store.replace_kinds(
tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()], now=500.0
)
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[self._session("one")],
@@ -295,20 +392,27 @@ class TestSyncTier:
assert len(store.list_episodes(tier=TIER_STRUCTURAL)) == 1
assert len(store.list_episodes(tier=TIER_TRANSCRIPT)) == 1
- def test_membership_larger_than_one_in_list_survives_chunking(self, tmp_path: Path) -> None:
- """A partial IN list would read as "these are gone" and delete live rows."""
+ def test_a_membership_past_the_variable_limit_annotates_nothing(
+ self, tmp_path: Path
+ ) -> None:
+ """The negation is all-or-nothing: a chunk of it would annotate live rows.
+
+ SQLite's per-statement variable limit is what a heavy machine's
+ transcript directory outruns, so the membership goes through a temp
+ table instead of an ``IN`` list.
+ """
from repowise.core.precedent.store import _IN_CHUNK
subjects = [f"s{i}" for i in range(_IN_CHUNK + 25)]
with _store(tmp_path) as store:
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[self._session(s) for s in subjects],
present_subjects=subjects,
now=1000.0,
)
- store.sync_tier(
+ noted = store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[],
@@ -316,8 +420,10 @@ class TestSyncTier:
now=2000.0,
)
rows = store.list_episodes(tier=TIER_TRANSCRIPT)
+ assert noted == 0
assert len(rows) == len(subjects)
assert {r["last_seen_at"] for r in rows} == {2000.0}
+ assert not [r for r in rows if SOURCE_GONE_NOTE in r["evidence"]]
def test_the_transcript_tier_is_capped_without_touching_the_others(
self, tmp_path: Path
@@ -334,7 +440,7 @@ class TestSyncTier:
tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()], now=500.0
)
subjects = [f"s{i}" for i in range(10)]
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[
@@ -357,7 +463,7 @@ class TestListFilters:
store.replace_kinds(
tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()]
)
- store.sync_tier(
+ store.accumulate_tier(
tier=TIER_TRANSCRIPT,
kind="session",
episodes=[
@@ -395,3 +501,284 @@ class TestListFilters:
with _store(tmp_path) as store, pytest.raises(ValueError, match="at most"):
store.list_episodes(subjects=[f"s{i}" for i in range(_IN_CHUNK + 1)])
+
+
+class TestSearch:
+ """Ranked retrieval over the bodies.
+
+ The tier this exists for holds one row per session with the session's prose
+ in it, so the only way to ask it a question is to rank; a filter over node
+ sets answers "what happened to this file", never "where did we discuss
+ this".
+ """
+
+ def _episodes(self) -> list[Episode]:
+ return [
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject="a.jsonl",
+ body="the traverser skips nested git repositories while walking",
+ evidence="e",
+ ),
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject="b.jsonl",
+ body="the embedder batches pages before calling the provider",
+ evidence="e",
+ ),
+ Episode(
+ tier=TIER_GIT,
+ kind="code_fix",
+ subject="fix the nested checkout walk",
+ body="a nested checkout was being descended into",
+ evidence="e",
+ ),
+ ]
+
+ def _fill(self, store: EpisodeStore) -> None:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[e for e in self._episodes() if e.tier == TIER_TRANSCRIPT],
+ present_subjects=["a.jsonl", "b.jsonl"],
+ )
+ store.append_tier(
+ tier=TIER_GIT,
+ episodes=[e for e in self._episodes() if e.tier == TIER_GIT],
+ oldest_birth_at=0.0,
+ )
+
+ def test_a_question_finds_the_episode_that_answers_it(self, tmp_path: Path) -> None:
+ with _store(tmp_path) as store:
+ self._fill(store)
+ hits = store.search("how are nested repositories handled when walking the tree")
+ assert hits
+ assert hits[0]["subject"] == "a.jsonl"
+ assert hits[0]["score"] > 0
+
+ def test_tiers_filter_the_ranking_rather_than_the_query(self, tmp_path: Path) -> None:
+ with _store(tmp_path) as store:
+ self._fill(store)
+ hits = store.search("nested repositories", tiers=[TIER_GIT])
+ assert {h["tier"] for h in hits} == {TIER_GIT}
+ assert store.search("nested repositories", tiers=[]) == []
+
+ def test_a_deleted_episode_leaves_the_index(self, tmp_path: Path) -> None:
+ """The triggers are what make this true without a second write path."""
+ with _store(tmp_path) as store:
+ self._fill(store)
+ assert store.search("embedder batches")
+ store.replace_kinds(tier=TIER_TRANSCRIPT, kinds=["session"], episodes=[])
+ assert store.search("embedder batches") == []
+
+ def test_an_edited_body_is_searchable_at_its_new_text(self, tmp_path: Path) -> None:
+ """An upsert is an UPDATE, and a stale index would answer with old prose."""
+ with _store(tmp_path) as store:
+ self._fill(store)
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject="a.jsonl",
+ body="the traverser now consults the submodule manifest",
+ evidence="e",
+ )
+ ],
+ present_subjects=["a.jsonl", "b.jsonl"],
+ )
+ assert [h["subject"] for h in store.search("submodule manifest")] == ["a.jsonl"]
+ assert not [h for h in store.search("nested git repositories") if h["subject"] == "a.jsonl"]
+
+ def test_a_query_of_pure_punctuation_says_nothing_rather_than_raising(
+ self, tmp_path: Path
+ ) -> None:
+ """FTS5's grammar is a thing to escape; the shared builder is where that lives."""
+ with _store(tmp_path) as store:
+ self._fill(store)
+ for hostile in ('" OR "', "*", "()", " ", "NEAR(a b", 'walker"'):
+ assert isinstance(store.search(hostile), list)
+
+ def test_rows_written_before_the_index_existed_are_backfilled(
+ self, tmp_path: Path
+ ) -> None:
+ """The upgrade path: an installed store predates its own search index."""
+ import sqlite3
+
+ with _store(tmp_path) as store:
+ self._fill(store)
+ path = store.db_path
+
+ # Drop the index and its triggers the way a build without FTS5 leaves it.
+ conn = sqlite3.connect(path)
+ conn.executescript(
+ "DROP TABLE episode_fts;"
+ "DROP TRIGGER episodes_fts_ai;"
+ "DROP TRIGGER episodes_fts_ad;"
+ "DROP TRIGGER episodes_fts_au;"
+ )
+ conn.commit()
+ conn.close()
+
+ with EpisodeStore(path) as reopened:
+ assert reopened.fts_enabled
+ assert [h["subject"] for h in reopened.search("embedder batches")] == ["b.jsonl"]
+
+ def test_no_index_is_silence_rather_than_an_error(self, tmp_path: Path) -> None:
+ """A store that cannot build an index still answers every other read."""
+ with _store(tmp_path) as store:
+ self._fill(store)
+ store.fts_enabled = False
+ assert store.search("nested repositories") == []
+ assert len(store.list_episodes()) == 3
+
+ def test_the_session_answering_more_of_the_question_outranks_the_louder_one(
+ self, tmp_path: Path
+ ) -> None:
+ """BM25 rewards repetition; a session repeats itself constantly.
+
+ The measured failure this pins: a body saying one term of the question
+ over and over outscored the body that carried every term of it, and the
+ answer sat at rank 9. Distinct-term coverage is the tie the ranking was
+ missing, and it fits no constant to any corpus.
+ """
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject="loud.jsonl",
+ # One term of the question, forty times over.
+ body=" ".join(["staleness"] * 40),
+ evidence="e",
+ ),
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject="answer.jsonl",
+ body=(
+ "the staleness divisor needed fifteen commits in ninety "
+ "days before a record ever moved, so it was inert"
+ ),
+ evidence="e",
+ ),
+ ],
+ present_subjects=["loud.jsonl", "answer.jsonl"],
+ )
+ hits = store.search("why did staleness need fifteen commits in ninety days")
+ assert hits[0]["subject"] == "answer.jsonl"
+
+ def test_the_rerank_window_is_bounded(self, tmp_path: Path) -> None:
+ """Every row in the window is a body this store scans, so it has a ceiling.
+
+ Filled past the ceiling on purpose: with a handful of rows the corpus
+ bounds the result and the assertion passes whether the cap exists or
+ not, which is a test of the fixture rather than of the code.
+ """
+ from repowise.core.precedent.store import _RERANK_WINDOW_MAX
+
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[
+ Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject=f"s{i}.jsonl",
+ body="the traverser skips nested git repositories",
+ evidence="e",
+ )
+ for i in range(_RERANK_WINDOW_MAX * 2)
+ ],
+ present_subjects=[f"s{i}.jsonl" for i in range(_RERANK_WINDOW_MAX * 2)],
+ )
+ assert store.count() > _RERANK_WINDOW_MAX
+ assert len(store.search("nested repositories", limit=1000)) == _RERANK_WINDOW_MAX
+
+ def test_a_noted_source_does_not_change_what_the_cap_evicts(self, tmp_path: Path) -> None:
+ """Presence must not become an eviction signal, and the reason is the point.
+
+ Every transcript is pruned by the harness eventually, so after a month
+ *every* episode's source is gone. Ranking eviction by presence would
+ therefore prefer whatever was recorded in the last thirty days — which
+ is precisely the ceiling this tier was repaired to escape. Age decides,
+ as it does for the other accumulating tier.
+ """
+ def session(subject: str, birth_at: float) -> Episode:
+ return Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject=subject,
+ body="b",
+ evidence="e",
+ birth_at=birth_at,
+ )
+
+ (tmp_path / ".repowise").mkdir(exist_ok=True)
+ with EpisodeStore(default_store_path(tmp_path), max_rows=2) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[
+ session("old-but-live", 1.0),
+ session("newer-but-gone", 2.0),
+ session("newest", 3.0),
+ ],
+ present_subjects=["old-but-live", "newest"],
+ now=1000.0,
+ )
+ kept = {r["subject"] for r in store.list_episodes(tier=TIER_TRANSCRIPT)}
+ assert kept == {"newer-but-gone", "newest"}
+
+ def test_a_source_that_comes_back_unread_loses_the_note(self, tmp_path: Path) -> None:
+ """The steady-state case, and the one a re-derivation test cannot reach.
+
+ An episode is only re-derived when its source is *read*, and an old
+ session has nothing new to read. If the note were cleared only on
+ re-derivation, a single directory listing that missed a file would
+ stamp it gone permanently while the file sat on disk.
+ """
+ def session(subject: str) -> Episode:
+ return Episode(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ subject=subject,
+ body="b",
+ evidence="e",
+ )
+
+ with _store(tmp_path) as store:
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[session("one"), session("two")],
+ present_subjects=["one", "two"],
+ now=1000.0,
+ )
+ # One run misses it.
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[],
+ present_subjects=["one"],
+ now=2000.0,
+ )
+ # The next run sees it again and reads nothing new from it.
+ store.accumulate_tier(
+ tier=TIER_TRANSCRIPT,
+ kind="session",
+ episodes=[],
+ present_subjects=["one", "two"],
+ now=3000.0,
+ )
+ rows = {r["subject"]: r for r in store.list_episodes(tier=TIER_TRANSCRIPT)}
+ assert SOURCE_GONE_NOTE not in rows["two"]["evidence"]
+ assert rows["two"]["evidence"] == "e"
diff --git a/tests/unit/precedent/test_transcript_episodes.py b/tests/unit/precedent/test_transcript_episodes.py
index a0e469043..f4aad47d2 100644
--- a/tests/unit/precedent/test_transcript_episodes.py
+++ b/tests/unit/precedent/test_transcript_episodes.py
@@ -14,6 +14,7 @@ import time
import pytest
from repowise.core.precedent.store import (
+ SOURCE_GONE_NOTE,
TIER_GIT,
TIER_TRANSCRIPT,
Episode,
@@ -388,13 +389,19 @@ def test_a_quiet_session_keeps_its_episode(tmp_path):
assert "hello" in row["body"]
-def test_a_session_whose_transcript_is_gone_loses_its_episode(tmp_path):
+def test_a_session_whose_transcript_is_gone_keeps_its_episode(tmp_path):
+ """The harness prunes transcripts on its own schedule; the episode is durable.
+
+ Measured on the machine this was written on: 1,509 transcripts, oldest 30
+ days, the age distribution stopping dead at the harness default nobody
+ sets. An episode that died with its transcript could never be older than
+ that, which is the one thing this tier claims to be.
+ """
root = _repo(tmp_path)
kept = _transcript(tmp_path, "kept.jsonl", [_line("user", "still here")])
gone = _transcript(tmp_path, "gone.jsonl", [_line("user", "not for long", session="s2")])
# One recorder across every discovered transcript, which is the production
- # shape: presence is what the whole run saw, so a per-file recorder would
- # have each write delete the file before it.
+ # shape: presence is what the whole run saw.
record_transcript_episodes(root, _fold(root, kept, gone))
assert len(_rows(root, tier=TIER_TRANSCRIPT)) == 2
@@ -403,8 +410,12 @@ def test_a_session_whose_transcript_is_gone_loses_its_episode(tmp_path):
still_there.observe(kept, iter(()))
record_transcript_episodes(root, still_there)
- rows = _rows(root, tier=TIER_TRANSCRIPT)
- assert [r["subject"] for r in rows] == [str(kept).replace("\\", "/")]
+ rows = {r["subject"]: r for r in _rows(root, tier=TIER_TRANSCRIPT)}
+ assert set(rows) == {str(kept).replace("\\", "/"), str(gone).replace("\\", "/")}
+ orphan = rows[str(gone).replace("\\", "/")]
+ # The prose it was made of survives; only the pointer is marked dead.
+ assert "not for long" in orphan["body"]
+ assert orphan["evidence"].endswith(SOURCE_GONE_NOTE)
def test_a_run_that_discovered_nothing_deletes_nothing(tmp_path):
diff --git a/tests/unit/test_fts_query.py b/tests/unit/test_fts_query.py
new file mode 100644
index 000000000..83376a9e0
--- /dev/null
+++ b/tests/unit/test_fts_query.py
@@ -0,0 +1,76 @@
+"""The shared FTS5 expression builder.
+
+Tested here rather than only through ``persistence.search``, which is where it
+used to live: it now has two callers with different corpora, and a module whose
+only test reaches it through one caller loses its own identity the moment that
+caller is refactored.
+
+The escaping is the part these tests exist for. Every character that could
+change the shape of an FTS5 expression has to be gone before a term is built,
+because the second caller is a sidecar store that cannot import the first.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.fts_query import (
+ PREFIX_MIN_CHARS,
+ STOP_WORDS,
+ build_fts5_query,
+ match_term,
+ meaningful_terms,
+)
+
+
+class TestMeaningfulTerms:
+ def test_stop_words_and_single_characters_are_dropped(self) -> None:
+ assert meaningful_terms("How does the a b walker work") == ["walker", "work"]
+
+ @pytest.mark.parametrize(
+ "hostile",
+ ['walker" OR "', "walker*", "(walker)", "walker: NEAR(a b)", "walker^2", "-walker"],
+ )
+ def test_fts5_syntax_never_survives_tokenizing(self, hostile: str) -> None:
+ """The reduction to [a-zA-Z0-9_] is the escaping; nothing else has to be."""
+ for term in meaningful_terms(hostile):
+ assert term.isascii()
+ assert all(c.isalnum() or c == "_" for c in term)
+
+ def test_stop_words_are_the_shared_set(self) -> None:
+ assert "the" in STOP_WORDS
+ assert "walker" not in STOP_WORDS
+
+
+class TestMatchTerm:
+ def test_a_long_term_gets_a_prefix_wildcard(self) -> None:
+ assert match_term("x" * PREFIX_MIN_CHARS) == f'"{"x" * PREFIX_MIN_CHARS}"*'
+
+ def test_a_short_term_is_matched_exactly(self) -> None:
+ short = "x" * (PREFIX_MIN_CHARS - 1)
+ assert match_term(short) == f'"{short}"'
+
+
+class TestBuildQuery:
+ def test_terms_are_or_joined(self) -> None:
+ built = build_fts5_query("walker nested checkouts")
+ assert built.count(" OR ") == 2
+
+ def test_a_question_of_only_stop_words_falls_back_to_a_phrase(self) -> None:
+ assert build_fts5_query("how does it") == '"how does it"'
+
+ def test_the_phrase_fallback_doubles_its_quotes(self) -> None:
+ """The one path where raw text reaches the expression."""
+ assert build_fts5_query('how "it" is') == '"how ""it"" is"'
+
+ def test_a_document_frequency_ceiling_drops_common_terms(self) -> None:
+ """Four terms, because the floor keeps three whatever the ceiling says."""
+ counts = {"": 1000, "walker": 10, "file": 900, "checkouts": 20, "nested": 30}
+ built = build_fts5_query("walker file checkouts nested", counts.__getitem__)
+ assert '"file"' not in built
+ assert '"walker"*' in built
+
+ def test_the_ceiling_never_empties_a_query(self) -> None:
+ counts = {"": 1000, "file": 900, "page": 950, "index": 980, "does": 990}
+ built = build_fts5_query("file page index does", counts.__getitem__)
+ assert built