Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(episodes): keep a session after its transcript is pruned, and search episode bodies (#1331)
Raghav Chamadiya873b820c390f
diff --git a/packages/core/src/repowise/core/fts_query.py b/packages/core/src/repowise/core/fts_query.pynew file mode 100644index 000000000..5b08897b0--- /dev/null+++ b/packages/core/src/repowise/core/fts_query.py@@ -0,0 +1,190 @@+"""Turning a natural-language question into a SQLite FTS5 MATCH expression.++Lifted out of :mod:`repowise.core.persistence.search` when a second store+needed the same expression. It sits at the top of ``repowise.core`` rather than+beside its first caller because the second one — the episode store — is+stdlib-only by test (``tests/unit/precedent/test_structural_perf.py``) and+importing ``repowise.core.persistence`` would pull SQLAlchemy onto a hook path+whose budget was fought down from 965 ms to 155 ms by deleting three+module-level imports.++The escaping in here is the part worth sharing rather than rewriting:+:func:`meaningful_terms` reduces a query to ``[a-zA-Z0-9_]+`` tokens, so every+character that could change the shape of an FTS5 expression is gone before a+term is built, and the one path that quotes raw text doubles its quotes.+"""++from __future__ import annotations++import os+import re+from collections.abc import Callable++#: Shortest term that still gets a prefix wildcard. See :func:`match_term`.+PREFIX_MIN_CHARS = 4++#: A term matching more than this share of the corpus is dropped from the query+#: expression: it cannot discriminate between documents, and every one of them+#: drags thousands of candidates in for BM25 to sort out afterwards.+#: Env-tunable because the right ceiling is a property of a corpus's+#: vocabulary, and the only honest way to set it is to measure recall on the+#: corpus in question.+DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))++#: Never let the ceiling empty a query. A question written entirely out of+#: common words keeps its rarest terms — a thin match beats no match.+MIN_KEPT_TERMS = 3++_TOKEN_RE = re.compile(r"[a-zA-Z0-9_]+")++#: Common English stop words to strip from FTS queries.+STOP_WORDS = frozenset(+ {+ "a",+ "an",+ "the",+ "is",+ "are",+ "was",+ "were",+ "be",+ "been",+ "being",+ "have",+ "has",+ "had",+ "do",+ "does",+ "did",+ "will",+ "would",+ "shall",+ "should",+ "may",+ "might",+ "must",+ "can",+ "could",+ "am",+ "to",+ "of",+ "in",+ "for",+ "on",+ "with",+ "at",+ "by",+ "from",+ "as",+ "into",+ "about",+ "it",+ "its",+ "this",+ "that",+ "these",+ "those",+ "i",+ "we",+ "you",+ "he",+ "she",+ "they",+ "me",+ "him",+ "her",+ "us",+ "them",+ "my",+ "your",+ "his",+ "our",+ "their",+ "what",+ "which",+ "who",+ "whom",+ "how",+ "when",+ "where",+ "why",+ "not",+ "no",+ "so",+ "if",+ "or",+ "and",+ "but",+ "all",+ "each",+ "very",+ "just",+ "also",+ "than",+ "too",+ "only",+ }+)+++def meaningful_terms(query: str) -> list[str]:+ """Alphanumeric tokens of *query*, minus stop words and single characters."""+ tokens = _TOKEN_RE.findall(query.lower())+ return [t for t in tokens if t not in STOP_WORDS and len(t) > 1]+++def match_term(term: str) -> str:+ """One FTS5 term, prefix-matched only when it is long enough to be specific.++ A prefix wildcard is what earns morphological recall — ``invalidat*`` finds+ "invalidates" and "invalidation" — and it is also what turns a short token+ into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.+ Below the cut the term is matched exactly, which costs the odd plural and+ buys back a candidate set that BM25 was being asked to sort out afterwards.+ """+ return f'"{term}"*' if len(term) >= PREFIX_MIN_CHARS else f'"{term}"'+++def build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:+ """Build an FTS5 MATCH expression from a natural-language query.++ Terms are OR-ed, because a developer question is a paraphrase of the page+ that answers it: its rarest words are frequently the asker's vocabulary+ rather than the page's, so requiring all of them (AND) excludes the right+ page far more often than it narrows usefully. Measured on a 3,678-page+ corpus, AND over the meaningful terms returned nothing at all for 65 of 99+ questions, and held the expected page for 25 of 99 against 99 of 99 for OR.++ What OR needs is not a different operator but fewer junk terms. A question+ carries words that are not stop words yet still match a large share of any+ code corpus ("file", "does", "page", "index"), and each one drags in+ thousands of pages that only BM25 tie-breaking then has to sort out. Given+ a *document_frequency* callable, terms matching more than :data:`DF_CEILING`+ of the corpus are dropped from the expression; the same measurement puts+ the median candidate set at 20% of the corpus rather than 65%. Without the+ callable the expression keeps every term, which is the prior behaviour.++ At least :data:`MIN_KEPT_TERMS` terms always survive — the rarest ones — so+ a question written entirely from common words still matches something.+ """+ meaningful = meaningful_terms(query)++ if not meaningful:+ # All stop words — fall back to exact phrase. The one place raw text+ # reaches the expression, so it is the one place quotes are doubled.+ safe = query.replace('"', '""')+ return f'"{safe}"'++ kept = meaningful+ if document_frequency is not None:+ total = document_frequency("") # corpus size, by convention+ if total > 0:+ ceiling = DF_CEILING * total+ selective = [t for t in meaningful if document_frequency(t) <= ceiling]+ if len(selective) < MIN_KEPT_TERMS:+ # Every term is common. Keep the rarest few rather than none:+ # a thin match beats an empty one.+ selective = sorted(meaningful, key=document_frequency)[:MIN_KEPT_TERMS]+ kept = selective++ return " OR ".join(match_term(t) for t in kept)diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex 79f264c04..b7af3f267 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -17,15 +17,20 @@ Usage::from __future__ import annotationsimport logging-import osfrom collections import OrderedDict-from collections.abc import Callablefrom dataclasses import dataclassfrom typing import Literalfrom sqlalchemy.ext.asyncio import AsyncEnginefrom sqlalchemy.sql import text+from repowise.core.fts_query import DF_CEILING as _DF_CEILING+from repowise.core.fts_query import MIN_KEPT_TERMS as _MIN_KEPT_TERMS+from repowise.core.fts_query import PREFIX_MIN_CHARS as _PREFIX_MIN_CHARS+from repowise.core.fts_query import build_fts5_query as _build_fts5_query+from repowise.core.fts_query import match_term as _match_term+from repowise.core.fts_query import meaningful_terms as _meaningful_terms+from .information_floor import information_floor, meets_information_floor, substantive_text@@ -83,21 +88,6 @@ PG_FTS_EXPRESSION = ("|| COALESCE(summary,'') || ' ' || COALESCE(target_path,''))")-# Shortest term that still gets a prefix wildcard. See :func:`_match_term`.-_PREFIX_MIN_CHARS = 4--# A term matching more than this share of the corpus is dropped from the query-# expression: it cannot discriminate between pages, and every one of them drags-# thousands of candidates in for BM25 to sort out afterwards. Env-tunable-# because the right ceiling is a property of a corpus's vocabulary, and the only-# honest way to set it is to measure recall on the corpus in question.-_DF_CEILING = float(os.environ.get("REPOWISE_FTS_DF_CEILING", "0.20"))--# Never let the ceiling empty a query. A question written entirely out of common-# words keeps its rarest terms — a thin match beats no match, especially since-# vector retrieval is answering the same question alongside this.-_MIN_KEPT_TERMS = 3-# How many distinct terms' document frequencies one FullTextSearch keeps. Term# frequency changes only when the corpus is rewritten, and questions reuse# vocabulary heavily, so a small cache removes almost all of the counting.@@ -109,115 +99,6 @@ _SCORE_EPSILON = 1e-6_log = logging.getLogger(__name__)-# Common English stop words to strip from FTS queries-_STOP_WORDS = frozenset(- {- "a",- "an",- "the",- "is",- "are",- "was",- "were",- "be",- "been",- "being",- "have",- "has",- "had",- "do",- "does",- "did",- "will",- "would",- "shall",- "should",- "may",- "might",- "must",- "can",- "could",- "am",- "to",- "of",- "in",- "for",- "on",- "with",- "at",- "by",- "from",- "as",- "into",- "about",- "it",- "its",- "this",- "that",- "these",- "those",- "i",- "we",- "you",- "he",- "she",- "they",- "me",- "him",- "her",- "us",- "them",- "my",- "your",- "his",- "our",- "their",- "what",- "which",- "who",- "whom",- "how",- "when",- "where",- "why",- "not",- "no",- "so",- "if",- "or",- "and",- "but",- "all",- "each",- "very",- "just",- "also",- "than",- "too",- "only",- }-)---def _meaningful_terms(query: str) -> list[str]:- """Alphanumeric tokens of *query*, minus stop words and single characters."""- import re-- tokens = re.findall(r"[a-zA-Z0-9_]+", query.lower())- return [t for t in tokens if t not in _STOP_WORDS and len(t) > 1]---def _match_term(term: str) -> str:- """One FTS5 term, prefix-matched only when it is long enough to be specific.-- A prefix wildcard is what earns morphological recall — ``invalidat*`` finds- "invalidates" and "invalidation" — and it is also what turns a short token- into a corpus-wide match: ``set*`` hits settings, setup, setter, setdefault.- Below the cut the term is matched exactly, which costs the odd plural and- buys back a candidate set that BM25 was being asked to sort out afterwards.- """- return f'"{term}"*' if len(term) >= _PREFIX_MIN_CHARS else f'"{term}"'-def _pg_term(term: str) -> str:"""One ``to_tsquery`` lexeme, prefix-matched on the same rule as FTS5.@@ -229,52 +110,6 @@ def _pg_term(term: str) -> str:return f"{term}:*" if len(term) >= _PREFIX_MIN_CHARS else term-def _build_fts5_query(query: str, document_frequency: Callable[[str], int] | None = None) -> str:- """Build an FTS5 MATCH expression from a natural-language query.-- Terms are OR-ed, because a developer question is a paraphrase of the page- that answers it: its rarest words are frequently the asker's vocabulary- rather than the page's, so requiring all of them (AND) excludes the right- page far more often than it narrows usefully. Measured on a 3,678-page- corpus, AND over the meaningful terms returned nothing at all for 65 of 99- questions, and held the expected page for 25 of 99 against 99 of 99 for OR.-- What OR needs is not a different operator but fewer junk terms. A question- carries words that are not stop words yet still match a large share of any- code corpus ("file", "does", "page", "index"), and each one drags in- thousands of pages that only BM25 tie-breaking then has to sort out. Given- a *document_frequency* callable, terms matching more than- :data:`_DF_CEILING` of the corpus are dropped from the expression; the same- measurement puts the median candidate set at 20% of the corpus rather than- 65%. Without the callable the expression keeps every term, which is the- prior behaviour.-- At least :data:`_MIN_KEPT_TERMS` terms always survive — the rarest ones —- so a question written entirely from common words still matches something.- """- meaningful = _meaningful_terms(query)-- if not meaningful:- # All stop words — fall back to exact phrase- safe = query.replace('"', '""')- return f'"{safe}"'-- kept = meaningful- if document_frequency is not None:- total = document_frequency("") # corpus size, by convention- if total > 0:- ceiling = _DF_CEILING * total- selective = [t for t in meaningful if document_frequency(t) <= ceiling]- if len(selective) < _MIN_KEPT_TERMS:- # Every term is common. Keep the rarest few rather than none:- # a thin match beats an empty one, and the vector arm is still- # answering alongside this.- selective = sorted(meaningful, key=document_frequency)[:_MIN_KEPT_TERMS]- kept = selective-- return " OR ".join(_match_term(t) for t in kept)--def snippet_around(content: str, query: str, length: int = _SNIPPET_LEN) -> str:"""A window of *content* centred on the first term of *query* it carries.diff --git a/packages/core/src/repowise/core/precedent/store.py b/packages/core/src/repowise/core/precedent/store.pyindex 6e874ba20..6328a5c7a 100644--- a/packages/core/src/repowise/core/precedent/store.py+++ b/packages/core/src/repowise/core/precedent/store.py@@ -20,12 +20,17 @@ from __future__ import annotationsimport hashlibimport json+import loggingimport sqlite3import timefrom collections.abc import Iterable, Sequencefrom dataclasses import dataclass, fieldfrom pathlib import Path+from repowise.core.fts_query import build_fts5_query, meaningful_terms++_log = logging.getLogger(__name__)+EPISODES_DIRNAME = "episodes"EPISODES_DB_FILENAME = "episodes.db"@@ -53,6 +58,17 @@ DEFAULT_MAX_ROWS = 5000#: chunk rather than one variable past whatever SQLite was compiled with._IN_CHUNK = 500+#: How much wider than the caller's limit the BM25 window is drawn, so the+#: rerank has something to reorder, and the hard ceiling on it, because every+#: row in the window is a body this store then scans.+_RERANK_FACTOR = 5+_RERANK_WINDOW_MAX = 50++#: Appended to an episode's evidence the first time a run finds the source it+#: was derived from gone. The episode itself stays: the body is the content and+#: the pointer was only ever provenance.+SOURCE_GONE_NOTE = " (source no longer on disk)"+_SCHEMA = """CREATE TABLE IF NOT EXISTS episodes (id TEXT PRIMARY KEY,@@ -70,6 +86,31 @@ CREATE INDEX IF NOT EXISTS idx_episodes_tier_kind ON episodes(tier, kind);CREATE INDEX IF NOT EXISTS idx_episodes_last_seen ON episodes(last_seen_at);"""+#: The searchable projection of an episode. External-content rather than a+#: table of its own: ``body`` is already a column and this corpus is 11 MB on+#: a real machine, so a standalone index would keep a second copy of all of it+#: for nothing. The columns are named after the ones they shadow, which is what+#: lets SQLite read them back out of ``episodes`` on a hit.+_FTS_SCHEMA = """+CREATE VIRTUAL TABLE IF NOT EXISTS episode_fts USING fts5(+ subject, body, content='episodes', content_rowid='rowid'+);+CREATE TRIGGER IF NOT EXISTS episodes_fts_ai AFTER INSERT ON episodes BEGIN+ INSERT INTO episode_fts(rowid, subject, body)+ VALUES (new.rowid, new.subject, new.body);+END;+CREATE TRIGGER IF NOT EXISTS episodes_fts_ad AFTER DELETE ON episodes BEGIN+ INSERT INTO episode_fts(episode_fts, rowid, subject, body)+ VALUES ('delete', old.rowid, old.subject, old.body);+END;+CREATE TRIGGER IF NOT EXISTS episodes_fts_au AFTER UPDATE ON episodes BEGIN+ INSERT INTO episode_fts(episode_fts, rowid, subject, body)+ VALUES ('delete', old.rowid, old.subject, old.body);+ INSERT INTO episode_fts(rowid, subject, body)+ VALUES (new.rowid, new.subject, new.body);+END;+"""+def default_store_path(repo_path: Path | str) -> Path:"""Path to *repo_path*'s episode database (not created here)."""@@ -137,6 +178,47 @@ class EpisodeStore:self._conn.execute("PRAGMA busy_timeout=5000")self._conn.executescript(_SCHEMA)self._conn.commit()+ self.fts_enabled = self._ensure_fts()++ def _ensure_fts(self) -> bool:+ """Create the search index and refill it if it is behind. Never raises.++ Modelled on ``persistence/search.py:_upgrade_sqlite_schema``, whose one+ binding rule is that it must not raise: every command opens the store+ through it, so an exception over an index would take down a product+ that is still perfectly usable without one. Here the stake is smaller+ and the rule is the same — a store that cannot build an index still+ answers every read the guard and the hooks make of it.++ Refill is one statement rather than a re-derivation. The rows are the+ system of record and ``body`` is a column of them, so FTS5's own+ ``rebuild`` reads what a hand-written refill would have selected.++ Drift is detected by which objects are missing, not by comparing row+ counts: an external-content table reads through to ``episodes`` on a+ scan, so ``count(*)`` answers with the content table's size and reports+ an index holding nothing as full. Presence is the honest signal, and it+ is enough — once the triggers exist no drift can accumulate behind them.+ """+ try:+ existing = {+ row[0]+ for row in self._conn.execute(+ "SELECT name FROM sqlite_master WHERE name IN "+ "('episode_fts', 'episodes_fts_ai', 'episodes_fts_ad', 'episodes_fts_au')"+ )+ }+ self._conn.executescript(_FTS_SCHEMA)+ if len(existing) < 4:+ # A store written before this index existed, or by a build+ # without FTS5, or one whose objects somebody dropped.+ self._conn.execute("INSERT INTO episode_fts(episode_fts) VALUES ('rebuild')")+ self._conn.commit()+ return True+ except sqlite3.Error:+ self._conn.rollback()+ _log.debug("episode search index unavailable", exc_info=True)+ return False@classmethoddef open_for_repo(cls, repo_path: Path | str) -> EpisodeStore:@@ -234,7 +316,7 @@ class EpisodeStore:self.prune(now=stamp)return dropped- def sync_tier(+ def accumulate_tier(self,*,tier: str,@@ -243,50 +325,94 @@ class EpisodeStore:present_subjects: Sequence[str],now: float | None = None,) -> int:- """Accumulate into *tier*, bounded by what still exists.+ """Accumulate into *tier*, outliving the sources the episodes came from.The third lifecycle, and the reason it is not one of the other two:* :meth:`replace_kinds` says *this run derived the whole kind*, which- is false here — a run reads only the transcript bytes appended since- the last one, so a sweep would delete every session it did not- revisit;+ is false here — a run reads only the bytes appended since the last+ one, so a sweep would delete every source it did not revisit;* :meth:`append_tier` says *this run covered a window and everything in- it is re-observed*, which is also false — the pass has no window, and- nothing about reading today's session says last March's still exists.-- What is true, and true of no other tier, is that this one's membership- can be **enumerated without being read**: the sources are files on- disk, so a run knows which episodes still have something behind them- for the cost of the directory listing it already did. That is the- vouch. *present_subjects* are marked re-observed whether or not they- were read this run — which is what stops a live episode dying of the- TTL simply because its session ended — and the rest of *kind* is- dropped, because its source is gone rather than merely quiet.-- Returns rows dropped for having no source left.+ it is re-observed*, which is also false — the pass has no window.++ What replaces both is that an episode is **self-contained**: the body+ holds the prose, so the source pointer is provenance and not content.+ The source is therefore free to disappear, and it will. The harness+ this tier reads from prunes its transcripts at 30 days by default,+ counted here at 1,509 files whose age distribution stops dead at 30 —+ so an earlier version of this method, which read "the file is gone" as+ "the episode is gone", capped the tier at the harness's retention and+ could never support the one claim the tier exists to make. Being the+ only durable copy is the strongest thing the store has to say.++ What bounds the tier instead is what bounds the other accumulating one:+ the TTL and the per-tier row cap. Every run marks the whole kind+ re-observed, because a run of the index is what vouches that this+ repository is still live; the TTL then evicts a tier whose repository+ has stopped being indexed altogether, which is what it is for.++ *present_subjects* survives the change with a smaller job. It is still+ the cheap enumeration — the sources are files on disk, so a listing+ answers it without a read — but a missing source is now recorded as a+ **note on the row** rather than acted on, so a reader can say the+ quotation outlived what it was quoted from instead of following a+ pointer into nothing. An empty *present_subjects* is no vouch at all+ (a run on a machine that never had the sources looks identical to one+ whose sources all vanished), so it annotates nothing.++ Returns rows newly marked as having lost their source."""stamp = time.time() if now is None else now- ids = [episode_id(tier, kind, s) for s in present_subjects]- with self._conn: # one transaction: never half-synced+ with self._conn: # one transaction: never half-writtenself._upsert(episodes, stamp)- # Chunked: a heavy machine's transcript directory can outrun- # SQLite's per-statement variable limit, and a partial IN list- # would read as "these are gone" and delete live episodes.- for start in range(0, len(ids), _IN_CHUNK):- batch = ids[start : start + _IN_CHUNK]- placeholders = ",".join("?" * len(batch))- self._conn.execute(- f"UPDATE episodes SET last_seen_at = ? WHERE id IN ({placeholders})",- (stamp, *batch),- )- cur = self._conn.execute(- "DELETE FROM episodes WHERE tier = ? AND kind = ? AND last_seen_at < ?",- (tier, kind, stamp),+ self._conn.execute(+ "UPDATE episodes SET last_seen_at = ? WHERE tier = ? AND kind = ?",+ (stamp, tier, kind),)- dropped = cur.rowcount or 0+ noted = self._note_missing_sources(tier, kind, present_subjects)self.prune(now=stamp)- return dropped+ return noted++ def _note_missing_sources(+ self, tier: str, kind: str, present_subjects: Sequence[str]+ ) -> int:+ """Make the note match what is on disk, in both directions. In-transaction.++ A temporary table rather than a ``NOT IN`` list, because this is a+ negation and a negation cannot be chunked: a partial list would read as+ "everything else is gone" and annotate live rows. The same reasoning+ that made the positive form chunk-safe makes this form all-or-nothing.++ The note is **cleared as well as written**, and the asymmetry it fixes+ is not hypothetical. Clearing it only on re-derivation would leave the+ note stuck forever on the ordinary case: a source has to be *read* to+ be re-derived, an old session has nothing new to read, and one+ directory listing that misses a file — a transient failure, no deletion+ needed — would then mark it gone permanently while the file sat there.+ Presence is a fact about now, so the row is made to agree with now.+ """+ if not present_subjects:+ return 0+ ids = [(episode_id(tier, kind, s),) for s in present_subjects]+ self._conn.execute("CREATE TEMP TABLE IF NOT EXISTS present_ids (id TEXT PRIMARY KEY)")+ self._conn.execute("DELETE FROM present_ids")+ self._conn.executemany("INSERT OR IGNORE INTO present_ids (id) VALUES (?)", ids)+ cur = self._conn.execute(+ # ``instr`` rather than LIKE: the marker is a literal, and a LIKE+ # pattern is a thing to escape even when it is a constant today.+ "UPDATE episodes SET evidence = evidence || ? "+ "WHERE tier = ? AND kind = ? AND instr(evidence, ?) = 0 "+ "AND id NOT IN (SELECT id FROM present_ids)",+ (SOURCE_GONE_NOTE, tier, kind, SOURCE_GONE_NOTE),+ )+ noted = cur.rowcount or 0+ self._conn.execute(+ "UPDATE episodes SET evidence = replace(evidence, ?, '') "+ "WHERE tier = ? AND kind = ? AND instr(evidence, ?) > 0 "+ "AND id IN (SELECT id FROM present_ids)",+ (SOURCE_GONE_NOTE, tier, kind, SOURCE_GONE_NOTE),+ )+ return noteddef _upsert(self, episodes: Iterable[Episode], stamp: float) -> None:"""Insert or refresh *episodes*. Caller owns the transaction.@@ -438,6 +564,96 @@ class EpisodeStore:sql += " ORDER BY birth_at DESC, id ASC"return [_row_to_dict(row) for row in self._conn.execute(sql, params)]+ def search(+ self,+ query: str,+ *,+ tiers: Sequence[str] | None = None,+ limit: int = 10,+ ) -> list[dict]:+ """Episodes matching *query*, best first, each with its BM25 ``score``.++ Ranked rather than filtered, which is the difference between this and+ :meth:`list_episodes`: a question is a paraphrase of the episode that+ answers it, so the expression is built by the same shared builder the+ wiki index uses (:mod:`repowise.core.fts_query`) — it is where the+ reduction to ``[a-zA-Z0-9_]+`` lives, and reimplementing that escaping+ is how a store learns about FTS5's grammar the expensive way.++ BM25 orders the candidates and a second pass reorders them by how many+ **distinct** question terms the body carries, BM25 breaking the ties.+ Measured on this repository's own store against a grep-established+ answer key: 74% to 84% hit@3, 58% to 68% hit@1. The reason is the shape+ of the document — a session is long and repeats itself, so a body+ mentioning one term of the question forty times outscores one that+ answers all four, and term frequency is exactly what BM25 rewards.+ Counting distinct terms fits no constant to this corpus; the signal is+ the question's own vocabulary.++ The opposite lever was tried first and is recorded because it looked+ obvious: passing the wiki index's document-frequency ceiling made this+ *worse* (74% to 68%), since a term in a fifth of the sessions is+ ordinary here rather than uninformative.++ Coverage counts a term as present by substring, so ``cat`` is found+ inside ``concatenate``. That is loose, and it is what the numbers above+ were measured with: tightening it to word boundaries is a change to the+ thing being measured, not a free correctness fix, so it belongs with a+ re-run rather than with a tidy-up.++ Returns nothing rather than raising when there is no index, and the+ same for a query FTS5 will not parse: a search that cannot run is a+ feature that says nothing, never a caller that has to handle it.+ """+ if not self.fts_enabled or not query.strip():+ return []+ if tiers is not None and not tiers:+ return []+ # Deliberately without the document-frequency ceiling the wiki index+ # passes here. Measured on this store: applying it moved r1's+ # grep-grounded regression from 74% to 68% hit@3, and pushed the+ # episode holding every rare term of a question from rank 9 to 11.+ # A session body is long and mentions many things, so a term in a fifth+ # of the sessions is ordinary rather than uninformative, and matching+ # more of a question's vocabulary discriminates better than matching+ # only its rarest words. The ceiling is a property of the wiki corpus's+ # vocabulary, not a general truth, and it does not transfer.+ expression = build_fts5_query(query)+ sql = (+ "SELECT e.id, e.tier, e.kind, e.subject, e.body, e.evidence, e.nodes, "+ "e.birth_commit, e.birth_at, e.last_seen_at, bm25(episode_fts) AS score "+ "FROM episode_fts JOIN episodes e ON e.rowid = episode_fts.rowid "+ "WHERE episode_fts MATCH ?"+ )+ params: list[object] = [expression]+ if tiers is not None:+ sql += f" AND e.tier IN ({','.join('?' * len(tiers))})"+ params.extend(tiers)+ # BM25 is negative and more negative is better, so plain ascending+ # order is best-first; the sign is flipped on the way out.+ sql += " ORDER BY score LIMIT ?"+ # Reranking can only reorder what BM25 hands it, so the window is wider+ # than the caller asked for — and bounded, because every row in it is a+ # body this method then scans.+ window = max(1, limit) * _RERANK_FACTOR+ params.append(min(window, _RERANK_WINDOW_MAX))+ try:+ rows = self._conn.execute(sql, params).fetchall()+ except sqlite3.Error:+ _log.debug("episode search failed for %r", query, exc_info=True)+ return []++ hits = [{**_row_to_dict(row), "score": -(row[10] or 0.0)} for row in rows]+ terms = set(meaningful_terms(query))+ if terms:+ hits.sort(+ key=lambda h: (+ -sum(1 for t in terms if t in h["body"].casefold()),+ -h["score"],+ )+ )+ return hits[: max(1, limit)]+def count(self) -> int:(rows,) = self._conn.execute("SELECT COUNT(*) FROM episodes").fetchone()return int(rows)diff --git a/packages/core/src/repowise/core/precedent/transcript_episodes.py b/packages/core/src/repowise/core/precedent/transcript_episodes.pyindex a2de8beab..0e94a1fb2 100644--- a/packages/core/src/repowise/core/precedent/transcript_episodes.py+++ b/packages/core/src/repowise/core/precedent/transcript_episodes.py@@ -123,7 +123,7 @@ class TranscriptEpisodeRecorder:Presence is registered when a transcript is *shown* to the recorder, notwhen it yields events: a session already fully consumed by the cursoryields nothing and must still count as present, or the next write would- read its silence as deletion.+ read its silence as a source that had gone away."""def __init__(self, repo_root: Path) -> None:@@ -134,11 +134,11 @@ class TranscriptEpisodeRecorder:def note_present(self, paths: Iterable[Path]) -> None:"""Register transcripts as existing without reading them.- Presence and reading are different claims, and conflating them deletes- data: a run may stop reading early on a large first sweep, but every- transcript it *listed* still has a session behind it. Called with the- whole discovery result, so a partial sweep never reads its own- unfinished work as sessions that have gone away.+ Presence and reading are different claims: a run may stop reading early+ on a large first sweep, but every transcript it *listed* still has a+ session behind it. Called with the whole discovery result, so a partial+ sweep never records its own unfinished work as sources that have gone+ away."""for path in paths:self._present.append(self._subject(path))@@ -354,13 +354,13 @@ def record_transcript_episodes(repo_path: Path | str, recorder: TranscriptEpisodpending = recorder.pending()if not subjects and not pending:# Nothing was even discovered: this run cannot vouch for the tier's- # membership, so it must not sweep it. Same rule as a git run that+ # membership, so it must not annotate it. Same rule as a git run that# walked no window.return 0with EpisodeStore.open_for_repo(root) as store:prior = _prior_rows(store, [f.subject for f in pending])episodes = derive_transcript_episodes(recorder, root, prior)- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind=KIND_SESSION,episodes=episodes,diff --git a/packages/core/src/repowise/core/sessions/miners/decisions.py b/packages/core/src/repowise/core/sessions/miners/decisions.pyindex 88e534e61..f217dec05 100644--- a/packages/core/src/repowise/core/sessions/miners/decisions.py+++ b/packages/core/src/repowise/core/sessions/miners/decisions.py@@ -706,7 +706,8 @@ async def mine_session_decisions(deferred = 0discovered = adapter.discover(repo_root, projects_root=projects_root)# Every discovered transcript is present whether or not this run gets- # to read it; the episode writer treats absence as deletion.+ # to read it; the episode writer notes absence on the row it can no+ # longer point at, and keeps the episode.recorder.note_present(discovered)for index, path in enumerate(discovered):if time.monotonic() > deadline:diff --git a/tests/unit/precedent/test_episode_store.py b/tests/unit/precedent/test_episode_store.pyindex 1be70f33c..210c9ee7a 100644--- a/tests/unit/precedent/test_episode_store.py+++ b/tests/unit/precedent/test_episode_store.py@@ -8,6 +8,7 @@ import pytestfrom repowise.core.precedent.store import (SHAREABLE_TIERS,+ SOURCE_GONE_NOTE,TIER_GIT,TIER_STRUCTURAL,TIER_TRANSCRIPT,@@ -220,11 +221,13 @@ class TestPaths:assert not path.exists() # resolving must not create anything-class TestSyncTier:- """The third lifecycle: accumulate, bounded by what still exists.+class TestAccumulateTier:+ """The third lifecycle: accumulate, and outlive the sources.- Its whole reason to exist is that this tier's membership can be enumerated- without being read, which is true of no other tier.+ The tier's sources are pruned by a harness on a schedule nobody sets, so an+ episode that dies with its source can only ever be as old as that schedule.+ The body is the content and the pointer is provenance, so the episode+ stays and the missing source becomes a note."""def _session(self, subject: str, body: str = "b", birth_at: float = 10.0) -> Episode:@@ -241,7 +244,7 @@ class TestSyncTier:def test_a_present_but_unread_member_is_kept_and_re_observed(self, tmp_path: Path) -> None:"""The case append_tier cannot express and replace_kinds would delete."""with _store(tmp_path) as store:- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[self._session("one"), self._session("two")],@@ -249,7 +252,7 @@ class TestSyncTier:now=1000.0,)# A later run reads nothing new but both sources are still there.- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[],@@ -262,30 +265,124 @@ class TestSyncTier:# Re-observation must not reset the birth.assert {r["birth_at"] for r in rows} == {10.0}- def test_a_member_whose_source_is_gone_is_dropped(self, tmp_path: Path) -> None:+ def test_a_member_whose_source_is_gone_is_kept_and_noted(self, tmp_path: Path) -> None:+ """The retention fix, stated as a test: the episode outlives its source."""with _store(tmp_path) as store:- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[self._session("one"), self._session("two")],present_subjects=["one", "two"],now=1000.0,)- store.sync_tier(+ noted = store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[],present_subjects=["one"],now=2000.0,)- assert [r["subject"] for r in store.list_episodes(tier=TIER_TRANSCRIPT)] == ["one"]+ assert noted == 1+ rows = {r["subject"]: r for r in store.list_episodes(tier=TIER_TRANSCRIPT)}+ assert set(rows) == {"one", "two"}+ assert rows["two"]["evidence"].endswith(SOURCE_GONE_NOTE)+ assert SOURCE_GONE_NOTE not in rows["one"]["evidence"]+ # The body — the thing the episode is actually made of — is intact.+ assert rows["two"]["body"] == "b"++ def test_the_note_is_written_once_however_often_it_is_missed(self, tmp_path: Path) -> None:+ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[self._session("one"), self._session("two")],+ present_subjects=["one", "two"],+ now=1000.0,+ )+ noted = [+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[],+ present_subjects=["one"],+ now=stamp,+ )+ for stamp in (2000.0, 3000.0, 4000.0)+ ]+ # Newly marked once, then nothing left to mark: the count is the+ # contract, and a note appended every run would still read as one+ # note if only the string were checked.+ assert noted == [1, 0, 0]+ (two,) = [+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"+ ]+ assert two["evidence"].count(SOURCE_GONE_NOTE) == 1++ def test_a_source_that_comes_back_loses_the_note(self, tmp_path: Path) -> None:+ """A re-derivation rewrites the evidence, so the note is not sticky."""+ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[self._session("one"), self._session("two")],+ present_subjects=["one", "two"],+ now=1000.0,+ )+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[],+ present_subjects=["one"],+ now=2000.0,+ )+ # Assert the transition, not just the destination: without this the+ # test passes on code that never writes a note at all.+ (gone,) = [+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"+ ]+ assert SOURCE_GONE_NOTE in gone["evidence"]++ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[self._session("two")],+ present_subjects=["one", "two"],+ now=3000.0,+ )+ (two,) = [+ r for r in store.list_episodes(tier=TIER_TRANSCRIPT) if r["subject"] == "two"+ ]+ assert SOURCE_GONE_NOTE not in two["evidence"]++ def test_a_run_that_enumerated_nothing_annotates_nothing(self, tmp_path: Path) -> None:+ """No listing is "cannot tell", not "every source has gone"."""+ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[self._session("one")],+ present_subjects=["one"],+ now=1000.0,+ )+ assert (+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[],+ present_subjects=[],+ now=2000.0,+ )+ == 0+ )+ (row,) = store.list_episodes(tier=TIER_TRANSCRIPT)+ assert SOURCE_GONE_NOTE not in row["evidence"]def test_it_leaves_other_tiers_and_kinds_alone(self, tmp_path: Path) -> None:with _store(tmp_path) as store:store.replace_kinds(tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()], now=500.0)- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[self._session("one")],@@ -295,20 +392,27 @@ class TestSyncTier:assert len(store.list_episodes(tier=TIER_STRUCTURAL)) == 1assert len(store.list_episodes(tier=TIER_TRANSCRIPT)) == 1- def test_membership_larger_than_one_in_list_survives_chunking(self, tmp_path: Path) -> None:- """A partial IN list would read as "these are gone" and delete live rows."""+ def test_a_membership_past_the_variable_limit_annotates_nothing(+ self, tmp_path: Path+ ) -> None:+ """The negation is all-or-nothing: a chunk of it would annotate live rows.++ SQLite's per-statement variable limit is what a heavy machine's+ transcript directory outruns, so the membership goes through a temp+ table instead of an ``IN`` list.+ """from repowise.core.precedent.store import _IN_CHUNKsubjects = [f"s{i}" for i in range(_IN_CHUNK + 25)]with _store(tmp_path) as store:- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[self._session(s) for s in subjects],present_subjects=subjects,now=1000.0,)- store.sync_tier(+ noted = store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[],@@ -316,8 +420,10 @@ class TestSyncTier:now=2000.0,)rows = store.list_episodes(tier=TIER_TRANSCRIPT)+ assert noted == 0assert len(rows) == len(subjects)assert {r["last_seen_at"] for r in rows} == {2000.0}+ assert not [r for r in rows if SOURCE_GONE_NOTE in r["evidence"]]def test_the_transcript_tier_is_capped_without_touching_the_others(self, tmp_path: Path@@ -334,7 +440,7 @@ class TestSyncTier:tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()], now=500.0)subjects = [f"s{i}" for i in range(10)]- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[@@ -357,7 +463,7 @@ class TestListFilters:store.replace_kinds(tier=TIER_STRUCTURAL, kinds=["nested_repos"], episodes=[_episode()])- store.sync_tier(+ store.accumulate_tier(tier=TIER_TRANSCRIPT,kind="session",episodes=[@@ -395,3 +501,284 @@ class TestListFilters:with _store(tmp_path) as store, pytest.raises(ValueError, match="at most"):store.list_episodes(subjects=[f"s{i}" for i in range(_IN_CHUNK + 1)])+++class TestSearch:+ """Ranked retrieval over the bodies.++ The tier this exists for holds one row per session with the session's prose+ in it, so the only way to ask it a question is to rank; a filter over node+ sets answers "what happened to this file", never "where did we discuss+ this".+ """++ def _episodes(self) -> list[Episode]:+ return [+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject="a.jsonl",+ body="the traverser skips nested git repositories while walking",+ evidence="e",+ ),+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject="b.jsonl",+ body="the embedder batches pages before calling the provider",+ evidence="e",+ ),+ Episode(+ tier=TIER_GIT,+ kind="code_fix",+ subject="fix the nested checkout walk",+ body="a nested checkout was being descended into",+ evidence="e",+ ),+ ]++ def _fill(self, store: EpisodeStore) -> None:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[e for e in self._episodes() if e.tier == TIER_TRANSCRIPT],+ present_subjects=["a.jsonl", "b.jsonl"],+ )+ store.append_tier(+ tier=TIER_GIT,+ episodes=[e for e in self._episodes() if e.tier == TIER_GIT],+ oldest_birth_at=0.0,+ )++ def test_a_question_finds_the_episode_that_answers_it(self, tmp_path: Path) -> None:+ with _store(tmp_path) as store:+ self._fill(store)+ hits = store.search("how are nested repositories handled when walking the tree")+ assert hits+ assert hits[0]["subject"] == "a.jsonl"+ assert hits[0]["score"] > 0++ def test_tiers_filter_the_ranking_rather_than_the_query(self, tmp_path: Path) -> None:+ with _store(tmp_path) as store:+ self._fill(store)+ hits = store.search("nested repositories", tiers=[TIER_GIT])+ assert {h["tier"] for h in hits} == {TIER_GIT}+ assert store.search("nested repositories", tiers=[]) == []++ def test_a_deleted_episode_leaves_the_index(self, tmp_path: Path) -> None:+ """The triggers are what make this true without a second write path."""+ with _store(tmp_path) as store:+ self._fill(store)+ assert store.search("embedder batches")+ store.replace_kinds(tier=TIER_TRANSCRIPT, kinds=["session"], episodes=[])+ assert store.search("embedder batches") == []++ def test_an_edited_body_is_searchable_at_its_new_text(self, tmp_path: Path) -> None:+ """An upsert is an UPDATE, and a stale index would answer with old prose."""+ with _store(tmp_path) as store:+ self._fill(store)+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject="a.jsonl",+ body="the traverser now consults the submodule manifest",+ evidence="e",+ )+ ],+ present_subjects=["a.jsonl", "b.jsonl"],+ )+ assert [h["subject"] for h in store.search("submodule manifest")] == ["a.jsonl"]+ assert not [h for h in store.search("nested git repositories") if h["subject"] == "a.jsonl"]++ def test_a_query_of_pure_punctuation_says_nothing_rather_than_raising(+ self, tmp_path: Path+ ) -> None:+ """FTS5's grammar is a thing to escape; the shared builder is where that lives."""+ with _store(tmp_path) as store:+ self._fill(store)+ for hostile in ('" OR "', "*", "()", " ", "NEAR(a b", 'walker"'):+ assert isinstance(store.search(hostile), list)++ def test_rows_written_before_the_index_existed_are_backfilled(+ self, tmp_path: Path+ ) -> None:+ """The upgrade path: an installed store predates its own search index."""+ import sqlite3++ with _store(tmp_path) as store:+ self._fill(store)+ path = store.db_path++ # Drop the index and its triggers the way a build without FTS5 leaves it.+ conn = sqlite3.connect(path)+ conn.executescript(+ "DROP TABLE episode_fts;"+ "DROP TRIGGER episodes_fts_ai;"+ "DROP TRIGGER episodes_fts_ad;"+ "DROP TRIGGER episodes_fts_au;"+ )+ conn.commit()+ conn.close()++ with EpisodeStore(path) as reopened:+ assert reopened.fts_enabled+ assert [h["subject"] for h in reopened.search("embedder batches")] == ["b.jsonl"]++ def test_no_index_is_silence_rather_than_an_error(self, tmp_path: Path) -> None:+ """A store that cannot build an index still answers every other read."""+ with _store(tmp_path) as store:+ self._fill(store)+ store.fts_enabled = False+ assert store.search("nested repositories") == []+ assert len(store.list_episodes()) == 3++ def test_the_session_answering_more_of_the_question_outranks_the_louder_one(+ self, tmp_path: Path+ ) -> None:+ """BM25 rewards repetition; a session repeats itself constantly.++ The measured failure this pins: a body saying one term of the question+ over and over outscored the body that carried every term of it, and the+ answer sat at rank 9. Distinct-term coverage is the tie the ranking was+ missing, and it fits no constant to any corpus.+ """+ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject="loud.jsonl",+ # One term of the question, forty times over.+ body=" ".join(["staleness"] * 40),+ evidence="e",+ ),+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject="answer.jsonl",+ body=(+ "the staleness divisor needed fifteen commits in ninety "+ "days before a record ever moved, so it was inert"+ ),+ evidence="e",+ ),+ ],+ present_subjects=["loud.jsonl", "answer.jsonl"],+ )+ hits = store.search("why did staleness need fifteen commits in ninety days")+ assert hits[0]["subject"] == "answer.jsonl"++ def test_the_rerank_window_is_bounded(self, tmp_path: Path) -> None:+ """Every row in the window is a body this store scans, so it has a ceiling.++ Filled past the ceiling on purpose: with a handful of rows the corpus+ bounds the result and the assertion passes whether the cap exists or+ not, which is a test of the fixture rather than of the code.+ """+ from repowise.core.precedent.store import _RERANK_WINDOW_MAX++ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[+ Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject=f"s{i}.jsonl",+ body="the traverser skips nested git repositories",+ evidence="e",+ )+ for i in range(_RERANK_WINDOW_MAX * 2)+ ],+ present_subjects=[f"s{i}.jsonl" for i in range(_RERANK_WINDOW_MAX * 2)],+ )+ assert store.count() > _RERANK_WINDOW_MAX+ assert len(store.search("nested repositories", limit=1000)) == _RERANK_WINDOW_MAX++ def test_a_noted_source_does_not_change_what_the_cap_evicts(self, tmp_path: Path) -> None:+ """Presence must not become an eviction signal, and the reason is the point.++ Every transcript is pruned by the harness eventually, so after a month+ *every* episode's source is gone. Ranking eviction by presence would+ therefore prefer whatever was recorded in the last thirty days — which+ is precisely the ceiling this tier was repaired to escape. Age decides,+ as it does for the other accumulating tier.+ """+ def session(subject: str, birth_at: float) -> Episode:+ return Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject=subject,+ body="b",+ evidence="e",+ birth_at=birth_at,+ )++ (tmp_path / ".repowise").mkdir(exist_ok=True)+ with EpisodeStore(default_store_path(tmp_path), max_rows=2) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[+ session("old-but-live", 1.0),+ session("newer-but-gone", 2.0),+ session("newest", 3.0),+ ],+ present_subjects=["old-but-live", "newest"],+ now=1000.0,+ )+ kept = {r["subject"] for r in store.list_episodes(tier=TIER_TRANSCRIPT)}+ assert kept == {"newer-but-gone", "newest"}++ def test_a_source_that_comes_back_unread_loses_the_note(self, tmp_path: Path) -> None:+ """The steady-state case, and the one a re-derivation test cannot reach.++ An episode is only re-derived when its source is *read*, and an old+ session has nothing new to read. If the note were cleared only on+ re-derivation, a single directory listing that missed a file would+ stamp it gone permanently while the file sat on disk.+ """+ def session(subject: str) -> Episode:+ return Episode(+ tier=TIER_TRANSCRIPT,+ kind="session",+ subject=subject,+ body="b",+ evidence="e",+ )++ with _store(tmp_path) as store:+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[session("one"), session("two")],+ present_subjects=["one", "two"],+ now=1000.0,+ )+ # One run misses it.+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[],+ present_subjects=["one"],+ now=2000.0,+ )+ # The next run sees it again and reads nothing new from it.+ store.accumulate_tier(+ tier=TIER_TRANSCRIPT,+ kind="session",+ episodes=[],+ present_subjects=["one", "two"],+ now=3000.0,+ )+ rows = {r["subject"]: r for r in store.list_episodes(tier=TIER_TRANSCRIPT)}+ assert SOURCE_GONE_NOTE not in rows["two"]["evidence"]+ assert rows["two"]["evidence"] == "e"diff --git a/tests/unit/precedent/test_transcript_episodes.py b/tests/unit/precedent/test_transcript_episodes.pyindex a0e469043..f4aad47d2 100644--- a/tests/unit/precedent/test_transcript_episodes.py+++ b/tests/unit/precedent/test_transcript_episodes.py@@ -14,6 +14,7 @@ import timeimport pytestfrom repowise.core.precedent.store import (+ SOURCE_GONE_NOTE,TIER_GIT,TIER_TRANSCRIPT,Episode,@@ -388,13 +389,19 @@ def test_a_quiet_session_keeps_its_episode(tmp_path):assert "hello" in row["body"]-def test_a_session_whose_transcript_is_gone_loses_its_episode(tmp_path):+def test_a_session_whose_transcript_is_gone_keeps_its_episode(tmp_path):+ """The harness prunes transcripts on its own schedule; the episode is durable.++ Measured on the machine this was written on: 1,509 transcripts, oldest 30+ days, the age distribution stopping dead at the harness default nobody+ sets. An episode that died with its transcript could never be older than+ that, which is the one thing this tier claims to be.+ """root = _repo(tmp_path)kept = _transcript(tmp_path, "kept.jsonl", [_line("user", "still here")])gone = _transcript(tmp_path, "gone.jsonl", [_line("user", "not for long", session="s2")])# One recorder across every discovered transcript, which is the production- # shape: presence is what the whole run saw, so a per-file recorder would- # have each write delete the file before it.+ # shape: presence is what the whole run saw.record_transcript_episodes(root, _fold(root, kept, gone))assert len(_rows(root, tier=TIER_TRANSCRIPT)) == 2@@ -403,8 +410,12 @@ def test_a_session_whose_transcript_is_gone_loses_its_episode(tmp_path):still_there.observe(kept, iter(()))record_transcript_episodes(root, still_there)- rows = _rows(root, tier=TIER_TRANSCRIPT)- assert [r["subject"] for r in rows] == [str(kept).replace("\\", "/")]+ rows = {r["subject"]: r for r in _rows(root, tier=TIER_TRANSCRIPT)}+ assert set(rows) == {str(kept).replace("\\", "/"), str(gone).replace("\\", "/")}+ orphan = rows[str(gone).replace("\\", "/")]+ # The prose it was made of survives; only the pointer is marked dead.+ assert "not for long" in orphan["body"]+ assert orphan["evidence"].endswith(SOURCE_GONE_NOTE)def test_a_run_that_discovered_nothing_deletes_nothing(tmp_path):diff --git a/tests/unit/test_fts_query.py b/tests/unit/test_fts_query.pynew file mode 100644index 000000000..83376a9e0--- /dev/null+++ b/tests/unit/test_fts_query.py@@ -0,0 +1,76 @@+"""The shared FTS5 expression builder.++Tested here rather than only through ``persistence.search``, which is where it+used to live: it now has two callers with different corpora, and a module whose+only test reaches it through one caller loses its own identity the moment that+caller is refactored.++The escaping is the part these tests exist for. Every character that could+change the shape of an FTS5 expression has to be gone before a term is built,+because the second caller is a sidecar store that cannot import the first.+"""++from __future__ import annotations++import pytest++from repowise.core.fts_query import (+ PREFIX_MIN_CHARS,+ STOP_WORDS,+ build_fts5_query,+ match_term,+ meaningful_terms,+)+++class TestMeaningfulTerms:+ def test_stop_words_and_single_characters_are_dropped(self) -> None:+ assert meaningful_terms("How does the a b walker work") == ["walker", "work"]++ @pytest.mark.parametrize(+ "hostile",+ ['walker" OR "', "walker*", "(walker)", "walker: NEAR(a b)", "walker^2", "-walker"],+ )+ def test_fts5_syntax_never_survives_tokenizing(self, hostile: str) -> None:+ """The reduction to [a-zA-Z0-9_] is the escaping; nothing else has to be."""+ for term in meaningful_terms(hostile):+ assert term.isascii()+ assert all(c.isalnum() or c == "_" for c in term)++ def test_stop_words_are_the_shared_set(self) -> None:+ assert "the" in STOP_WORDS+ assert "walker" not in STOP_WORDS+++class TestMatchTerm:+ def test_a_long_term_gets_a_prefix_wildcard(self) -> None:+ assert match_term("x" * PREFIX_MIN_CHARS) == f'"{"x" * PREFIX_MIN_CHARS}"*'++ def test_a_short_term_is_matched_exactly(self) -> None:+ short = "x" * (PREFIX_MIN_CHARS - 1)+ assert match_term(short) == f'"{short}"'+++class TestBuildQuery:+ def test_terms_are_or_joined(self) -> None:+ built = build_fts5_query("walker nested checkouts")+ assert built.count(" OR ") == 2++ def test_a_question_of_only_stop_words_falls_back_to_a_phrase(self) -> None:+ assert build_fts5_query("how does it") == '"how does it"'++ def test_the_phrase_fallback_doubles_its_quotes(self) -> None:+ """The one path where raw text reaches the expression."""+ assert build_fts5_query('how "it" is') == '"how ""it"" is"'++ def test_a_document_frequency_ceiling_drops_common_terms(self) -> None:+ """Four terms, because the floor keeps three whatever the ceiling says."""+ counts = {"": 1000, "walker": 10, "file": 900, "checkouts": 20, "nested": 30}+ built = build_fts5_query("walker file checkouts nested", counts.__getitem__)+ assert '"file"' not in built+ assert '"walker"*' in built++ def test_the_ceiling_never_empties_a_query(self) -> None:+ counts = {"": 1000, "file": 900, "page": 950, "index": 980, "does": 990}+ built = build_fts5_query("file page index does", counts.__getitem__)+ assert built