Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Make chat search fast and reliable using only its dedicated index (#993)

Regan Bell75b7190f2ee0
diff --git a/plugins/web-ui/src/search.ts b/plugins/web-ui/src/search.ts
index eeb565c14..41ff6fdac 100644
--- a/plugins/web-ui/src/search.ts
+++ b/plugins/web-ui/src/search.ts
@@ -1,10 +1,3 @@
-/** Chat search — a ⌘K palette over every conversation the viewer can see.
- *
- * Results come from the core's Postgres full-text index and are grouped by
- * session, newest first. Enter opens the selected conversation; ⌘/Ctrl+Enter
- * (or the pinned bottom row) pipes the query to QM as the prompt of a new
- * chat, so a fruitless search becomes a question.
- */
import { html, nothing, render, type TemplateResult } from "lit";
import { CornerDownLeft, Search } from "lucide";
import { api, userSendMessage, type CoreSession } from "./core-bridge";
@@ -39,7 +32,6 @@ const searchState = {
hits: [] as ChatSearchHit[],
failed: false,
loading: false,
- /** Selection over hits (0..hits.length-1) plus the trailing ask-QM row. */
sel: 0,
};
@@ -64,6 +56,7 @@ export function openChatSearch(): void {
searchState.query = "";
searchState.hits = [];
searchState.loading = false;
+ searchState.failed = false;
searchState.sel = 0;
draw();
requestAnimationFrame(() => host?.querySelector<HTMLInputElement>(".chat-search-input")?.focus());
@@ -72,6 +65,7 @@ export function openChatSearch(): void {
function closeChatSearch(): void {
if (!searchState.open) return;
searchState.open = false;
+ fetchSeq++;
if (debounceTimer) clearTimeout(debounceTimer);
debounceTimer = null;
inflight?.abort();
@@ -108,12 +102,14 @@ function onQueryInput(e: InputEvent): void {
searchState.query = (e.currentTarget as HTMLInputElement).value;
searchState.sel = 0;
if (debounceTimer) clearTimeout(debounceTimer);
+ debounceTimer = null;
+ fetchSeq++;
+ inflight?.abort();
+ inflight = null;
+ searchState.hits = [];
+ searchState.failed = false;
const q = searchState.query.trim();
if (q.length < MIN_QUERY_LEN) {
- inflight?.abort();
- inflight = null;
- searchState.hits = [];
- searchState.failed = false;
searchState.loading = false;
draw();
return;
@@ -143,8 +139,6 @@ async function runSearch(q: string): Promise<void> {
draw();
}
-/** Cluster hits by session (sessions ordered by their newest hit) so a
- * conversation never fragments into repeated group headers. */
function groupHitsBySession(hits: ChatSearchHit[]): ChatSearchHit[] {
const order: string[] = [];
const bySession = new Map<string, ChatSearchHit[]>();
@@ -179,6 +173,7 @@ function onPaletteKeydown(e: KeyboardEvent): void {
if (e.key === "Enter") {
e.preventDefault();
if (e.metaKey || e.ctrlKey) return void askQm();
+ if (searchState.loading) return;
const hit = searchState.hits[searchState.sel];
if (hit) return void openHit(hit);
if (askRowShown() && searchState.sel === searchState.hits.length) return void askQm();
@@ -302,9 +297,7 @@ function paletteTpl(): TemplateResult {
} else if (searchState.loading && !searchState.hits.length) {
body = html`<div class="chat-search-empty">Searching…</div>`;
} else if (searchState.failed) {
- body = html`<div class="chat-search-empty chat-search-failed">
- Search failed. Check the connection and try again.
- </div>`;
+ body = html`<div class="chat-search-empty chat-search-failed">Search failed. Try again.</div>`;
} else if (!searchState.hits.length) {
body = html`<div class="chat-search-empty">No messages match “${q}”.</div>`;
} else {
diff --git a/plugins/web-ui/test/search-race.test.ts b/plugins/web-ui/test/search-race.test.ts
new file mode 100644
index 000000000..6eac16b81
--- /dev/null
+++ b/plugins/web-ui/test/search-race.test.ts
@@ -0,0 +1,139 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { stripTypeScriptTypes } from "node:module";
+import { test } from "node:test";
+import { createContext, runInContext } from "node:vm";
+
+const source = readFileSync(new URL("../src/search.ts", import.meta.url), "utf8");
+const handlers = source.slice(source.indexOf("function onQueryInput("), source.indexOf("function groupHitsBySession("));
+const lifecycle = source
+ .slice(source.indexOf("export function openChatSearch("), source.indexOf("function ensureHost("))
+ .replace("export function", "function");
+
+const keyboard = source.slice(
+ source.indexOf("function onPaletteKeydown("),
+ source.indexOf("function scrollSelectedIntoView("),
+);
+
+function harness() {
+ const state = { open: true, query: "", hits: [] as unknown[], loading: false, failed: false, sel: 0 };
+ const requests: Array<{ signal: AbortSignal; resolve: (value: unknown) => void; reject: (error: Error) => void }> =
+ [];
+ let asks = 0;
+ let timer: (() => void) | undefined;
+ const context = createContext({
+ searchState: state,
+ MIN_QUERY_LEN: 2,
+ DEBOUNCE_MS: 150,
+ debounceTimer: null,
+ inflight: null,
+ fetchSeq: 0,
+ AbortController,
+ draw() {},
+ clampSel() {},
+ askQm() {
+ asks++;
+ },
+ askRowShown: () => true,
+ requestAnimationFrame() {},
+ groupHitsBySession: (hits: unknown[]) => hits,
+ clearTimeout() {
+ timer = undefined;
+ },
+ setTimeout(fn: () => void) {
+ timer = fn;
+ return 1;
+ },
+ api: (_path: string, { signal }: { signal: AbortSignal }) =>
+ new Promise((resolve, reject) => {
+ requests.push({ signal, resolve, reject });
+ }),
+ });
+ runInContext(stripTypeScriptTypes(handlers + lifecycle + keyboard), context);
+ return {
+ state,
+ requests,
+ asks: () => asks,
+ enter() {
+ runInContext("onPaletteKeydown({key: 'Enter', preventDefault(){}})", context);
+ },
+ input(value: string) {
+ context.inputValue = value;
+ runInContext("onQueryInput({currentTarget:{value:inputValue}})", context);
+ },
+ fire() {
+ const fn = timer;
+ timer = undefined;
+ fn?.();
+ },
+ close() {
+ runInContext("closeChatSearch()", context);
+ },
+ open() {
+ runInContext("openChatSearch()", context);
+ },
+ };
+}
+
+for (const outcome of ["resolve", "reject"] as const) {
+ test(`typing invalidates an old ${outcome} before the next debounce fires`, async () => {
+ const h = harness();
+ h.input("old");
+ h.fire();
+ h.input("document");
+ assert.equal(h.requests[0]!.signal.aborted, true);
+ if (outcome === "resolve") h.requests[0]!.resolve({ hits: [{ sessionId: "old" }] });
+ else h.requests[0]!.reject(new Error("timeout"));
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(h.state.hits.length, 0);
+ assert.equal(h.state.failed, false);
+ assert.equal(h.state.loading, true);
+ h.fire();
+ h.requests[1]!.resolve({ hits: [{ sessionId: "document" }] });
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.deepEqual(h.state.hits, [{ sessionId: "document" }]);
+ assert.equal(h.state.loading, false);
+ });
+}
+
+test("new input clears selectable stale hits and previous errors", () => {
+ const h = harness();
+ h.state.hits = [{ sessionId: "old" }];
+ h.state.failed = true;
+ h.input("document");
+ assert.equal(h.state.hits.length, 0);
+ assert.equal(h.state.failed, false);
+});
+
+test("closing and reopening cannot accept a previous request", async () => {
+ const h = harness();
+ h.input("old");
+ h.fire();
+ h.close();
+ h.open();
+ h.requests[0]!.resolve({ hits: [{ sessionId: "old" }] });
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(h.state.hits.length, 0);
+ assert.equal(h.state.query, "");
+});
+
+test("clearing the query cancels pending and in-flight searches", async () => {
+ const h = harness();
+ h.input("old");
+ h.fire();
+ h.input("document");
+ h.input("");
+ h.fire();
+ assert.equal(h.requests.length, 1);
+ h.requests[0]!.resolve({ hits: [{ sessionId: "old" }] });
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(h.state.hits.length, 0);
+ assert.equal(h.state.loading, false);
+});
+
+test("Enter while searching does not start an unintended agent conversation", () => {
+ const h = harness();
+ h.input("document");
+ h.enter();
+ assert.equal(h.asks(), 0);
+});
diff --git a/scripts/backfill-search-index.ts b/scripts/backfill-search-index.ts
index aabe06696..e22a51362 100644
--- a/scripts/backfill-search-index.ts
+++ b/scripts/backfill-search-index.ts
@@ -1,4 +1,4 @@
-import { syncSearchIndex } from "../src/harness/tape-projection.ts";
+import { searchRowsFromEntries } from "../src/harness/tape-projection.ts";
import { openSessionStore, parseBackfillArgs, resolveSessions, runBackfill } from "./lib/backfill-runner.ts";
const { apply, only } = parseBackfillArgs();
@@ -12,16 +12,17 @@ await runBackfill({
sessions,
apply,
preview: async (session) => {
- const latest = await store.latestEntrySeq(session.id);
- if (latest < 0) return { action: "skip", reason: "empty", quiet: true };
- const lastSearchable = await store.lastSearchableEntrySeq(session.id);
- const watermark = await store.searchIndexCoverage(session.id);
- if (watermark >= lastSearchable) return { action: "skip", reason: "covered", quiet: true };
- return { action: "work", detail: `seq ${watermark + 1}..${lastSearchable}` };
+ const missing = await store.missingSearchEntries(session.id);
+ if (!missing) return { action: "skip", reason: "covered", quiet: true };
+ return { action: "work", detail: `${missing} missing messages` };
},
- applyStep: async (_session, lease) => {
- const sync = await syncSearchIndex(store, lease);
- if (!sync.servable) return { action: "skip", reason: "unservable projection (entries index keeps serving)" };
- return { action: "work", detail: `${sync.indexed} rows, covered through seq ${sync.coveredSeq}` };
+ applyStep: async (session, lease) => {
+ const rows = searchRowsFromEntries(await store.getEntries(session.id), -1);
+ for (let i = 0; i < rows.length; i += 500) {
+ await store.appendSearchEntries(lease, rows.slice(i, i + 500));
+ }
+ const missing = await store.missingSearchEntries(session.id);
+ if (missing) throw new Error(`${missing} searchable messages remain unindexed`);
+ return { action: "work", detail: "all searchable messages covered" };
},
});
diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.ts
index 00b85915f..afb30139d 100644
--- a/src/api/app-sessions.ts
+++ b/src/api/app-sessions.ts
@@ -91,6 +91,7 @@ export function createSessionMethods(
const {
sessionsForViewer,
sessionForViewer,
+ managedProjectMembership,
filesForViewer,
canUseContext,
currentResourceScopesForViewer,
@@ -425,11 +426,17 @@ export function createSessionMethods(
const capped = Math.max(1, Math.min(limit, 100));
const hits = await deps.sessions.searchEntries(principalId, query, capped);
if (!hits.length) return [];
- const visible = new Map((await sessionsForViewer(principalId)).map((s) => [s.id, s]));
+ const allowed = new Map(
+ await Promise.all(
+ [...new Set(hits.map((hit) => hit.scopeId))].map(
+ async (scope) => [scope, (await managedProjectMembership(scope, principalId)) !== false] as const,
+ ),
+ ),
+ );
const terms = searchTerms(query);
return hits.flatMap((hit) => {
- const session = visible.get(hit.sessionId);
- if (!session) return [];
+ if (!allowed.get(hit.scopeId)) return [];
+ const session = hit;
return [
{
sessionId: hit.sessionId,
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index bd6be7acf..46f53e555 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -115,7 +115,7 @@ import {
tapeNeedsInterruptHeal,
} from "../harness/tape-fold.ts";
import { openSessionEntry, searchSessionEntries } from "../sessions/history-search.ts";
-import { createTranscriptSource, syncSearchIndex } from "../harness/tape-projection.ts";
+import { createTranscriptSource } from "../harness/tape-projection.ts";
import { defaultPublishAudience } from "../resolution/publish-audience.ts";
import {
INBOX_DIR,
@@ -3018,7 +3018,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
);
}
latchedCoverageSeq = lastSeq;
- await syncSearchIndex(deps.sessions, lease).catch((e) => swallow("tape-search: sync", e));
};
if (
input.addressed &&
diff --git a/src/harness/tape-projection.ts b/src/harness/tape-projection.ts
index b807a4d6b..717f863c4 100644
--- a/src/harness/tape-projection.ts
+++ b/src/harness/tape-projection.ts
@@ -1,5 +1,5 @@
import type { EntryType, ScopeId, SessionEntry } from "../types.ts";
-import type { GetEntriesOptions, Lease, NewSearchEntry, SessionStore, TapeRecord } from "../sessions/session-store.ts";
+import type { GetEntriesOptions, NewSearchEntry, SessionStore, TapeRecord } from "../sessions/session-store.ts";
import { entryWithinTenure, TAPE_RENDER_VERSION } from "../sessions/session-store.ts";
import { entrySearchAuthor, entrySearchText, SEARCHABLE_ENTRY_TYPES } from "../sessions/entry-search.ts";
import { deliveryNoteManifest, legacyDeliveryNoteManifest } from "../core/attachments.ts";
@@ -497,12 +497,6 @@ export function createTranscriptSource(sessions: TranscriptStore): TranscriptSou
};
}
-export interface SearchIndexSync {
- servable: boolean;
- indexed: number;
- coveredSeq: number;
-}
-
export function searchRowsFromEntries(entries: readonly SessionEntry[], sinceSeq: number): NewSearchEntry[] {
return entries.flatMap((entry) => {
if (entry.seq <= sinceSeq || !SEARCHABLE_ENTRY_TYPES.has(entry.type)) return [];
@@ -520,32 +514,3 @@ export function searchRowsFromEntries(entries: readonly SessionEntry[], sinceSeq
];
});
}
-
-const SEARCH_SYNC_ROW_CAP = 500;
-
-export async function syncSearchIndex(
- sessions: Pick<SessionStore, "getTape" | "appendSearchEntries" | "searchIndexCoverage">,
- lease: Lease,
-): Promise<SearchIndexSync> {
- const watermark = await sessions.searchIndexCoverage(lease.sessionId);
- if (unservableTapes.has(lease.sessionId)) return { servable: false, indexed: 0, coveredSeq: watermark };
- let fullRows: TapeRecord[] | null = null;
- const projection = await (async () => {
- const suffix = await sessions.getTape(lease.sessionId, { limit: SEARCH_SYNC_ROW_CAP });
- if (suffix.length < SEARCH_SYNC_ROW_CAP) {
- fullRows = suffix;
- return projectTapeEntries(lease.sessionId, suffix);
- }
- const anchored = projectTapeEntries(lease.sessionId, suffix, { anchored: true });
- if (anchored && anchored.baseSeq <= watermark) return anchored;
- fullRows = await sessions.getTape(lease.sessionId);
- return projectTapeEntries(lease.sessionId, fullRows);
- })();
- if (!projection) {
- if (fullRows !== null && tapeHasRenderBlockers(fullRows)) unservableTapes.remember(lease.sessionId);
- return { servable: false, indexed: 0, coveredSeq: watermark };
- }
- const fresh = searchRowsFromEntries(projection.entries, watermark);
- if (fresh.length) await sessions.appendSearchEntries(lease, fresh);
- return { servable: true, indexed: fresh.length, coveredSeq: projection.coveredSeq };
-}
diff --git a/src/sessions/memory-session-store.ts b/src/sessions/memory-session-store.ts
index 76ae7f65e..1c31d70fa 100644
--- a/src/sessions/memory-session-store.ts
+++ b/src/sessions/memory-session-store.ts
@@ -248,6 +248,13 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
createdAt: now(),
};
log.push(full);
+ const text = SEARCHABLE_ENTRY_TYPES.has(full.type) ? entrySearchText(full.payload) : null;
+ if (text?.trim()) {
+ const index = searchIndex.get(full.sessionId) ?? [];
+ const author = entrySearchAuthor(full);
+ index.push({ seq: full.seq, type: full.type, text, createdAt: full.createdAt, ...(author ? { author } : {}) });
+ searchIndex.set(full.sessionId, index);
+ }
return full;
},
@@ -498,12 +505,18 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
const win = windows.get(sessionId)?.get(principalId);
if (!win) continue;
const indexed = searchIndex.get(sessionId) ?? [];
- const indexedSeqs = new Set(indexed.map((row) => row.seq));
+ const session = sessions.get(sessionId);
+ if (!session) continue;
for (const row of indexed) {
if (!entryWithinTenure(row, win)) continue;
if (!matchesSearchTerms(row.text, terms)) continue;
hits.push({
sessionId,
+ scopeId: session.scopeId,
+ ...((win.title ?? session.title) != null ? { title: win.title ?? session.title } : {}),
+ ...(session.channelName ? { channelName: session.channelName } : {}),
+ ...(session.surface ? { surface: session.surface } : {}),
+ ...(win.archived ? { archived: true } : {}),
seq: row.seq,
type: row.type,
...(row.author ? { author: row.author } : {}),
@@ -511,22 +524,6 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
createdAt: row.createdAt,
});
}
- for (const e of entries.get(sessionId) ?? []) {
- if (!SEARCHABLE_ENTRY_TYPES.has(e.type)) continue;
- if (indexedSeqs.has(e.seq)) continue;
- if (!entryWithinTenure(e, win)) continue;
- const text = entrySearchText(e.payload);
- if (!text || !matchesSearchTerms(text, terms)) continue;
- const author = entrySearchAuthor(e);
- hits.push({
- sessionId,
- seq: e.seq,
- type: e.type,
- ...(author ? { author } : {}),
- text,
- createdAt: e.createdAt,
- });
- }
}
hits.sort((a, b) => b.createdAt - a.createdAt || idDesc(a.sessionId, b.sessionId) || b.seq - a.seq);
return hits.slice(0, Math.max(1, Math.min(limit, 200)));
@@ -554,6 +551,14 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
return index?.length ? index[index.length - 1]!.seq : -1;
},
+ async missingSearchEntries(sessionId): Promise<number> {
+ const indexed = new Set((searchIndex.get(sessionId) ?? []).map((row) => row.seq));
+ return (entries.get(sessionId) ?? []).filter(
+ (entry) =>
+ SEARCHABLE_ENTRY_TYPES.has(entry.type) && entrySearchText(entry.payload)?.trim() && !indexed.has(entry.seq),
+ ).length;
+ },
+
async lastSearchableEntrySeq(sessionId): Promise<number> {
const log = entries.get(sessionId) ?? [];
for (let i = log.length - 1; i >= 0; i--) {
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts
index 12a113f4d..d8c85427a 100644
--- a/src/sessions/postgres-session-store.ts
+++ b/src/sessions/postgres-session-store.ts
@@ -463,6 +463,64 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
ON session_entry_search USING GIN (session_id, search_tsv)`,
],
},
+ {
+ id: "sessions/store/0014-search-write-through-v1",
+ statements: [
+ `SET LOCAL lock_timeout = '3s'`,
+ `CREATE OR REPLACE FUNCTION sync_session_entry_search() RETURNS trigger
+ LANGUAGE plpgsql AS $sync_session_entry_search$
+ DECLARE body text;
+ BEGIN
+ IF TG_OP = 'DELETE' THEN
+ DELETE FROM session_entry_search WHERE session_id = OLD.session_id AND seq = OLD.seq;
+ RETURN OLD;
+ END IF;
+ IF TG_OP = 'UPDATE' AND (OLD.session_id, OLD.seq) IS DISTINCT FROM (NEW.session_id, NEW.seq) THEN
+ DELETE FROM session_entry_search WHERE session_id = OLD.session_id AND seq = OLD.seq;
+ END IF;
+ IF NEW.type IN ('user', 'assistant', 'text') THEN
+ body := entry_search_text(NEW.payload);
+ END IF;
+ IF body IS NULL OR btrim(body) = '' THEN
+ DELETE FROM session_entry_search WHERE session_id = NEW.session_id AND seq = NEW.seq;
+ ELSE
+ INSERT INTO session_entry_search(session_id, seq, type, author, text, created_at)
+ VALUES (NEW.session_id, NEW.seq, NEW.type,
+ CASE WHEN NEW.type = 'user' THEN
+ (SELECT CASE WHEN json_typeof(j -> 'name') = 'string' THEN j ->> 'name' END
+ FROM (SELECT safe_json(replace(NEW.payload, '\\u0000', '')) AS j) _) END,
+ body, NEW.created_at)
+ ON CONFLICT (session_id, seq) DO UPDATE
+ SET type = EXCLUDED.type, author = EXCLUDED.author, text = EXCLUDED.text,
+ created_at = EXCLUDED.created_at;
+ END IF;
+ RETURN NEW;
+ END $sync_session_entry_search$`,
+ `CREATE TRIGGER session_entries_search_write_through
+ AFTER INSERT OR UPDATE OF payload, type, created_at, session_id, seq OR DELETE ON session_entries
+ FOR EACH ROW EXECUTE FUNCTION sync_session_entry_search()`,
+ ],
+ },
+ {
+ id: "sessions/store/0015-search-backfill-v1",
+ statements: [
+ `WITH missing AS MATERIALIZED (
+ SELECT e.session_id, e.seq, e.type,
+ CASE WHEN e.type = 'user' THEN
+ (SELECT CASE WHEN json_typeof(j -> 'name') = 'string' THEN j ->> 'name' END
+ FROM (SELECT safe_json(replace(e.payload, '\\u0000', '')) AS j) _) END AS author,
+ entry_search_text(e.payload) AS text, e.created_at
+ FROM session_entries e
+ WHERE e.type IN ('user', 'assistant', 'text')
+ AND NOT EXISTS (SELECT 1 FROM session_entry_search s WHERE s.session_id = e.session_id AND s.seq = e.seq)
+ FOR SHARE OF e
+ )
+ INSERT INTO session_entry_search(session_id, seq, type, author, text, created_at)
+ SELECT session_id, seq, type, author, text, created_at FROM missing
+ WHERE COALESCE(btrim(text), '') <> ''
+ ON CONFLICT (session_id, seq) DO NOTHING`,
+ ],
+ },
],
[
{
@@ -1093,32 +1151,20 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
const ts = tsPrefixQuery(query);
if (!ts) return [];
const rows = await q(
- `SELECT h.session_id, h.seq, h.type, h.author, h.text, h.created_at
- FROM participants p
- CROSS JOIN LATERAL
- ((SELECT s.session_id, s.seq, s.type, s.author, s.text, s.created_at
- FROM session_entry_search s
- WHERE s.session_id = p.session_id
- AND s.search_tsv @@ to_tsquery('simple', $2)
- AND ${withinParticipantWindow("s", "p")}
- ORDER BY s.created_at DESC, s.seq DESC
- LIMIT $3)
- UNION ALL
- (SELECT e.session_id, e.seq, e.type,
- (SELECT CASE WHEN e.type = 'user' AND json_typeof(j -> 'name') = 'string' THEN j ->> 'name' END
- FROM (SELECT safe_json(replace(e.payload, '\\u0000', '')) AS j) _) AS author,
- entry_search_text(e.payload) AS text,
- e.created_at
- FROM session_entries e
- WHERE e.session_id = p.session_id
- AND e.type IN ('user', 'assistant', 'text')
- AND e.search_tsv @@ to_tsquery('simple', $2)
- AND ${withinParticipantWindow("e", "p")}
- AND NOT EXISTS (SELECT 1 FROM session_entry_search x
- WHERE x.session_id = e.session_id AND x.seq = e.seq)
- ORDER BY e.created_at DESC, e.seq DESC
- LIMIT $3)) h
- WHERE p.principal_id = $1
+ `WITH viewer AS MATERIALIZED (
+ SELECT session_id, valid_from_seq, valid_from, valid_to_seq, valid_to, title, archived
+ FROM participants WHERE principal_id = $1
+ ), candidates AS MATERIALIZED (
+ SELECT session_id, seq, type, author, text, created_at
+ FROM session_entry_search
+ WHERE session_id = ANY(ARRAY(SELECT session_id FROM viewer))
+ AND search_tsv @@ to_tsquery('simple', $2)
+ )
+ SELECT h.*, s.scope_id, COALESCE(p.title, s.title) AS title, s.channel_name, s.surface, p.archived
+ FROM candidates h
+ JOIN viewer p ON p.session_id = h.session_id
+ JOIN sessions s ON s.id = h.session_id
+ WHERE ${withinParticipantWindow("h", "p")}
ORDER BY h.created_at DESC, h.session_id, h.seq DESC
LIMIT $3`,
[principalId, ts, Math.max(1, Math.min(limit, 200))],
@@ -1130,6 +1176,11 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
return [
{
sessionId: r.session_id as string,
+ scopeId: r.scope_id as ScopeId,
+ ...(r.title != null ? { title: r.title as string } : {}),
+ ...(r.channel_name ? { channelName: r.channel_name as string } : {}),
+ ...(r.surface ? { surface: r.surface as string } : {}),
+ ...(r.archived ? { archived: true } : {}),
seq: Number(r.seq),
type: r.type as EntrySearchHit["type"],
...(r.author ? { author: r.author as string } : {}),
@@ -1167,6 +1218,19 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
return Number(rows[0]?.n ?? -1);
},
+ async missingSearchEntries(sessionId): Promise<number> {
+ const rows = await q(
+ `WITH missing AS MATERIALIZED (
+ SELECT entry_search_text(e.payload) AS text FROM session_entries e
+ WHERE e.session_id = $1 AND e.type IN ('user', 'assistant', 'text')
+ AND NOT EXISTS (SELECT 1 FROM session_entry_search s WHERE s.session_id = e.session_id AND s.seq = e.seq)
+ )
+ SELECT COUNT(*) AS n FROM missing WHERE COALESCE(btrim(text), '') <> ''`,
+ [sessionId],
+ );
+ return Number(rows[0]?.n ?? 0);
+ },
+
async lastSearchableEntrySeq(sessionId): Promise<number> {
const rows = await q(
`SELECT COALESCE(MAX(seq), -1) AS n FROM session_entries
diff --git a/src/sessions/session-store.ts b/src/sessions/session-store.ts
index 512b44e60..49260b223 100644
--- a/src/sessions/session-store.ts
+++ b/src/sessions/session-store.ts
@@ -473,7 +473,7 @@ export interface DistinctScope {
channelName?: string;
}
-export interface EntrySearchHit {
+export interface EntrySearchHit extends Pick<Session, "scopeId" | "title" | "channelName" | "surface" | "archived"> {
sessionId: string;
seq: number;
type: EntryType;
@@ -704,6 +704,7 @@ export interface SessionStore {
appendSearchEntries(lease: Lease, rows: readonly NewSearchEntry[]): Promise<void>;
searchIndexCoverage(sessionId: string): Promise<number>;
+ missingSearchEntries(sessionId: string): Promise<number>;
lastSearchableEntrySeq(sessionId: string): Promise<number>;
scanAll(): Promise<Session[]>;
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index 38bb068c3..59a8901a0 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -1,6 +1,7 @@
import { test, before } from "node:test";
import { randomUUID } from "node:crypto";
import assert from "node:assert/strict";
+import { migrateRegisteredPgSchemas, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
import { PARALLEL_EXCEPTION_QUERY } from "../src/deployment/postdeploy-smoke.ts";
import {
backfillSessionOriginBatch,
@@ -1754,101 +1755,89 @@ test("pg search: full-text over entries with prefix match, window ACL, and type
assert.deepEqual(await s.searchEntries("USRCH", "memo missing"), [], "every term must match");
});
-test(
- "pg search: tape-index rows serve hits, entries fill the gaps, tool results stay unfindable",
- { skip },
- async () => {
- const s = createPostgresSessionStore(URL!);
- const scope = scopeId("personal", "UIDX");
- const sess = await s.getOrCreateByThread("srch-idx-1", "dm", scope);
- await s.addParticipant(sess.id, "UIDX", undefined, { includeHistory: true });
- const lease = (await s.acquireLease(sess.id)).lease!;
- const user = await s.append(lease, {
- type: "user",
- payload: { text: "please rotate the deploy key", name: "alex" },
- scopeLabel: scope,
- });
- await s.append(lease, {
- type: "tool_result",
- payload: { tool: "execute", callId: "c1", isError: false, result: "secret broker value QQ17" },
- scopeLabel: scope,
- });
- const reply = await s.append(lease, {
- type: "assistant",
- payload: { text: "Rotated the deploy key." },
- scopeLabel: scope,
- });
+test("pg search: message writes populate the index and tool results stay unfindable", { skip }, async () => {
+ const s = createPostgresSessionStore(URL!);
+ const scope = scopeId("personal", "UIDX");
+ const sess = await s.getOrCreateByThread("srch-idx-1", "dm", scope);
+ await s.addParticipant(sess.id, "UIDX", undefined, { includeHistory: true });
+ const lease = (await s.acquireLease(sess.id)).lease!;
+ const user = await s.append(lease, {
+ type: "user",
+ payload: { text: "please rotate the deploy key", name: "alex" },
+ scopeLabel: scope,
+ });
+ await s.append(lease, {
+ type: "tool_result",
+ payload: { tool: "execute", callId: "c1", isError: false, result: "secret broker value QQ17" },
+ scopeLabel: scope,
+ });
+ const reply = await s.append(lease, {
+ type: "assistant",
+ payload: { text: "Rotated the deploy key." },
+ scopeLabel: scope,
+ });
- const before = await s.searchEntries("UIDX", "deploy key");
- assert.equal(before.length, 2, "entries index serves an un-backfilled session");
+ const before = await s.searchEntries("UIDX", "deploy key");
+ assert.equal(before.length, 2, "message writes serve an indexed session");
- await s.appendSearchEntries(lease, [
- { seq: user.seq, type: "user", author: "alex", text: "please rotate the deploy key", createdAt: user.createdAt },
- { seq: reply.seq, type: "assistant", text: "Rotated the deploy key.", createdAt: reply.createdAt },
- ]);
- assert.equal(await s.searchIndexCoverage(sess.id), reply.seq);
- assert.deepEqual(
- await s.searchEntries("UIDX", "deploy key"),
- before,
- "tape-index hits match the entries-index hits",
- );
+ await s.appendSearchEntries(lease, [
+ { seq: user.seq, type: "user", author: "alex", text: "please rotate the deploy key", createdAt: user.createdAt },
+ { seq: reply.seq, type: "assistant", text: "Rotated the deploy key.", createdAt: reply.createdAt },
+ ]);
+ assert.equal(await s.searchIndexCoverage(sess.id), reply.seq);
+ assert.deepEqual(await s.searchEntries("UIDX", "deploy key"), before, "tape-index hits match the entries-index hits");
- await s.append(lease, { type: "user", payload: { text: "also rotate the staging deploy key" }, scopeLabel: scope });
- assert.equal(
- (await s.searchEntries("UIDX", "deploy key")).length,
- 3,
- "a not-yet-indexed entry still hits via entries",
- );
+ await s.append(lease, { type: "user", payload: { text: "also rotate the staging deploy key" }, scopeLabel: scope });
+ assert.equal((await s.searchEntries("UIDX", "deploy key")).length, 3, "a new message is indexed immediately");
- assert.deepEqual(
- await s.searchEntries("UIDX", "secret broker"),
- [],
- "tool results are unfindable in both branches",
- );
+ assert.deepEqual(await s.searchEntries("UIDX", "secret broker"), [], "tool results are unfindable");
- await s.appendSearchEntries(lease, [
- { seq: user.seq, type: "user", text: "replacement text is ignored", createdAt: user.createdAt },
- ]);
- assert.equal((await s.searchEntries("UIDX", "deploy key")).length, 3, "re-appending an indexed seq is a no-op");
+ await s.appendSearchEntries(lease, [
+ { seq: user.seq, type: "user", text: "replacement text is ignored", createdAt: user.createdAt },
+ ]);
+ assert.equal((await s.searchEntries("UIDX", "deploy key")).length, 3, "re-appending an indexed seq is a no-op");
- await s.addParticipant(sess.id, "ULATE2");
- const post = await s.append(lease, { type: "user", payload: { text: "deploy key postscript" }, scopeLabel: scope });
- await s.appendSearchEntries(lease, [
- { seq: post.seq, type: "user", text: "deploy key postscript", createdAt: post.createdAt },
- ]);
- const late = await s.searchEntries("ULATE2", "deploy key");
- assert.equal(late.length, 1, "a latecomer only searches index rows inside their window");
- assert.equal(late[0]!.text, "deploy key postscript");
+ await s.addParticipant(sess.id, "ULATE2");
+ const post = await s.append(lease, { type: "user", payload: { text: "deploy key postscript" }, scopeLabel: scope });
+ await s.appendSearchEntries(lease, [
+ { seq: post.seq, type: "user", text: "deploy key postscript", createdAt: post.createdAt },
+ ]);
+ const late = await s.searchEntries("ULATE2", "deploy key");
+ assert.equal(late.length, 1, "a latecomer only searches index rows inside their window");
+ assert.equal(late[0]!.text, "deploy key postscript");
- const nul = await s.append(lease, { type: "user", payload: { text: "nulriddled deploy key" }, scopeLabel: scope });
- await s.appendSearchEntries(lease, [
- {
- seq: nul.seq,
- type: "user",
- author: "e\u0000ve",
- text: "nul\u0000riddled deploy key",
- createdAt: nul.createdAt,
- },
- ]);
- const nulHits = await s.searchEntries("UIDX", "nulriddled");
- assert.equal(nulHits.length, 1, "a NUL byte in the text never wedges the index write");
- assert.equal(nulHits[0]!.author, "eve");
+ const nul = await s.append(lease, {
+ type: "user",
+ payload: { text: "nul\u0000riddled deploy key", name: "e\u0000ve" },
+ scopeLabel: scope,
+ });
+ await s.appendSearchEntries(lease, [
+ {
+ seq: nul.seq,
+ type: "user",
+ author: "e\u0000ve",
+ text: "nul\u0000riddled deploy key",
+ createdAt: nul.createdAt,
+ },
+ ]);
+ const nulHits = await s.searchEntries("UIDX", "nulriddled");
+ assert.equal(nulHits.length, 1, "a NUL byte in the text never wedges the index write");
+ assert.equal(nulHits[0]!.author, "eve");
- await s.append(lease, {
- type: "tool_result",
- payload: { tool: "execute", callId: "c2", isError: false, result: "trailing tool output" },
- scopeLabel: scope,
- });
- const emptyReply = await s.append(lease, { type: "assistant", payload: { text: " " }, scopeLabel: scope });
- assert.equal(
- await s.lastSearchableEntrySeq(sess.id),
- nul.seq,
- "convergence tracks the last searchable entry, not trailing tool output or blank replies",
- );
- assert.ok(emptyReply.seq > nul.seq);
- await s.releaseLease(lease);
- },
-);
+ await s.append(lease, {
+ type: "tool_result",
+ payload: { tool: "execute", callId: "c2", isError: false, result: "trailing tool output" },
+ scopeLabel: scope,
+ });
+ const emptyReply = await s.append(lease, { type: "assistant", payload: { text: " " }, scopeLabel: scope });
+ assert.equal(
+ await s.lastSearchableEntrySeq(sess.id),
+ nul.seq,
+ "convergence tracks the last searchable entry, not trailing tool output or blank replies",
+ );
+ assert.ok(emptyReply.seq > nul.seq);
+ await s.releaseLease(lease);
+});
test(
"pg deleteSessionIfEmpty: an expired lease forfeits, and the stale holder cannot orphan entries",
@@ -1951,3 +1940,123 @@ test("pg deleteSessionIfEmpty: a held lease or landed entries refuse the discard
assert.equal(Number(orphans.rows[0].e), 0, "no orphaned entries survive the discard");
assert.equal(Number(orphans.rows[0].l), 0, "no orphaned lease survives the discard");
});
+
+test("pg search: globally limits indexed hits across sessions", { skip }, async () => {
+ const store = createPostgresSessionStore(URL!, { now: () => 5000 });
+ const scope = scopeId("personal", "ULIMITSEARCH");
+ const expected: Array<{ sessionId: string; seq: number }> = [];
+ for (let i = 0; i < 3; i++) {
+ const session = await store.getOrCreateByThread(`search-global-limit-${i}`, "dm", scope);
+ await store.addParticipant(session.id, "ULIMITSEARCH", undefined, { includeHistory: true });
+ const lease = (await store.acquireLease(session.id)).lease!;
+ for (let j = 0; j < 4; j++) {
+ const entry = await store.append(lease, { type: "user", payload: { text: "document limit" }, scopeLabel: scope });
+ expected.push({ sessionId: session.id, seq: entry.seq });
+ if (j % 2 === 0) {
+ await store.appendSearchEntries(lease, [
+ { seq: entry.seq, type: "user", text: "document limit", createdAt: entry.createdAt },
+ ]);
+ }
+ }
+ await store.releaseLease(lease);
+ }
+ expected.sort((a, b) => a.sessionId.localeCompare(b.sessionId) || b.seq - a.seq);
+ for (const limit of [1, 5, 12, 20]) {
+ const hits = await store.searchEntries("ULIMITSEARCH", "doc lim", limit);
+ assert.deepEqual(
+ hits.map(({ sessionId, seq }) => ({ sessionId, seq })),
+ expected.slice(0, limit),
+ );
+ }
+});
+
+test("pg search: writes are atomic and updates and deletes keep the index current", { skip }, async () => {
+ const store = createPostgresSessionStore(URL!);
+ const session = await store.getOrCreateByThread("search-atomic", "dm", scopeId("personal", "UATOMIC"));
+ await store.addParticipant(session.id, "UATOMIC", undefined, { includeHistory: true });
+ const pg = (await import("pg")).default;
+ const pool = new pg.Pool({ connectionString: URL });
+ const lease = (await store.acquireLease(session.id)).lease!;
+ try {
+ await pool.query(
+ "ALTER TABLE session_entry_search ADD CONSTRAINT search_test_failure CHECK (text <> 'rejectindex')",
+ );
+ await assert.rejects(
+ store.append(lease, { type: "user", payload: { text: "rejectindex" }, scopeLabel: session.scopeId }),
+ );
+ assert.equal(await store.latestEntrySeq(session.id), -1);
+ assert.equal(await store.missingSearchEntries(session.id), 0);
+ await pool.query("ALTER TABLE session_entry_search DROP CONSTRAINT search_test_failure");
+ const entry = await store.append(lease, {
+ type: "user",
+ payload: { text: "original document" },
+ scopeLabel: session.scopeId,
+ });
+ await pool.query("UPDATE session_entries SET payload=$3 WHERE session_id=$1 AND seq=$2", [
+ session.id,
+ entry.seq,
+ JSON.stringify({ text: "edited document", name: "Editor" }),
+ ]);
+ assert.deepEqual(await store.searchEntries("UATOMIC", "original"), []);
+ assert.equal((await store.searchEntries("UATOMIC", "edited"))[0]!.author, "Editor");
+ await pool.query("UPDATE session_entries SET type='tool_result' WHERE session_id=$1", [session.id]);
+ assert.deepEqual(await store.searchEntries("UATOMIC", "edited"), []);
+ await pool.query("UPDATE session_entries SET type='user' WHERE session_id=$1", [session.id]);
+ assert.equal((await store.searchEntries("UATOMIC", "edited")).length, 1);
+ await pool.query("DELETE FROM session_entry_search WHERE session_id=$1", [session.id]);
+ assert.equal(await store.missingSearchEntries(session.id), 1);
+ assert.deepEqual(await store.searchEntries("UATOMIC", "edited"), [], "search never falls back to the legacy table");
+ await pool.query("UPDATE session_entries SET payload=payload WHERE session_id=$1", [session.id]);
+ await pool.query("DELETE FROM session_entries WHERE session_id=$1", [session.id]);
+ assert.deepEqual(await store.searchEntries("UATOMIC", "edited"), []);
+ } finally {
+ await pool.query("ALTER TABLE session_entry_search DROP CONSTRAINT IF EXISTS search_test_failure");
+ await store.releaseLease(lease);
+ await pool.end();
+ }
+});
+
+test("pg search: migration fills holes below the watermark and covers legacy-only sessions", { skip }, async () => {
+ const store = createPostgresSessionStore(URL!);
+ const session = await store.getOrCreateByThread("search-migration", "dm", scopeId("personal", "UMIGSEARCH"));
+ await store.addParticipant(session.id, "UMIGSEARCH", undefined, { includeHistory: true });
+ await store.updateParticipantView(session.id, "UMIGSEARCH", { title: "My documents", archived: true });
+ const pg = (await import("pg")).default;
+ const pool = new pg.Pool({ connectionString: URL });
+ try {
+ await pool.query("DROP TRIGGER session_entries_search_write_through ON session_entries");
+ await pool.query(
+ "DELETE FROM qm_schema_migrations WHERE id IN ('sessions/store/0014-search-write-through-v1','sessions/store/0015-search-backfill-v1')",
+ );
+ for (let seq = 0; seq < 3; seq++) {
+ await pool.query(
+ "INSERT INTO session_entries(session_id,seq,parent_seq,type,payload,scope_label,created_at) VALUES($1,$2,NULL,'user',$3,$4,$5)",
+ [session.id, seq, JSON.stringify({ text: `historical document ${seq}` }), session.scopeId, seq],
+ );
+ }
+ await pool.query(
+ "INSERT INTO session_entry_search(session_id,seq,type,text,created_at) VALUES($1,2,'user','historical document 2',2)",
+ [session.id],
+ );
+ assert.equal(await store.searchIndexCoverage(session.id), 2);
+ assert.equal(await store.missingSearchEntries(session.id), 2);
+ const ids = registeredPgMigrations(URL!).map((migration) => migration.id);
+ assert.ok(
+ ids.indexOf("sessions/store/0014-search-write-through-v1") <
+ ids.indexOf("sessions/store/0015-search-backfill-v1"),
+ );
+ await migrateRegisteredPgSchemas(URL!);
+ const migrated = store;
+ assert.equal(await migrated.missingSearchEntries(session.id), 0);
+ const hits = await migrated.searchEntries("UMIGSEARCH", "historical document");
+ assert.equal(hits.length, 3);
+ assert.ok(hits.every((hit) => hit.title === "My documents" && hit.archived));
+ await pool.query(
+ "INSERT INTO session_entries(session_id,seq,parent_seq,type,payload,scope_label,created_at) VALUES($1,3,NULL,'user',$2,$3,3)",
+ [session.id, JSON.stringify({ text: "old writer document" }), session.scopeId],
+ );
+ assert.equal((await migrated.searchEntries("UMIGSEARCH", "old writer")).length, 1);
+ } finally {
+ await pool.end();
+ }
+});
diff --git a/test/search-index.test.ts b/test/search-index.test.ts
index 2fee7303f..9b35bce34 100644
--- a/test/search-index.test.ts
+++ b/test/search-index.test.ts
@@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
-import { searchRowsFromEntries, syncSearchIndex } from "../src/harness/tape-projection.ts";
+import { searchRowsFromEntries } from "../src/harness/tape-projection.ts";
import { TAPE_RENDER_VERSION, type Lease, type SessionStore } from "../src/sessions/session-store.ts";
import type { ScopeId, Session, SessionEntry } from "../src/types.ts";
@@ -117,12 +117,10 @@ async function secretTurn(sim: Sim): Promise<void> {
});
}
-test("turn-end sync indexes only conversational text — tool results and thinking stay unsearchable", async () => {
+test("message writes index only conversational text — tool results and thinking stay unsearchable", async () => {
const sim = await simSession();
await secretTurn(sim);
- const sync = await syncSearchIndex(sim.store, sim.lease);
- assert.ok(sync.servable);
- assert.ok(sync.indexed >= 2);
+ assert.equal(await sim.store.missingSearchEntries(sim.session.id), 0);
assert.deepEqual(await sim.store.searchEntries(VIEWER, TOOL_SECRET), []);
assert.deepEqual(await sim.store.searchEntries(VIEWER, THINKING_SECRET), []);
assert.deepEqual(await sim.store.searchEntries(VIEWER, "brokered credential"), []);
@@ -136,7 +134,6 @@ test("turn-end sync indexes only conversational text — tool results and thinki
test("raw tape payloads are never indexed — the env footer stays unsearchable", async () => {
const sim = await simSession();
await secretTurn(sim);
- await syncSearchIndex(sim.store, sim.lease);
assert.deepEqual(await sim.store.searchEntries(VIEWER, FOOTER_SECRET), []);
});
@@ -171,81 +168,11 @@ test("searchRowsFromEntries drops every non-conversational entry type", () => {
assert.ok(searchRowsFromEntries(entries, -1).every((r) => !r.text.includes(TOOL_SECRET)));
});
-test("tape-index hits match the entries-index hits exactly", async () => {
+test("messages are searchable before a turn completes or tape can be projected", async () => {
const sim = await simSession();
- await simTurn(sim, { input: "where is the runbook", author: "Alex", reply: "The runbook lives in the wiki." });
- await simTurn(sim, { input: "thanks a lot", reply: "Any time." });
- const before = await sim.store.searchEntries(VIEWER, "runbook");
- assert.equal(before.length, 2);
- const sync = await syncSearchIndex(sim.store, sim.lease);
- assert.ok(sync.servable);
- assert.equal(await sim.store.searchIndexCoverage(sim.session.id), sync.coveredSeq);
- const after = await sim.store.searchEntries(VIEWER, "runbook");
- assert.deepEqual(after, before);
-});
-
-test("sessions not yet backfilled keep serving search from the entries index", async () => {
- const sim = await simSession();
- await simTurn(sim, { input: "unbackfilled question", reply: "unbackfilled answer" });
- assert.equal(await sim.store.searchIndexCoverage(sim.session.id), -1);
- const hits = await sim.store.searchEntries(VIEWER, "unbackfilled");
- assert.equal(hits.length, 2);
-});
-
-test("syncSearchIndex is idempotent and advances the watermark", async () => {
- const sim = await simSession();
- await simTurn(sim, { input: "first question", reply: "first answer" });
- const first = await syncSearchIndex(sim.store, sim.lease);
- assert.ok(first.indexed > 0);
- const again = await syncSearchIndex(sim.store, sim.lease);
- assert.equal(again.indexed, 0);
- await simTurn(sim, { input: "second question", reply: "second answer" });
- const next = await syncSearchIndex(sim.store, sim.lease);
- assert.ok(next.indexed > 0);
- assert.equal(await sim.store.searchIndexCoverage(sim.session.id), next.coveredSeq);
-});
-
-test("an unservable projection leaves the index untouched and reports it", async () => {
- const sim = await simSession();
- await sim.store.append(sim.lease, { type: "user", payload: { text: "legacy body" }, scopeLabel: scope });
- await sim.store.appendTape(sim.lease, {
- kind: "context_event",
- payload: { event: "legacy_import", messages: [{ role: "user", content: "legacy body" }], scopes: [scope] },
- scopeLabel: scope,
- coversEntrySeq: 0,
- });
- const sync = await syncSearchIndex(sim.store, sim.lease);
- assert.equal(sync.servable, false);
- assert.equal(await sim.store.searchIndexCoverage(sim.session.id), -1);
- assert.equal((await sim.store.searchEntries(VIEWER, "legacy body")).length, 1);
-});
-
-test("a permanently unservable session is memoized: the next sync never re-reads the tape", async () => {
- const sim = await simSession();
- await sim.store.append(sim.lease, { type: "user", payload: { text: "legacy fork body" }, scopeLabel: scope });
- await sim.store.appendTape(sim.lease, {
- kind: "context_event",
- payload: { event: "legacy_import", messages: [{ role: "user", content: "legacy fork body" }], scopes: [scope] },
- scopeLabel: scope,
- coversEntrySeq: 0,
- });
- const first = await syncSearchIndex(sim.store, sim.lease);
- assert.equal(first.servable, false);
- let tapeReads = 0;
- const spy = {
- ...sim.store,
- getTape: (sessionId: string, opts?: { limit?: number; sinceSeq?: number }) => {
- tapeReads++;
- return sim.store.getTape(sessionId, opts);
- },
- } as SessionStore;
- const second = await syncSearchIndex(spy, sim.lease);
- assert.equal(second.servable, false);
- assert.equal(tapeReads, 0, "the memoized unservable session is never re-read");
- const other = await simSession("dm:search-index-memo-other");
- await simTurn(other, { input: "unrelated question", reply: "unrelated answer" });
- const otherSync = await syncSearchIndex(other.store, other.lease);
- assert.ok(otherSync.servable, "the memo is per session, not global");
+ await sim.store.append(sim.lease, { type: "user", payload: { text: "legacy document" }, scopeLabel: scope });
+ assert.equal((await sim.store.searchEntries(VIEWER, "legacy document")).length, 1);
+ assert.equal(await sim.store.missingSearchEntries(sim.session.id), 0);
});
test("tenure windows filter tape-index hits the same as entries-index hits", async () => {
@@ -253,36 +180,11 @@ test("tenure windows filter tape-index hits the same as entries-index hits", asy
await simTurn(sim, { input: "early private question", reply: "early answer" });
await sim.store.addParticipant(sim.session.id, "latecomer");
await simTurn(sim, { input: "late shared question", reply: "late answer" });
- await syncSearchIndex(sim.store, sim.lease);
assert.deepEqual(await sim.store.searchEntries("latecomer", "early private"), []);
assert.equal((await sim.store.searchEntries("latecomer", "late shared")).length, 1);
assert.equal((await sim.store.searchEntries(VIEWER, "early private")).length, 1);
});
-test("a settled session's turn-end sync reads a bounded tape suffix, not the whole tape", async () => {
- const sim = await simSession();
- for (let i = 0; i < 100; i++) await simTurn(sim, { input: `question ${i}`, reply: `answer ${i}` });
- await syncSearchIndex(sim.store, sim.lease);
- await simTurn(sim, { input: "one more question", reply: "one more answer" });
- const calls: Array<{ limit?: number } | undefined> = [];
- const spy = {
- ...sim.store,
- getTape: (sessionId: string, opts?: { limit?: number; sinceSeq?: number }) => {
- calls.push(opts);
- return sim.store.getTape(sessionId, opts);
- },
- } as SessionStore;
- const sync = await syncSearchIndex(spy, sim.lease);
- assert.ok(sync.servable);
- assert.equal(sync.indexed, 2);
- assert.ok(calls.length > 0);
- assert.ok(
- calls.every((c) => c?.limit !== undefined),
- "the settled hot path never reads the tape unbounded",
- );
- assert.equal((await sim.store.searchEntries(VIEWER, "one more question")).length, 1);
-});
-
test("lastSearchableEntrySeq skips trailing tool output and empty replies", async () => {
const sim = await simSession();
await simTurn(sim, { input: "find the report", reply: "Found it." });
@@ -334,9 +236,7 @@ test("a foreign-harness turn indexes its trigger and reply from the coarse proje
scopeLabel: scope,
entrySeq: finalEntry.seq,
});
- const sync = await syncSearchIndex(sim.store, sim.lease);
- assert.ok(sync.servable);
- assert.equal(sync.indexed, 2);
+ assert.equal(await sim.store.missingSearchEntries(sim.session.id), 0);
assert.deepEqual(await sim.store.searchEntries(VIEWER, TOOL_SECRET), []);
assert.equal((await sim.store.searchEntries(VIEWER, "codex please")).length, 1);
assert.equal((await sim.store.searchEntries(VIEWER, "summary")).length, 1);
diff --git a/test/session-metadata-authz.test.ts b/test/session-metadata-authz.test.ts
index 8e7b3a502..7fd288480 100644
--- a/test/session-metadata-authz.test.ts
+++ b/test/session-metadata-authz.test.ts
@@ -66,7 +66,9 @@ test("the single-session read applies the managed-project check the session list
"the list agrees while membership stands",
);
+ assert.ok((await built.app.searchSessions("member", "push")).some((hit) => hit.sessionId === sessionId));
assert.equal((await built.app.removeProjectMember(project.id, "owner", "member")).status, "ok");
+ assert.deepEqual(await built.app.searchSessions("member", "push"), []);
assert.ok(
await built.sessions.getForParticipant(sessionId, "member"),
"the participant row outlives the project membership, so the project check is what must reject",
diff --git a/test/session-search.test.ts b/test/session-search.test.ts
index 88fb4202a..5e568b02b 100644
--- a/test/session-search.test.ts
+++ b/test/session-search.test.ts
@@ -106,6 +106,12 @@ test("app.searchSessions decorates hits with session metadata and enforces visib
await sessions.updateTitle(id, "Trip planning");
await seed(sessions, "web:U2:other", "U2", ["zanzibar for U2 only"]);
+ sessions.listByParticipant = async () => {
+ throw new Error("search must not list every session");
+ };
+ sessions.getForParticipant = async () => {
+ throw new Error("search must not read legacy message activity");
+ };
const hits = await app.searchSessions("U1", "zanzibar");
assert.equal(hits.length, 2, "only U1's own conversation is searched");
assert.ok(hits.every((h) => h.sessionId === id));
diff --git a/test/tape-retirement.test.ts b/test/tape-retirement.test.ts
index ee0bd2159..5cdafd28b 100644
--- a/test/tape-retirement.test.ts
+++ b/test/tape-retirement.test.ts
@@ -6,7 +6,6 @@ import {
projectTapeEntries,
renderableTapeSlice,
RENDER_IMPORT_EVENT,
- syncSearchIndex,
} from "../src/harness/tape-projection.ts";
import { foldTape } from "../src/harness/tape-fold.ts";
import { TAPE_RENDER_VERSION, type Lease, type SessionStore, type TapeRecord } from "../src/sessions/session-store.ts";
@@ -378,13 +377,10 @@ test("an uncovered tape gets a fold import before the render stamp", async () =>
assert.deepEqual(projection!.entries, entries);
});
-test("the search sync advances across a render import instead of wedging", async () => {
+test("search remains complete across a render import", async () => {
const sim = await preCutoverSession();
await importSession(sim);
- const sync = await syncSearchIndex(sim.store, sim.lease);
- assert.equal(sync.servable, true);
- assert.ok(sync.indexed >= 2);
- assert.equal(sync.coveredSeq, (await sim.store.getEntries(sim.session.id)).length - 1);
+ assert.equal(await sim.store.missingSearchEntries(sim.session.id), 0);
});
test("a tainted uncovered session is refused without a coverage claim", async () => {