Multiplayer agent harness for work.
Rank semantic memory in Postgres with pgvector (#2323)
Regan Bell7c6220132660
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.ymlindex 424b9198b..7c472f14b 100644--- a/.github/workflows/cicd.yml+++ b/.github/workflows/cicd.yml@@ -268,7 +268,7 @@ jobs:runs-on: ubuntu-latestservices:postgres:- image: postgres:16+ image: pgvector/pgvector:0.8.1-pg16env:POSTGRES_PASSWORD: postgresPOSTGRES_DB: qmdiff --git a/cli/src/backends/docker.ts b/cli/src/backends/docker.tsindex e963a2b23..18c90c335 100644--- a/cli/src/backends/docker.ts+++ b/cli/src/backends/docker.ts@@ -259,7 +259,7 @@ function ensurePostgres(ctx: DockerCtx, dryRun: boolean): string {const url = (password: string): string => `postgres://postgres:${password}@pg:5432/qm`;if (dryRun) {- step(`Postgres: would run ${pgName} (image postgres:16, volume ${pgVolume(ctx)})`);+ step(`Postgres: would run ${pgName} (image pgvector/pgvector:0.8.1-pg16, volume ${pgVolume(ctx)})`);return url(readDeploymentState(ctx.config.orgId)?.pgPassword ?? "<generated>");}return withDeploymentLock(ctx.config.orgId, () => {@@ -306,7 +306,7 @@ function ensurePostgres(ctx: DockerCtx, dryRun: boolean): string {"POSTGRES_DB=qm","-v",`${pgVolume(ctx)}:/var/lib/postgresql/data`,- "postgres:16",+ "pgvector/pgvector:0.8.1-pg16",]);} finally {secretFile.cleanup();diff --git a/docs/memory-providers.md b/docs/memory-providers.mdindex 529ba6e7b..02c7eca7b 100644--- a/docs/memory-providers.md+++ b/docs/memory-providers.md@@ -5,18 +5,31 @@ QM keeps its built-in notebook memory unless `MEMORY_PROVIDER_CONFIG` defines a## Prompt memoryEach turn carries at most 6,000 characters of memory, selected across every notebook-the conversation may read, one heading per notebook. With an embedding model, facts-are ranked by similarity to the current message and recent dialogue; otherwise the-newest facts come first. Later turns add only newly selected facts.+the conversation may read, one heading per notebook. With PostgreSQL and an embedding+model, facts are ranked by exact cosine similarity to the current message and recent+dialogue; otherwise the newest facts come first. Later turns add only newly selected facts.Choose the model with the org setting `memory-embedding-model`(`{ "modelId": "gateway/<model>" }`, or `null` to turn ranking off).Gateway models with `mode=embedding` are listed automatically; a direct OpenAI-compatiblemodel is registered in the model registry with `"kind": "embedding"`. The provider-receives fact text and the query. Vectors are cached per notebook in the-`memory_vectors` map; if the provider fails or its vector width changes, recall falls-back to newest-first. Text search (`memory` search, `POST /v1/memory/search`) does not-use embeddings.+receives fact text during background indexing and the query during recall. PostgreSQL+stores typed vectors in `memory_fact_vectors`; recall ranks live, disclosed facts once per+scope in SQL and returns compact ranked matches. No approximate vector index is used.++Semantic recall requires the PostgreSQL `vector` extension. Its migration is registered+only after a configured embedding model resolves, not during an unconfigured startup.+If the application database role cannot install extensions, an administrator must install+`vector` first. File-backed installations use newest-first recall, even with a model selected.++A single background indexer embeds at most 64 missing facts and removes stale vectors in+bounded batches every 30 seconds. It also sends a short dimension probe every ten minutes so a provider's+vector-width change is detected even when its model identifier stays the same.++New or edited facts may wait for a background tick before becoming semantic matches.+Unindexed facts participate at the recency fallback score alongside indexed semantic+matches. If the provider/database fails, recall falls back to newest-first. Text search (`memory` search, `POST /v1/memory/search`) remains+immediate and does not use embeddings.## External routingdiff --git a/package.json b/package.jsonindex 4fe84e499..5f876208e 100644--- a/package.json+++ b/package.json@@ -48,7 +48,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/scripts/dev/lib/postgres.ts b/scripts/dev/lib/postgres.tsindex 9a7b1f3ed..efd2b8b03 100644--- a/scripts/dev/lib/postgres.ts+++ b/scripts/dev/lib/postgres.ts@@ -3,7 +3,8 @@ import type { Pool } from "pg";import { run } from "./proc.ts";import { sleep } from "./util.ts";-const POSTGRES_IMAGE = "postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777";+const POSTGRES_IMAGE =+ "pgvector/pgvector:0.8.1-pg16@sha256:33198da2828a14c30348d2ccb4750833d5ed9a44c88d840a0e523d7417120337";function docker(args: string[],diff --git a/src/memory/cross-scope.ts b/src/memory/cross-scope.tsindex 9c32b82aa..34d5aa152 100644--- a/src/memory/cross-scope.ts+++ b/src/memory/cross-scope.ts@@ -18,6 +18,7 @@ export async function recallAcrossScopes(labels: ReadonlyMap<string, string> = new Map(),): Promise<RecalledFact[]> {const budget = Math.max(0, Math.min(RECALL_MAX_CHARS, Math.floor(context.maxChars ?? RECALL_MAX_CHARS)));+ if (!budget) return [];const unique = [...new Set(scopes)];const batches = await Promise.all(unique.map(async (scope) => (await memory.recall(scope, context)).map((row) => ({ ...row, scope }))),diff --git a/src/memory/postgres-vector-store.ts b/src/memory/postgres-vector-store.tsnew file mode 100644index 000000000..d92cbde01--- /dev/null+++ b/src/memory/postgres-vector-store.ts@@ -0,0 +1,168 @@+import { createPgPool } from "../persistence/pg-pool.ts";+import { unitVector } from "../model/embeddings.ts";++export interface MemoryVector {+ scope: string;+ model: string;+ factId: string;+ hash: string;+ vector: number[];+}++interface MemoryVectorMatch {+ factId: string;+ score: number;+}++interface MemoryVectorSearch {+ scope: string;+ model: string;+ vector: number[];+ facts: readonly { id: string; hash: string; at: number }[];+ minSimilarity?: number;+}++export interface MemoryVectorStore {+ search(input: MemoryVectorSearch): Promise<MemoryVectorMatch[]>;+ put(rows: readonly MemoryVector[], signal?: AbortSignal): Promise<void>;+ missing(+ model: string,+ limit: number,+ dimensions?: number,+ signal?: AbortSignal,+ afterId?: string,+ ): Promise<Array<{ scope: string; factId: string; hash: string; text: string }>>;+ prune(model: string, limit: number, dimensions?: number, signal?: AbortSignal): Promise<number>;+ close(): Promise<void>;+}++const SCHEMA = [+ `CREATE TABLE IF NOT EXISTS memory_fact_vectors (+ scope_id TEXT NOT NULL,+ model TEXT NOT NULL,+ fact_id BIGINT NOT NULL,+ fact_hash TEXT NOT NULL,+ dimensions INTEGER NOT NULL CHECK (dimensions BETWEEN 1 AND 16000),+ embedding vector NOT NULL,+ PRIMARY KEY (scope_id, model, fact_id),+ CHECK (vector_dims(embedding) = dimensions)+ )`,+];++const FACT_HASH = "encode(sha256(convert_to(f.text, 'UTF8')), 'hex')";+const batchSize = (limit: number) => Math.max(1, Math.min(256, Math.floor(limit) || 1));++function vectorWidth(dimensions: number | undefined): number | null {+ if (dimensions === undefined) return null;+ if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 16000)+ throw new Error("Invalid memory embedding dimensions");+ return dimensions;+}++export function createPostgresVectorStore(connectionString: string): MemoryVectorStore {+ const db = createPgPool(connectionString, [+ { id: "memory/vectors/0001-extension", statements: ["CREATE EXTENSION IF NOT EXISTS vector"] },+ { id: "memory/vectors/0002", statements: SCHEMA },+ ]);+ return {+ async search(input) {+ if (!input.facts.length) return [];+ const vector = unitVector(input.vector);+ const minSimilarity = input.minSimilarity ?? 0.2;+ const rows = await db.q(+ `WITH scoped AS MATERIALIZED (+ SELECT wanted.id, wanted.hash, wanted.at, ${FACT_HASH} AS live_hash, v.fact_hash, v.embedding+ FROM unnest($4::bigint[], $5::text[], $6::bigint[]) AS wanted(id, hash, at)+ JOIN memory_facts f ON f.scope_id = $1 AND f.id = wanted.id AND f.deleted_at IS NULL+ LEFT JOIN memory_fact_vectors v ON v.scope_id = $1 AND v.model = $2 AND v.dimensions = $3+ AND v.fact_id = wanted.id+ ),+ ranked AS MATERIALIZED (+ SELECT id, hash, at, fact_hash IS DISTINCT FROM hash AS unindexed,+ CASE WHEN fact_hash = hash THEN embedding <=> $7::vector ELSE $9::double precision END AS distance+ FROM scoped WHERE hash = live_hash+ )+ SELECT id, distance FROM ranked WHERE unindexed OR distance <= $8 ORDER BY distance, at DESC, id`,+ [+ input.scope,+ input.model,+ vector.length,+ input.facts.map((fact) => fact.id),+ input.facts.map((fact) => fact.hash),+ input.facts.map((fact) => fact.at),+ JSON.stringify(vector),+ 1 - minSimilarity,+ 1 - Math.min(0.2, minSimilarity),+ ],+ { timeoutMs: 5000 },+ );+ return rows.map((row) => ({+ factId: String(row.id),+ score: 1 - Number(row.distance),+ }));+ },+ async put(rows, signal) {+ if (!rows.length) return;+ const latest = new Map(+ rows.map((row) => [+ JSON.stringify([row.scope, row.model, row.factId]),+ { ...row, vector: unitVector(row.vector) },+ ]),+ );+ await db.q(+ `WITH input AS (+ SELECT * FROM jsonb_to_recordset($1::jsonb) AS x(scope text, model text, "factId" bigint, hash text, vector jsonb)+ ),+ live AS MATERIALIZED (+ SELECT x.scope, x.model, x."factId", x.hash, x.vector+ FROM input x JOIN memory_facts f ON f.scope_id = x.scope AND f.id = x."factId"+ AND f.deleted_at IS NULL AND ${FACT_HASH} = x.hash+ ORDER BY f.id FOR SHARE OF f+ )+ INSERT INTO memory_fact_vectors(scope_id, model, fact_id, fact_hash, dimensions, embedding)+ SELECT scope, model, "factId", hash, jsonb_array_length(vector), vector::text::vector FROM live+ ON CONFLICT (scope_id, model, fact_id) DO UPDATE+ SET fact_hash = EXCLUDED.fact_hash, dimensions = EXCLUDED.dimensions, embedding = EXCLUDED.embedding`,+ [JSON.stringify([...latest.values()])],+ { signal, timeoutMs: 5000 },+ );+ },+ async missing(model, limit, dimensions, signal, afterId) {+ const rows = await db.q(+ `SELECT f.scope_id, f.id, f.text, ${FACT_HASH} AS hash+ FROM memory_facts f LEFT JOIN memory_fact_vectors v+ ON v.scope_id = f.scope_id AND v.fact_id = f.id AND v.model = $1+ AND v.fact_hash = ${FACT_HASH}+ AND ($3::integer IS NULL OR v.dimensions = $3)+ WHERE f.deleted_at IS NULL AND v.fact_id IS NULL AND ($4::bigint IS NULL OR f.id > $4)+ ORDER BY f.id LIMIT $2`,+ [model, batchSize(limit), vectorWidth(dimensions), afterId ?? null],+ { signal, timeoutMs: 5000 },+ );+ return rows.map((row) => ({+ scope: String(row.scope_id),+ factId: String(row.id),+ hash: String(row.hash),+ text: String(row.text),+ }));+ },+ async prune(model, limit, dimensions, signal) {+ const { rowCount } = await db.query(+ `DELETE FROM memory_fact_vectors v USING (+ SELECT s.scope_id, s.model, s.fact_id FROM memory_fact_vectors s+ WHERE s.model <> $1 OR ($3::integer IS NOT NULL AND s.dimensions <> $3) OR NOT EXISTS (+ SELECT 1 FROM memory_facts f+ WHERE f.scope_id = s.scope_id AND f.id = s.fact_id+ AND f.deleted_at IS NULL AND ${FACT_HASH} = s.fact_hash+ )+ LIMIT $2 FOR UPDATE OF s SKIP LOCKED+ ) stale+ WHERE v.scope_id = stale.scope_id AND v.model = stale.model AND v.fact_id = stale.fact_id`,+ [model, batchSize(limit), vectorWidth(dimensions)],+ { signal, timeoutMs: 30_000 },+ );+ return rowCount;+ },+ close: () => db.close(),+ };+}diff --git a/src/memory/semantic-memory.ts b/src/memory/semantic-memory.tsnew file mode 100644index 000000000..b9e7d700d--- /dev/null+++ b/src/memory/semantic-memory.ts@@ -0,0 +1,51 @@+import type { MemoryEmbedder } from "../model/embeddings.ts";+import type { LeaderLease } from "../persistence/leader-lease.ts";+import { reportFailure } from "../util/errors.ts";+import type { createSweeper } from "../util/sweeper.ts";+import type { MemoryService } from "./memory-service.ts";+import { createPostgresVectorStore, type MemoryVectorStore } from "./postgres-vector-store.ts";+import { createSemanticMemoryService } from "./semantic-recall.ts";+import { createMemoryVectorIndexer } from "./vector-indexer.ts";++export function createSemanticMemory(opts: {+ notebook: MemoryService;+ databaseUrl: string | undefined;+ embeddingModel: () => Promise<string | null | undefined>;+ resolveEmbedder: (id: string) => Promise<MemoryEmbedder>;+ leaderLease: LeaderLease;+ createSweeper: typeof createSweeper;+}) {+ const { databaseUrl } = opts;+ if (!databaseUrl)+ return {+ memory: opts.notebook,+ start() {},+ async stop() {},+ async close() {},+ };+ let store: MemoryVectorStore | undefined;+ const resolve = async () => {+ const id = await opts.embeddingModel();+ if (!id) return undefined;+ const embedder = await opts.resolveEmbedder(id);+ store ??= createPostgresVectorStore(databaseUrl);+ return { embedder, store };+ };+ const memory = createSemanticMemoryService(+ opts.notebook,+ async () => (await resolve())?.embedder,+ { search: (input) => store!.search(input) },+ { onError: (error) => reportFailure("memory semantic recall; using recent facts", error) },+ );+ const indexer = createMemoryVectorIndexer({ resolve, leaderLease: opts.leaderLease });+ const sweeper = opts.createSweeper(() => indexer.tick(), 30_000, { label: "memory-vector-index", immediate: true });+ return {+ memory,+ start: () => sweeper.start(),+ stop: () => sweeper.stop(),+ async close() {+ await store?.close();+ store = undefined;+ },+ };+}diff --git a/src/memory/semantic-recall.ts b/src/memory/semantic-recall.tsindex 61be9de63..454654875 100644--- a/src/memory/semantic-recall.ts+++ b/src/memory/semantic-recall.ts@@ -1,18 +1,9 @@-import type { DurableMap } from "../persistence/durable-map.ts";import type { SessionEntry } from "../types.ts";-import { createKeyedQueue } from "../util/async.ts";import type { MemoryService, MemoryRecallContext } from "./memory-service.ts";-import { factCandidate, factHash, recentCandidates, type MemoryCandidate, type MemoryFact } from "./facts.ts";-import type { MemoryEmbedder } from "../model/embeddings.ts";--const INDEX_WRITE_LANES = 4;--export interface MemoryVector {- scope: string;- model: string;- hash: string;- vector: number[];-}+import { factCandidate, factHash, recentCandidates, type MemoryCandidate } from "./facts.ts";+import { unitVector, type MemoryEmbedder } from "../model/embeddings.ts";+import type { MemoryVectorStore } from "./postgres-vector-store.ts";+import { disclosedMemory } from "./disclosure.ts";export function memoryRecallQuery(message: string, visibleEntries: readonly SessionEntry[]): string {const recent = visibleEntries@@ -31,109 +22,54 @@ export function memoryRecallQuery(message: string, visibleEntries: readonly Sessexport function createSemanticMemoryService(base: MemoryService,resolveEmbedder: () => Promise<MemoryEmbedder | undefined>,- index: Pick<DurableMap<MemoryVector>, "select" | "put" | "delete">,+ index?: Pick<MemoryVectorStore, "search">,opts: { minSimilarity?: number; onError?: (error: unknown) => void } = {},): MemoryService {- const queue = createKeyedQueue<string>();- const writeLanes = createKeyedQueue<number>();- let nextLane = 0;- const writeAll = <T>(items: readonly T[], write: (item: T) => Promise<void>): Promise<void[]> =>- Promise.all(items.map((item) => writeLanes(nextLane++ % INDEX_WRITE_LANES, () => write(item))));- const vectorId = ({ scope, model, hash }: Omit<MemoryVector, "vector">) => JSON.stringify([scope, model, hash]);- const cache = new Map<string, number[]>();- const remember = (model: string, hash: string, vector: number[]) => {- cache.delete(`${model} ${hash}`);- cache.set(`${model} ${hash}`, vector);- if (cache.size > 2_000) cache.delete(cache.keys().next().value!);- };- async function indexed(scope: string, facts: readonly MemoryFact[], signal: AbortSignal, embedder: MemoryEmbedder) {- return queue(scope, async () => {- signal.throwIfAborted();- const wanted = new Set(facts.map((fact) => factHash(fact.text)));- const vectors: Record<string, number[]> = {};- for (const hash of wanted) {- const vector = cache.get(`${embedder.id} ${hash}`);- if (vector) vectors[hash] = vector;- }- if (!wanted.size || Object.keys(vectors).length < wanted.size) {- const stale: MemoryVector[] = [];- for (const row of await index.select({ where: { field: "scope", anyOfFold: [scope] } })) {- if (row.scope !== scope) continue;- if (row.model !== embedder.id || !wanted.has(row.hash)) stale.push(row);- else {- vectors[row.hash] = row.vector;- remember(row.model, row.hash, row.vector);- }- }- await writeAll(stale, (row) => index.delete(vectorId(row)));- }- const missing = [...new Set(facts.map((fact) => fact.text))].filter((text) => !vectors[factHash(text)]);- for (let start = 0; start < missing.length; start += 64) {- const batch = missing.slice(start, start + 64);- const embedded = await embedder.embed(- batch.map((text) => text.slice(0, 8000)),- signal,- );- if (embedded.length !== batch.length) throw new Error("Missing memory embeddings");- const rows = batch.map((text, i) => ({- scope,- model: embedder.id,- hash: factHash(text),- vector: embedded[i]!,- }));- await writeAll(rows, (row) => index.put(vectorId(row), row));- for (const row of rows) {- vectors[row.hash] = row.vector;- remember(row.model, row.hash, row.vector);- }- }- return vectors;- });- }- const embedders = new WeakMap<MemoryRecallContext, Promise<MemoryEmbedder | undefined>>();- function embedderFor(context: MemoryRecallContext): Promise<MemoryEmbedder | undefined> {- let resolving = embedders.get(context);- if (!resolving) {- resolving = resolveEmbedder();- embedders.set(context, resolving);+ const queries = new WeakMap<MemoryRecallContext, Promise<{ model: string; vector: number[] } | undefined>>();+ function queryFor(context: MemoryRecallContext) {+ let pending = queries.get(context);+ if (!pending) {+ pending = (async () => {+ const signal = AbortSignal.timeout(8000);+ const embedder = await resolveEmbedder();+ if (!embedder) return undefined;+ const vectors = await embedder.embed([context.query!.trim().slice(0, 4000)], signal);+ signal.throwIfAborted();+ return { model: embedder.id, vector: unitVector(vectors[0]) };+ })();+ queries.set(context, pending);}- return resolving;+ return pending;}- const queries = new WeakMap<MemoryRecallContext, Promise<number[]>>();- async function recall(scope: string, context: MemoryRecallContext = {}): Promise<MemoryCandidate[]> {- const facts = (await base.facts?.(scope)) ?? [];- const query = context.query?.trim();- try {- const embedder = await embedderFor(context);- if (!embedder || !query) return recentCandidates(facts);- const signal = AbortSignal.timeout(8000);- const vectors = await indexed(scope, facts, signal, embedder);- if (!facts.length) return [];- let request = queries.get(context);- if (!request) {- request = embedder.embed([query.slice(0, 4000)], signal).then((result) => result[0]!);- queries.set(context, request);- }- const queryVector = await request;- if (facts.some((fact) => vectors[factHash(fact.text)]?.length !== queryVector.length)) {- await writeAll(Object.keys(vectors), (hash) => {- cache.delete(`${embedder.id} ${hash}`);- return index.delete(vectorId({ scope, model: embedder.id, hash }));- });- throw new Error("Memory embedding dimensions changed; rebuilding the index on the next recall");- }- return facts- .map((fact) => ({- fact,- score: vectors[factHash(fact.text)]!.reduce((sum, value, i) => sum + value * queryVector[i]!, 0),- }))- .filter(({ score }) => score >= (opts.minSimilarity ?? 0.2))- .sort((a, b) => b.score - a.score)- .map(({ fact, score }) => factCandidate(fact, score));- } catch (error) {- opts.onError?.(error);- return recentCandidates(facts);- }+ function view(notebook: MemoryService): MemoryService {+ return {+ ...notebook,+ withDisclosure: (access) => view(disclosedMemory(notebook, access)),+ async recall(scope: string, context: MemoryRecallContext = {}): Promise<MemoryCandidate[]> {+ const facts = (await notebook.facts?.(scope)) ?? [];+ if (!facts.length) return [];+ try {+ const query = index && context.query?.trim() ? await queryFor(context) : undefined;+ if (query && index) {+ const byId = new Map(facts.map((fact) => [fact.id, fact]));+ const matches = await index.search({+ scope,+ ...query,+ facts: facts.map((fact) => ({ id: fact.id, hash: factHash(fact.text), at: fact.createdAt })),+ minSimilarity: opts.minSimilarity,+ });+ return matches.flatMap((match) => {+ const fact = byId.get(match.factId);+ return fact ? [factCandidate(fact, match.score)] : [];+ });+ }+ return recentCandidates(facts);+ } catch (error) {+ opts.onError?.(error);+ return recentCandidates((await notebook.facts?.(scope)) ?? []);+ }+ },+ };}- return { ...base, recall };+ return view(base);}diff --git a/src/memory/vector-indexer.ts b/src/memory/vector-indexer.tsnew file mode 100644index 000000000..291600a01--- /dev/null+++ b/src/memory/vector-indexer.ts@@ -0,0 +1,80 @@+import { unitVector, type MemoryEmbedder } from "../model/embeddings.ts";+import type { LeaderLease } from "../persistence/leader-lease.ts";+import { reportFailure } from "../util/errors.ts";+import type { MemoryVectorStore } from "./postgres-vector-store.ts";++export function createMemoryVectorIndexer(opts: {+ resolve: () => Promise<+ { embedder: MemoryEmbedder; store: Pick<MemoryVectorStore, "missing" | "put" | "prune"> } | undefined+ >;+ leaderLease: LeaderLease;+ onError?: (error: unknown) => void;+ now?: () => number;+}) {+ const onError = opts.onError ?? ((error: unknown) => reportFailure("memory vector index", error));+ let pending: Promise<void> | undefined;+ let model: string | undefined;+ let dimensions: number | undefined;+ let afterId: string | undefined;+ let nextProbeAt = 0;+ async function run(): Promise<void> {+ await opts.leaderLease.hold("memory:vector-index", async (lost) => {+ const current = await opts.resolve();+ if (!current) return;+ const controller = new AbortController();+ void lost.then(() => controller.abort());+ const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(60_000)]);+ const { embedder, store } = current;+ const now = (opts.now ?? Date.now)();+ if (model !== embedder.id || dimensions === undefined || now >= nextProbeAt) {+ const probe = await embedder.embed(["Memory index dimension probe."], signal);+ if (probe.length !== 1) throw new Error("Invalid memory index dimension probe");+ const width = unitVector(probe[0]).length;+ if (width > 16000) throw new Error("Memory embedding exceeds pgvector dimensions");+ if (model !== embedder.id || dimensions !== width) afterId = undefined;+ model = embedder.id;+ dimensions = width;+ nextProbeAt = now + 10 * 60_000;+ }+ signal.throwIfAborted();+ const facts = await store.missing(embedder.id, 64, dimensions, signal, afterId);+ afterId = facts.at(-1)?.factId;+ try {+ if (facts.length) {+ const vectors = await embedder.embed(+ facts.map((fact) => fact.text.slice(0, 8000)),+ signal,+ );+ if (vectors.length !== facts.length) throw new Error("Invalid memory index embedding count");+ const rows = facts.map((fact, i) => ({+ scope: fact.scope,+ factId: fact.factId,+ hash: fact.hash,+ model: embedder.id,+ vector: unitVector(vectors[i]),+ }));+ if (rows.some((row) => row.vector.length !== dimensions)) {+ dimensions = undefined;+ nextProbeAt = 0;+ afterId = undefined;+ throw new Error("Memory embedding dimensions changed during indexing");+ }+ signal.throwIfAborted();+ await store.put(rows, signal);+ }+ } finally {+ if (!signal.aborted) await store.prune(embedder.id, 200, dimensions, signal).catch(onError);+ }+ });+ }+ return {+ tick(): Promise<void> {+ pending ??= run()+ .catch(onError)+ .finally(() => {+ pending = undefined;+ });+ return pending;+ },+ };+}diff --git a/src/wiring.ts b/src/wiring.tsindex d4fe72315..2d6c8feb3 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -46,7 +46,7 @@ import {SUBAGENT_TREE_RUN_CAP,} from "./sessions/session-syscalls.ts";import { createMemoryEmbedder } from "./model/embeddings.ts";-import { createSemanticMemoryService, type MemoryVector } from "./memory/semantic-recall.ts";+import { createSemanticMemory } from "./memory/semantic-memory.ts";import { createPostgresBrokerSessions, type BrokerSessionStore } from "./auth/broker-sessions.ts";import { createDirectFileUploads, type DirectFileUploads } from "./files/direct-file-upload.ts";import { createPostgresFileUploadStore } from "./files/file-upload-store.ts";@@ -433,7 +433,7 @@ import {import { createPostgresErrorLog } from "./admin/postgres-error-log.ts";import { createMetricsSink, type MetricsSink } from "./admin/metrics-sink.ts";import { createPostgresMetricsSink } from "./admin/postgres-metrics-sink.ts";-import { errMessage, reportFailure, reportFailureAs, swallowAs } from "./util/errors.ts";+import { errMessage, reportFailureAs, swallowAs } from "./util/errors.ts";import { sleep, withTimeout } from "./util/async.ts";import { createSlackInstallationStore, type SlackInstallationStore } from "./surfaces/slack-installation.ts";@@ -845,22 +845,19 @@ export function buildApp(const notebookMemory: MemoryService = config.databaseUrl? createPostgresMemoryService(config.databaseUrl): createMemoryService(workspace);- const defaultMemory = createSemanticMemoryService(- notebookMemory,- async () => {- const id = await configStore.getEmbeddingModelDurable();- return id ? resolveMemoryEmbedder(id) : undefined;- },- artifactMap<MemoryVector>("memory_vectors", ["scope"]),- {- onError: (error) => reportFailure("memory semantic recall; using recent facts", error),- },- );+ const semanticMemory = createSemanticMemory({+ notebook: notebookMemory,+ databaseUrl: config.databaseUrl,+ embeddingModel: () => configStore.getEmbeddingModelDurable(),+ resolveEmbedder: (id) => resolveMemoryEmbedder(id),+ leaderLease,+ createSweeper,+ });// Session storage is built further down; trace-derived providers only read it after the first turn.const memorySessions: { store?: SessionStore } = {};const baseMemory: MemoryService = withMemoryTidy(createConfiguredMemoryService({- defaultMemory,+ defaultMemory: semanticMemory.memory,config: config.memoryProviderConfig,sessionEntries: (sessionId) => {if (!memorySessions.store) throw new Error("session store is not ready");@@ -2780,6 +2777,7 @@ export function buildApp(.catch(swallowAs("wiring: monitor resume failed", undefined));monitorRetentionSweeper.start();spendSweeper?.start();+ semanticMemory.start();if (config.skillSyncPollMs > 0) skillSyncEngine.start(config.skillSyncPollMs);blobSweeper.start();composioReturnSweeper.start();@@ -2810,6 +2808,7 @@ export function buildApp(processReaper?.stop(),monitorRetentionSweeper.stop(),spendSweeper?.stop(),+ semanticMemory.stop(),skillSyncEngine.stop(),idleSweeper?.stop(),deployMaintenanceSweeper.stop(),@@ -2887,6 +2886,7 @@ export function buildApp(].map((closing) => closing?.catch(reportFailureAs("shutdown: close event stores", undefined))),);await harness.turns.close?.();+ await semanticMemory.close();await flyTunnel?.stop();},};diff --git a/test/cross-scope-memory.test.ts b/test/cross-scope-memory.test.tsindex 07fd6919c..ef2c38f46 100644--- a/test/cross-scope-memory.test.ts+++ b/test/cross-scope-memory.test.ts@@ -3,10 +3,10 @@ import assert from "node:assert/strict";import { mkdtemp, rm } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { scoredMemoryIndex } from "./support/scored-memory-index.ts";import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";import { createMemoryService } from "../src/memory/memory-service.ts";-import { createSemanticMemoryService, type MemoryVector } from "../src/memory/semantic-recall.ts";+import { createSemanticMemoryService } from "../src/memory/semantic-recall.ts";import { createRoutedMemoryService } from "../src/memory/provider-router.ts";import { recallAcrossScopes, renderRecall, searchAcrossScopes } from "../src/memory/cross-scope.ts";@@ -30,7 +30,7 @@ async function fixture(t: { after: (fn: () => Promise<void>) => void }) {});},}),- createMemoryMap<MemoryVector>(),+ scoredMemoryIndex(base, (text) => Number(text.split(" ")[0])),);return { base, memory, calls, workspace };}diff --git a/test/memory-disclosure.test.ts b/test/memory-disclosure.test.tsindex 5ec640276..45ffc9139 100644--- a/test/memory-disclosure.test.ts+++ b/test/memory-disclosure.test.ts@@ -1,3 +1,4 @@+import { createSemanticMemoryService } from "../src/memory/semantic-recall.ts";import { createCurrentScopeMembers } from "../src/resolution/scope-membership.ts";import { createDirectoryStore } from "../src/directory/directory-store.ts";import { test } from "node:test";@@ -314,3 +315,22 @@ test("a recall-only routed provider still reaches prompt recall through disclosuassert.match(recalled, /BRAIN_FACT/);assert.doesNotMatch(recalled, /OPAQUE_SENTINEL/);});++test("semantic recall ranks only currently disclosed facts and rechecks membership on every recall", async () => {+ const f = fixture();+ f.put("personal:alice", "hidden", "restricted");+ f.put("personal:alice", "late ordinary fact", "ordinary");+ const queries: string[][] = [];+ const semantic = createSemanticMemoryService(f.raw, async () => ({ id: "test", embed: async () => [[1, 0]] }), {+ async search(input) {+ queries.push(input.facts.map((fact) => fact.id));+ return input.facts.map((fact) => ({ factId: fact.id, hash: fact.hash, score: 1 }));+ },+ });+ const view = disclosedMemory(semantic, f.access);+ assert.match((await view.recall("personal:alice", { query: "recall" }))[0]!.text, /late ordinary fact/);+ assert.deepEqual(queries, [["2"]]);+ f.access.currentScopeMembers = async () => [];+ assert.deepEqual(await view.recall("personal:alice", { query: "recall" }), []);+ assert.deepEqual(queries, [["2"]]);+});diff --git a/test/memory-vector-indexer.test.ts b/test/memory-vector-indexer.test.tsnew file mode 100644index 000000000..5d939344a--- /dev/null+++ b/test/memory-vector-indexer.test.ts@@ -0,0 +1,199 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createMemoryVectorIndexer } from "../src/memory/vector-indexer.ts";+import { createNoopLeaderLease } from "../src/persistence/leader-lease.ts";+import type { MemoryVectorStore, MemoryVector } from "../src/memory/postgres-vector-store.ts";+import type { MemoryEmbedder } from "../src/model/embeddings.ts";++function fixture() {+ let now = 1000;+ const requests: Array<{ model: string; limit: number; dimensions?: number; afterId?: string }> = [];+ const writes: MemoryVector[][] = [];+ const failures: unknown[] = [];+ const prunes: string[] = [];+ const store: Pick<MemoryVectorStore, "missing" | "put" | "prune"> = {+ async missing(model, limit, dimensions, _signal, afterId) {+ requests.push({ model, limit, dimensions, afterId });+ return [+ {+ scope: "personal:a",+ factId: String(Number(afterId ?? 0) + 64),+ hash: "hash",+ text: afterId ? "good" : "bad",+ },+ ];+ },+ async put(rows) {+ writes.push([...rows]);+ },+ async prune(model, limit) {+ assert.equal(limit, 200);+ prunes.push(model);+ return 0;+ },+ };+ const embedder: MemoryEmbedder = {+ id: "model",+ async embed(texts) {+ return texts.map(() => [1, 0]);+ },+ };+ const indexer = createMemoryVectorIndexer({+ resolve: async () => ({ store, embedder }),+ leaderLease: createNoopLeaderLease(),+ now: () => now,+ onError: (error) => {+ failures.push(error);+ },+ });+ return {+ store,+ embedder,+ indexer,+ requests,+ writes,+ failures,+ prunes,+ advance: () => {+ now += 600001;+ },+ };+}++test("indexer advances the bounded cursor before a failed batch so later facts can progress", async () => {+ const f = fixture();+ f.embedder.embed = async (texts) => {+ if (texts.includes("bad")) throw new Error("poison batch");+ return texts.map(() => [1, 0]);+ };+ await f.indexer.tick();+ await f.indexer.tick();+ assert.deepEqual(f.requests, [+ { model: "model", limit: 64, dimensions: 2, afterId: undefined },+ { model: "model", limit: 64, dimensions: 2, afterId: "64" },+ ]);+ assert.equal(f.failures.length, 1);+ assert.equal(f.writes.length, 1);+ assert.equal(f.writes[0]![0]!.factId, "128");+ assert.equal(f.prunes.length, 2);+});++test("indexer allows only one batch in flight even if a sweep overlaps", async () => {+ const f = fixture();+ let release!: () => void;+ const gate = new Promise<void>((resolve) => {+ release = resolve;+ });+ let entered!: () => void;+ const started = new Promise<void>((resolve) => {+ entered = resolve;+ });+ f.embedder.embed = async (texts) => {+ if (texts[0] === "bad") {+ entered();+ await gate;+ }+ return texts.map(() => [1, 0]);+ };+ const first = f.indexer.tick();+ await started;+ const second = f.indexer.tick();+ assert.equal(first, second);+ release();+ await Promise.all([first, second]);+ assert.equal(f.requests.length, 1);+ assert.equal(f.writes.length, 1);+});++test("indexer resets its cursor on model and same-model dimension changes, and wraps at the end", async () => {+ const f = fixture();+ await f.indexer.tick();+ f.advance();+ f.embedder.embed = async (texts) => texts.map(() => [1, 0, 0]);+ await f.indexer.tick();+ f.embedder.id = "new-model";+ await f.indexer.tick();+ assert.ok(f.requests.every((request) => request.afterId === undefined));+ assert.deepEqual(+ f.requests.map((request) => request.dimensions),+ [2, 3, 3],+ );+ const missing = f.store.missing;+ f.store.missing = async (...args) => {+ await missing(...args);+ return [];+ };+ await f.indexer.tick();+ f.store.missing = missing;+ await f.indexer.tick();+ assert.equal(f.requests.at(-1)!.afterId, undefined);+});++test("indexer refuses malformed batches and never writes after losing its leader lease", async () => {+ const f = fixture();+ f.embedder.embed = async (texts) => (texts[0] === "bad" ? [[1, 0, 0]] : [[1, 0]]);+ await f.indexer.tick();+ assert.equal(f.writes.length, 0);+ assert.equal(f.failures.length, 1);+ let lose!: () => void;+ const lost = new Promise<void>((resolve) => {+ lose = resolve;+ });+ f.embedder.embed = async (texts) => {+ if (texts[0] === "bad") lose();+ return texts.map(() => [1, 0]);+ };+ const indexer = createMemoryVectorIndexer({+ resolve: async () => ({ store: f.store, embedder: f.embedder }),+ leaderLease: { hold: async (_key, fn) => fn(lost) },+ onError: (error) => {+ f.failures.push(error);+ },+ });+ await indexer.tick();+ assert.equal(f.writes.length, 0);+ assert.equal(f.failures.length, 2);+});++test("fact embeddings stay bounded and dimension probes are cached for ten minutes", async () => {+ const f = fixture();+ const calls: string[][] = [];+ f.store.missing = async () => [{ scope: "personal:a", factId: "1", hash: "full-text-hash", text: "x".repeat(12000) }];+ f.embedder.embed = async (texts) => {+ calls.push(texts);+ return texts.map(() => [1, 0]);+ };+ await f.indexer.tick();+ await f.indexer.tick();+ assert.equal(calls.filter((texts) => texts[0] === "Memory index dimension probe.").length, 1);+ assert.equal(calls.filter((texts) => texts[0]?.length === 8000).length, 2);+ assert.equal(f.writes[0]![0]!.hash, "full-text-hash");+ f.advance();+ await f.indexer.tick();+ assert.equal(calls.filter((texts) => texts[0] === "Memory index dimension probe.").length, 2);+});++for (const operation of ["embed", "put"] as const) {+ test(`indexer reports both ${operation} and prune failures without masking either`, async () => {+ const f = fixture();+ const primary = new Error(`${operation} failed`);+ const cleanup = new Error("prune failed");+ if (operation === "embed") {+ f.embedder.embed = async (texts) => {+ if (texts[0] === "bad") throw primary;+ return texts.map(() => [1, 0]);+ };+ } else {+ f.store.put = async () => {+ throw primary;+ };+ }+ f.store.prune = async () => {+ throw cleanup;+ };+ await f.indexer.tick();+ assert.equal(f.failures.length, 2);+ assert.ok(f.failures.includes(primary));+ assert.ok(f.failures.includes(cleanup));+ });+}diff --git a/test/semantic-memory-http.test.ts b/test/semantic-memory-http.test.tsindex 90456f9d8..e1534b404 100644--- a/test/semantic-memory-http.test.ts+++ b/test/semantic-memory-http.test.ts@@ -1,5 +1,12 @@import "./support/auto-fake-sprites.ts";-import { test } from "node:test";+import { test, type TestContext } from "node:test";+import { randomUUID } from "node:crypto";+import pg from "pg";+import { migrateRegisteredPgSchemas } from "../src/persistence/pg-pool.ts";+import { createPostgresVectorStore } from "../src/memory/postgres-vector-store.ts";+import { createMemoryEmbedder } from "../src/model/embeddings.ts";+import { factHash } from "../src/memory/facts.ts";+import type { MemoryService } from "../src/memory/memory-service.ts";import assert from "node:assert/strict";import { createServer as httpServer, type Server } from "node:http";import type { AddressInfo } from "node:net";@@ -22,7 +29,62 @@ const close = async (server: Server) => {await new Promise<void>((r) => server.close(() => r()));};-test("HTTP turn injects semantic matches; grep remains independent and authorized", async (t) => {+const database = process.env.DATABASE_URL;+const pgTest = database ? test : test.skip;+async function isolatedDatabase(t: TestContext) {+ const admin = new pg.Client({ connectionString: database! });+ await admin.connect();+ const name = `memory_http_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE DATABASE ${name}`);+ const fixture = { url: "", stop: async () => {} };+ t.after(async () => {+ await fixture.stop();+ await admin.query(`DROP DATABASE ${name} WITH (FORCE)`);+ await admin.end();+ });+ const url = new URL(database!);+ url.pathname = `/${name}`;+ fixture.url = url.toString();+ return fixture;+}++async function seedVectors(+ memory: MemoryService,+ databaseUrl: string,+ model: string,+ baseUrl: string,+ scopes: string[],+) {+ const index = createPostgresVectorStore(databaseUrl);+ const embedder = createMemoryEmbedder(+ { kind: "embedding", id: model, name: model, provider: "openai", baseUrl },+ "test-key",+ );+ try {+ for (const scope of scopes) {+ const facts = await memory.facts!(scope);+ const vectors = await embedder.embed(+ facts.map((fact) => fact.text),+ AbortSignal.timeout(5000),+ );+ await index.put(+ facts.map((fact, i) => ({+ scope,+ model: embedder.id,+ factId: fact.id,+ hash: factHash(fact.text),+ vector: vectors[i]!,+ })),+ );+ }+ } finally {+ await index.close();+ }+}++pgTest("HTTP turn injects semantic matches; grep remains independent and authorized", async (t) => {+ const databaseFixture = await isolatedDatabase(t);+ const databaseUrl = databaseFixture.url;const inputs: string[][] = [];const provider = httpServer(async (req, res) => {let body = "";@@ -41,10 +103,26 @@ test("HTTP turn injects semantic matches; grep remains independent and authorizeconst built = buildApp(testConfig({memoryStrategy: "agent-only",+ databaseUrl,providerBaseUrls: { openai: providerUrl },openaiApiKey: "test-key",}),);+ databaseFixture.stop = () => built.runtime.stop();+ await migrateRegisteredPgSchemas(databaseUrl);+ await built.memory.capture("personal:unconfigured", ["a fact before model configuration"], Date.now());+ await built.memory.recall("personal:unconfigured", { query: "no model configured" });+ const inspector = new pg.Client({ connectionString: databaseUrl });+ await inspector.connect();+ try {+ const { rows } = await inspector.query(+ "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed, to_regclass('memory_fact_vectors') AS vectors",+ );+ assert.equal(rows[0].installed, false);+ assert.equal(rows[0].vectors, null);+ } finally {+ await inspector.end();+ }const embeddingId = "test-model";await built.modelRegistry.upsert({ kind: "embedding", id: embeddingId, name: "Embeddings", provider: "openai" },@@ -56,6 +134,7 @@ test("HTTP turn injects semantic matches; grep remains independent and authorizeconst otherFact = "Another user's confidential browser preference";await built.memory.capture("personal:U1", [fact, ...filler], Date.now());await built.memory.capture("personal:U2", [otherFact], Date.now());+ await seedVectors(built.memory, databaseUrl, embeddingId, providerUrl, ["org:default-org", "personal:U1"]);const server = createInsecureTestServer(built.app, {memory: built.memory,capabilitySecret: TEST_CAPABILITY_SECRET,@@ -119,7 +198,11 @@ test("HTTP turn injects semantic matches; grep remains independent and authorizeinputs.every((batch) => batch.every((text) => !/FOREIGN_HISTORY_SENTINEL|TAINTED_HISTORY_SENTINEL/.test(text))),);- await built.memory.edit!("personal:U1", [{ id: "1", text: "Updated preference: use browser smoke tests." }]);+ const personalFact = (await built.memory.facts!("personal:U1")).find((row) => row.text === fact)!;+ await built.memory.edit!("personal:U1", [+ { id: personalFact.id, text: "Updated preference: use browser smoke tests." },+ ]);+ await seedVectors(built.memory, databaseUrl, embeddingId, providerUrl, ["personal:U1"]);const updated = await post("/v1/turns", { ...request, conversation: { kind: "dm", threadRef: "updated" } });const newResult = (await updated.json()) as { reply: string };assert.ok(!newResult.reply.includes(fact));@@ -139,7 +222,9 @@ test("HTTP turn injects semantic matches; grep remains independent and authorize);});-test("HTTP Open DM combines room and current memory, then blocks reads after membership revocation", async (t) => {+pgTest("HTTP Open DM combines room and current memory, then blocks reads after membership revocation", async (t) => {+ const databaseFixture = await isolatedDatabase(t);+ const databaseUrl = databaseFixture.url;const inputs: string[][] = [];const provider = httpServer(async (req, res) => {let body = "";@@ -163,10 +248,12 @@ test("HTTP Open DM combines room and current memory, then blocks reads after memtestConfig({sharingPosture: "open",memoryStrategy: "agent-only",+ databaseUrl,providerBaseUrls: { openai: providerUrl },openaiApiKey: "test",}),);+ databaseFixture.stop = () => built.runtime.stop();await built.modelRegistry.upsert({ kind: "embedding", id: "test", name: "Embeddings", provider: "openai" }, "admin");await built.config.setEmbeddingModel("test");const personal = "personal:U1",@@ -194,6 +281,7 @@ test("HTTP Open DM combines room and current memory, then blocks reads after mem);await built.memory.capture(room, ["0.99 REMOTE_BEST", "0.72 REMOTE_NEAR"], Date.now());await built.memory.capture(denied, ["1 DENIED_NOTEBOOK"], Date.now());+ await seedVectors(built.memory, databaseUrl, "test", providerUrl, [personal, room]);const server = createInsecureTestServer(built.app);const base = await listen(server);t.after(() => close(server));diff --git a/test/semantic-memory-large-pg.test.ts b/test/semantic-memory-large-pg.test.tsnew file mode 100644index 000000000..413030df3--- /dev/null+++ b/test/semantic-memory-large-pg.test.ts@@ -0,0 +1,51 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { randomUUID } from "node:crypto";+import pg from "pg";+import { createPostgresMemoryService } from "../src/memory/postgres-memory-service.ts";+import { createPostgresVectorStore } from "../src/memory/postgres-vector-store.ts";+import { factHash } from "../src/memory/facts.ts";++const url = process.env.DATABASE_URL;++test("a 4500-fact scope fully drains 3072-dimensional recall in under one second", { skip: !url }, async (t) => {+ const scope = `personal:large-${randomUUID()}`;+ const model = `large-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ const client = new pg.Client({ connectionString: url! });+ await client.connect();+ t.after(async () => {+ await client.end();+ await index.close();+ });+ await base.facts!(scope);+ await index.missing(model, 1);+ await client.query(+ `INSERT INTO memory_facts(scope_id, text, sensitivity, source_unknown, created_at, updated_at)+ SELECT $1, 'Large notebook fact ' || g, 'ordinary', false, g, g FROM generate_series(1,4500) g`,+ [scope],+ );+ const vector = Array.from({ length: 3072 }, (_, i) => Math.sin(i * 123.4567));+ await client.query(+ `INSERT INTO memory_fact_vectors(scope_id, model, fact_id, fact_hash, dimensions, embedding)+ SELECT scope_id, $2, id, encode(sha256(convert_to(text,'UTF8')),'hex'), 3072, $3::vector+ FROM memory_facts WHERE scope_id = $1 AND created_at <= 4490`,+ [scope, model, JSON.stringify(vector)],+ );+ await client.query("ANALYZE memory_facts; ANALYZE memory_fact_vectors");+ const facts = (await base.facts!(scope)).map((fact) => ({+ id: fact.id,+ hash: factHash(fact.text),+ at: fact.createdAt,+ }));+ const start = performance.now();+ const matches = await index.search({ scope, model, vector, facts });+ const elapsed = performance.now() - start;+ assert.equal(matches.length, 4500);+ assert.ok(matches.slice(0, 4490).every((row) => row.score > 0.99));+ assert.ok(matches.slice(4490).every((row) => Math.abs(row.score - 0.2) < 1e-9));+ assert.equal(matches[0]!.factId, facts[4489]!.id);+ assert.ok(elapsed < 1000, `4500-fact full drain took ${elapsed.toFixed(1)}ms`);+ t.diagnostic(`4500 facts / 3072 dimensions / full drain: ${elapsed.toFixed(1)}ms`);+});diff --git a/test/semantic-memory-pg.test.ts b/test/semantic-memory-pg.test.tsindex 9fa000dc4..cfa72b5df 100644--- a/test/semantic-memory-pg.test.ts+++ b/test/semantic-memory-pg.test.ts@@ -1,47 +1,224 @@import { test } from "node:test";import assert from "node:assert/strict";import { randomUUID } from "node:crypto";+import pg from "pg";import { createPostgresMemoryService } from "../src/memory/postgres-memory-service.ts";-import { createPostgresMapFactory } from "../src/persistence/durable-map.ts";-import { createSemanticMemoryService, type MemoryVector } from "../src/memory/semantic-recall.ts";-import type { MemoryEmbedder } from "../src/model/embeddings.ts";-import { recallAcrossScopes } from "../src/memory/cross-scope.ts";-import type { MemoryService } from "../src/memory/memory-service.ts";--const recall = async (memory: MemoryService, scope: string, context = {}) =>- (await recallAcrossScopes(memory, [scope], context)).map((fact) => fact.text).join("\n");+import { createPostgresVectorStore, type MemoryVectorStore } from "../src/memory/postgres-vector-store.ts";+import { createSemanticMemoryService } from "../src/memory/semantic-recall.ts";+import { factHash, type MemoryFact } from "../src/memory/facts.ts";const url = process.env.DATABASE_URL;-test("Postgres persists vectors across instances and follows fact edits and deletes", { skip: !url }, async () => {- const scope = `personal:semantic-${randomUUID()}`;- const index = createPostgresMapFactory(url!).map<MemoryVector>("memory_vectors", ["scope"]);- const calls: string[][] = [];- const embedder: MemoryEmbedder = {- id: "test-v1",- async embed(texts) {- calls.push(texts);- return texts.map(() => [1, 0]);- },+const requested = (facts: readonly MemoryFact[]) =>+ facts.map((fact) => ({ id: fact.id, hash: factHash(fact.text), at: fact.createdAt }));+const write = (+ store: MemoryVectorStore,+ scope: string,+ facts: readonly MemoryFact[],+ model = "test-v1",+ vector: (fact: MemoryFact) => number[] = () => [1, 0],+) =>+ store.put(facts.map((fact) => ({ scope, model, factId: fact.id, hash: factHash(fact.text), vector: vector(fact) })));++test(+ "Postgres native cosine persists across instances and rejects edited/deleted snapshots",+ { skip: !url },+ async (t) => {+ const scope = `personal:semantic-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ t.after(() => index.close());+ await base.capture(scope, ["original browser preference"], Date.now());+ const facts = await base.facts!(scope);+ await write(index, scope, facts);+ const search = () => index.search({ scope, model: "test-v1", vector: [1, 0], facts: requested(facts) });+ assert.equal((await search())[0]!.factId, facts[0]!.id);+ const secondIndex = createPostgresVectorStore(url!);+ t.after(() => secondIndex.close());+ const calls: string[][] = [];+ const memory = createSemanticMemoryService(+ base,+ async () => ({+ id: "test-v1",+ async embed(texts) {+ calls.push(texts);+ return texts.map(() => [1, 0]);+ },+ }),+ secondIndex,+ );+ assert.match((await memory.recall(scope, { query: "verify" }))[0]!.text, /original/);+ assert.deepEqual(calls, [["verify"]]);+ await base.edit!(scope, [{ id: facts[0]!.id, text: "revised browser preference" }]);+ assert.deepEqual(await search(), []);+ assert.match((await memory.recall(scope, { query: "verify" }))[0]!.text, /revised/);+ const revised = await base.facts!(scope);+ const unindexed = await index.search({+ scope,+ model: "test-v1",+ vector: [1, 0],+ facts: requested(revised),+ minSimilarity: 0.9,+ });+ assert.deepEqual(+ unindexed.map((row) => row.factId),+ [revised[0]!.id],+ );+ assert.ok(Math.abs(unindexed[0]!.score - 0.2) < 1e-9);+ await write(index, scope, revised);+ assert.equal(+ (await index.search({ scope, model: "test-v1", vector: [1, 0], facts: requested(revised) }))[0]!.score,+ 1,+ );+ await base.delete!(scope, [facts[0]!.id]);+ assert.deepEqual(await search(), []);+ assert.deepEqual(await index.search({ scope, model: "test-v1", vector: [1, 0], facts: requested(revised) }), []);+ assert.deepEqual(await memory.recall(scope, { query: "verify" }), []);+ assert.ok(!(await index.missing("test-v1", 256)).some((row) => row.factId === facts[0]!.id));+ },+);++test("native cosine returns one stable ranking and excludes other scopes/models", { skip: !url }, async (t) => {+ const scope = `personal:ranking-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ t.after(() => index.close());+ await base.capture(scope, ["first", "second"], 1000);+ const facts = await base.facts!(scope);+ await write(index, scope, facts, "rank-model", () => [3, 4]);+ await write(index, scope, facts, "other-model", () => [1, 0]);+ await write(index, "personal:other", facts, "rank-model", () => [1, 0]);+ const ranked = await index.search({ scope, model: "rank-model", vector: [2, 0], facts: requested(facts) });+ assert.ok(ranked.every((row) => Math.abs(row.score - 0.6) < 1e-6));+ assert.deepEqual(+ await index.search({ scope, model: "rank-model", vector: [2, 0], facts: requested(facts), minSimilarity: 0.9 }),+ [],+ );+ assert.deepEqual(+ ranked.map((row) => row.factId),+ facts.map((fact) => fact.id).sort((a, b) => Number(BigInt(a) - BigInt(b))),+ );+});++test("model and dimension drift fall back until reindexed", { skip: !url }, async (t) => {+ const scope = `personal:width-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ t.after(() => index.close());+ await base.capture(scope, ["real embedding width"], 1000);+ const facts = await base.facts!(scope);+ const wide = [1, 0, 0];+ await write(index, scope, facts, "wide-model", () => wide);+ const unindexedScore = async (model: string, vector: number[]) => {+ const rows = await index.search({ scope, model, vector, facts: requested(facts) });+ return rows.length === 1 && Math.abs(rows[0]!.score - 0.2) < 1e-9;};- const first = createSemanticMemoryService(createPostgresMemoryService(url!), async () => embedder, index);- await first.capture(scope, ["original browser preference"], Date.now());- await recall(first, scope, { query: "verify" });- const reopened = createSemanticMemoryService(- createPostgresMemoryService(url!),- async () => embedder,- createPostgresMapFactory(url!).map<MemoryVector>("memory_vectors", ["scope"]),- );- calls.length = 0;- assert.match(await recall(reopened, scope, { query: "verify" }), /original/);- assert.deepEqual(calls, [["verify"]]);- const [fact] = await reopened.facts!(scope);- await reopened.edit!(scope, [{ id: fact!.id, text: "revised browser preference" }]);- assert.match(- await recall(reopened, scope, { query: "verify" }),- /^- \(\d{4}-\d\d-\d\d\) revised browser preference$/,- );- assert.equal((await index.select({ where: { field: "scope", anyOfFold: [scope] } })).length, 1);- await reopened.delete!(scope, [fact!.id]);- assert.equal(await recall(reopened, scope, { query: "verify" }), "");- assert.deepEqual(await index.select({ where: { field: "scope", anyOfFold: [scope] } }), []);+ assert.ok(await unindexedScore("wide-model", [1, 0]));+ assert.ok(await unindexedScore("new-model", wide));+ assert.ok(+ (await index.missing("wide-model", 256, 2, undefined, String(BigInt(facts[0]!.id) - 1n))).some(+ (row) => row.factId === facts[0]!.id,+ ),+ );+ assert.ok(+ !(await index.missing("wide-model", 256, 2, undefined, facts[0]!.id)).some((row) => row.factId === facts[0]!.id),+ );+ await write(index, scope, facts, "wide-model", () => [1, 0]);+ assert.equal(+ (await index.search({ scope, model: "wide-model", vector: [1, 0], facts: requested(facts) }))[0]!.score,+ 1,+ );+ await assert.rejects(+ index.put([{ scope, model: "wide-model", factId: facts[0]!.id, hash: factHash(facts[0]!.text), vector: [0, 0] }]),+ /Invalid memory embedding/,+ );+});++const storedVectors = async (client: pg.Client, scope: string, model: string) =>+ (+ await client.query(+ `SELECT fact_id, fact_hash, dimensions, embedding::text AS embedding FROM memory_fact_vectors+ WHERE scope_id = $1 AND model = $2 ORDER BY fact_id`,+ [scope, model],+ )+ ).rows as { fact_id: string; fact_hash: string; dimensions: number; embedding: string }[];++test("put waits on in-flight edits and never writes stale or deleted fact vectors", { skip: !url }, async (t) => {+ const scope = `personal:race-${randomUUID()}`;+ const model = `race-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ const client = new pg.Client({ connectionString: url! });+ await client.connect();+ t.after(async () => {+ await client.end();+ await index.close();+ });+ await base.capture(scope, ["race alpha", "race beta"], 1000);+ const [alpha, beta] = await base.facts!(scope);+ await client.query("BEGIN");+ await client.query("UPDATE memory_facts SET text = 'race alpha edited' WHERE id = $1", [alpha!.id]);+ let settled = false;+ const pending = write(index, scope, [alpha!, beta!], model).finally(() => {+ settled = true;+ });+ await new Promise((resolve) => setTimeout(resolve, 300));+ assert.equal(settled, false);+ await client.query("COMMIT");+ await pending;+ assert.deepEqual(+ (await storedVectors(client, scope, model)).map((row) => row.fact_id),+ [beta!.id],+ );+ const edited = (await base.facts!(scope)).find((fact) => fact.id === alpha!.id)!;+ await write(index, scope, [edited], model, () => [0, 1]);+ await write(index, scope, [alpha!], model, () => [1, 0]);+ const fresh = (await storedVectors(client, scope, model)).find((row) => row.fact_id === alpha!.id)!;+ assert.equal(fresh.fact_hash, factHash("race alpha edited"));+ assert.equal(fresh.embedding, "[0,1]");+ await base.delete!(scope, [beta!.id]);+ await write(index, scope, [beta!], model, () => [0, 1]);+ assert.equal((await storedVectors(client, scope, model)).find((row) => row.fact_id === beta!.id)!.embedding, "[1,0]");+ while (await index.prune(model, 256, 2));+ assert.deepEqual(+ (await storedVectors(client, scope, model)).map((row) => row.fact_id),+ [alpha!.id],+ );+});++test("prune deletes stale model, hash, deleted, and dimension rows in bounded batches", { skip: !url }, async (t) => {+ const scope = `personal:prune-${randomUUID()}`;+ const model = `prune-${randomUUID()}`;+ const base = createPostgresMemoryService(url!);+ const index = createPostgresVectorStore(url!);+ const client = new pg.Client({ connectionString: url! });+ await client.connect();+ t.after(async () => {+ await client.end();+ await index.close();+ });+ await base.capture(scope, ["prune keep", "prune edit", "prune delete", "prune wide", "prune unknown width"], 1000);+ const [keep, edit, gone, wide, unknown] = await base.facts!(scope);+ await write(index, scope, [keep!, edit!, gone!, unknown!], model);+ await write(index, scope, [wide!], model, () => [1, 0, 0]);+ await write(index, scope, [keep!], `${model}-old`);+ await base.edit!(scope, [{ id: edit!.id, text: "prune edit revised" }]);+ await base.delete!(scope, [gone!.id]);+ assert.equal(await index.prune(model, 1, 2), 1);+ let removed = 1;+ for (let deleted = -1; deleted; removed += deleted) {+ deleted = await index.prune(model, 2, 2);+ assert.ok(deleted <= 2);+ }+ assert.ok(removed >= 4);+ assert.deepEqual(+ (await storedVectors(client, scope, model)).map((row) => row.fact_id),+ [keep!.id, unknown!.id],+ );+ assert.deepEqual(await storedVectors(client, scope, `${model}-old`), []);+ await write(index, scope, [wide!], model, () => [1, 0, 0]);+ while (await index.prune(model, 256));+ assert.deepEqual(+ (await storedVectors(client, scope, model)).map((row) => row.fact_id),+ [keep!.id, wide!.id, unknown!.id],+ );});diff --git a/test/semantic-memory.test.ts b/test/semantic-memory.test.tsindex 43e6b596a..538e0a809 100644--- a/test/semantic-memory.test.ts+++ b/test/semantic-memory.test.ts@@ -3,10 +3,13 @@ import assert from "node:assert/strict";import { mkdtemp, rm } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-import { createMemoryMap } from "../src/persistence/durable-map.ts";+import { createSemanticMemory } from "../src/memory/semantic-memory.ts";+import { createNoopLeaderLease } from "../src/persistence/leader-lease.ts";+import { createSweeper } from "../src/util/sweeper.ts";+import { scoredMemoryIndex } from "./support/scored-memory-index.ts";import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";import { createMemoryService } from "../src/memory/memory-service.ts";-import { createSemanticMemoryService, memoryRecallQuery, type MemoryVector } from "../src/memory/semantic-recall.ts";+import { createSemanticMemoryService, memoryRecallQuery } from "../src/memory/semantic-recall.ts";import { createMemoryEmbedder, unitVector, type MemoryEmbedder } from "../src/model/embeddings.ts";import { recallAcrossScopes, renderRecall } from "../src/memory/cross-scope.ts";import type { MemoryRecallContext, MemoryService } from "../src/memory/memory-service.ts";@@ -21,7 +24,7 @@ async function fixture(t: { after: (fn: () => Promise<void>) => void }) {const dir = await mkdtemp(join(tmpdir(), "semantic-memory-"));t.after(() => rm(dir, { recursive: true, force: true }));const base = createMemoryService(createLocalWorkspaceStore(dir));- const index = createMemoryMap<MemoryVector>();+ const index = scoredMemoryIndex(base, (text) => (/browser|verify|updated/i.test(text) ? 1 : 0));const calls: string[][] = [];const embedder: MemoryEmbedder = {id: "test-v1",@@ -53,7 +56,7 @@ test("semantic recall selects an old match, respects budget, and leaves full gre});test("empty/deleted notebooks and other scopes cannot return cached facts", async (t) => {- const { base, memory, index, calls } = await fixture(t);+ const { base, memory, calls } = await fixture(t);await base.capture("personal:a", ["browser preference"], AT);await recall(memory, "personal:a", { query: "verify" });calls.length = 0;@@ -61,7 +64,7 @@ test("empty/deleted notebooks and other scopes cannot return cached facts", asynassert.deepEqual(calls, []);await base.delete!("personal:a", ["1"]);assert.equal(await recall(memory, "personal:a", { query: "verify" }), "");- assert.deepEqual(await index.all(), []);+ assert.deepEqual(calls, []);});test("embedding outage falls back to whole bullets, without disabling grep or writes", async (t) => {@@ -172,28 +175,42 @@ test("without an embedding model, prompt recall takes the newest facts across noassert.doesNotMatch(out, /old 0 /);});-test("a provider vector-width change falls back to recency and rebuilds the index", async (t) => {- const { base, index } = await fixture(t);- let width = 2;- const embedder: MemoryEmbedder = {- id: "same-model",- embed: async (texts) => texts.map(() => Array.from({ length: width }, () => 1)),- };- const memory = createSemanticMemoryService(base, async () => embedder, index);+test("query failures fall back without rebuilding facts in recall", async (t) => {+ const { base, embedder, calls } = await fixture(t);await base.capture("personal:a", ["A fact"], AT);- await recall(memory, "personal:a", { query: "fact" });- width = 3;- assert.equal(await recall(memory, "personal:a", { query: "fact" }), "- (2026-10-01) A fact");- await recall(memory, "personal:a", { query: "fact" });- assert.deepEqual(- (await index.all()).map((row) => row.vector.length),- [3],+ let errors = 0;+ const memory = createSemanticMemoryService(+ base,+ async () => embedder,+ {+ async search(input) {+ assert.deepEqual(input.vector, [1, 0]);+ throw new Error("Database query unavailable");+ },+ },+ { onError: () => errors++ },);+ assert.equal(await recall(memory, "personal:a", { query: "verify" }), "- (2026-10-01) A fact");+ assert.equal(errors, 1);+ assert.deepEqual(calls, [["verify"]]);});-test("moving the embedding gateway to another URL keeps the vector index", async (t) => {- const { base, index } = await fixture(t);- await base.capture("personal:a", ["browser preference", "release note"], AT);+test("without Postgres semantic recall stays readable and never embeds facts", async (t) => {+ const { base, embedder, calls } = await fixture(t);+ await base.capture("personal:a", ["older fact", "newer fact"], AT);+ const { memory } = createSemanticMemory({+ notebook: base,+ databaseUrl: undefined,+ embeddingModel: async () => "test",+ resolveEmbedder: async () => embedder,+ leaderLease: createNoopLeaderLease(),+ createSweeper,+ });+ assert.match(await recall(memory, "personal:a", { query: "verify" }), /newer fact[\s\S]*older fact/);+ assert.deepEqual(calls, []);+});++test("moving the embedding gateway keeps the same model identity", async () => {const model = {kind: "embedding" as const,baseUrl: "https://gateway.invalid",@@ -201,64 +218,21 @@ test("moving the embedding gateway to another URL keeps the vector index", asyncname: "embed",provider: "qm:gateway",};- const embedded: string[] = [];- const through = (url: string) => {- const transport = { url, apiKey: "k", apiKeyHeader: "x-k", models: { "gateway/embed": "embed" } };- const embedder = createMemoryEmbedder(model, null, transport, async (_url, init) => {- const input = JSON.parse(String(init?.body)).input as string[];- embedded.push(...input);- return Response.json({- data: input.map((text, i) => ({ index: i, embedding: /browser|verify/i.test(text) ? [1, 0] : [0, 1] })),- });- });- return createSemanticMemoryService(base, async () => embedder, index);- };- assert.equal(- await recall(through("http://signer.one:8080/v1"), "personal:a", { query: "verify" }),- "- (2026-10-01) browser preference",- );- const rows = await index.all();- embedded.length = 0;- assert.equal(- await recall(through("http://signer.two:8080/v1"), "personal:a", { query: "verify" }),- "- (2026-10-01) browser preference",- );- assert.deepEqual(embedded, ["verify"]);- assert.deepEqual(await index.all(), rows);+ const through = (url: string) =>+ createMemoryEmbedder(model, null, { url, apiKey: "k", apiKeyHeader: "x-k", models: { "gateway/embed": "embed" } });+ assert.equal(through("http://signer.one:8080/v1").id, through("http://signer.two:8080/v1").id);});-test("re-indexing a large notebook for a new model keeps index writes bounded", async (t) => {- const { base, index } = await fixture(t);- await base.capture(- "personal:a",- Array.from({ length: 40 }, (_, i) => `fact ${i}`),- AT,- );- let inFlight = 0;- let peak = 0;- const track = async (write: () => Promise<void>) => {- peak = Math.max(peak, ++inFlight);- await new Promise((resolve) => setImmediate(resolve));- try {- await write();- } finally {- inFlight--;- }- };- const tracked = {- ...index,- put: (id: string, value: MemoryVector) => track(() => index.put(id, value)),- delete: (id: string) => track(() => index.delete(id)),- };- const model = (id: string): MemoryEmbedder => ({ id, embed: async (texts) => texts.map(() => [1, 0]) });- for (const id of ["old-model", "new-model"])- await recall(- createSemanticMemoryService(base, async () => model(id), tracked),- "personal:a",- { query: "fact" },- );- assert.ok(peak <= 4, `peak concurrent index writes ${peak}`);- const rows = await index.all();- assert.equal(rows.length, 40);- assert.ok(rows.every((row) => row.model === "new-model"));+test("fallback reloads current disclosed facts after an edit races the index query", async (t) => {+ const { base, embedder } = await fixture(t);+ await base.capture("personal:a", ["old private fact"], AT);+ const memory = createSemanticMemoryService(base, async () => embedder, {+ async search() {+ await base.edit!("personal:a", [{ id: "1", text: "current fact" }]);+ throw new Error("old hash no longer exists");+ },+ });+ const result = await memory.recall("personal:a", { query: "verify" });+ assert.match(result[0]!.text, /current fact/);+ assert.doesNotMatch(result[0]!.text, /old private/);});diff --git a/test/support/scored-memory-index.ts b/test/support/scored-memory-index.tsnew file mode 100644index 000000000..9c7d51272--- /dev/null+++ b/test/support/scored-memory-index.ts@@ -0,0 +1,27 @@+import type { MemoryService } from "../../src/memory/memory-service.ts";+import type { MemoryVectorStore } from "../../src/memory/postgres-vector-store.ts";+import { factHash } from "../../src/memory/facts.ts";++export function scoredMemoryIndex(+ base: MemoryService,+ score: (text: string) => number,+): Pick<MemoryVectorStore, "search"> {+ return {+ async search(input) {+ const facts = await base.facts!(input.scope);+ const rows = facts+ .flatMap((fact) => {+ const hash = factHash(fact.text);+ const similarity = score(fact.text);+ if (+ !input.facts.some((wanted) => wanted.id === fact.id && wanted.hash === hash) ||+ similarity < (input.minSimilarity ?? 0.2)+ )+ return [];+ return [{ factId: fact.id, hash, score: similarity, at: fact.createdAt }];+ })+ .sort((a, b) => b.score - a.score || b.at - a.at);+ return rows;+ },+ };+}