Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: reuse stored lexemes for hybrid memory search

Adam Eivyad63adc237b9
diff --git a/server/lib/db.ddlParity.test.js b/server/lib/db.ddlParity.test.js
index 4cddee04f7..4cc0017edd 100644
--- a/server/lib/db.ddlParity.test.js
+++ b/server/lib/db.ddlParity.test.js
@@ -26,6 +26,7 @@
import { describe, it, expect } from 'vitest';
import { FTS_PAYLOAD_FIELDS } from './catalogTypes.js';
import { syncFeedDdl, syncFeedTables, buildSyncFeedTriggers } from './db/schema/syncFeed.js';
+import { memorySearchDdl } from './db/schema/core.js';
import { readFileSync, readdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
@@ -275,6 +276,15 @@ describe('DDL parity (init-db.sql ↔ db/schema ensureSchema)', () => {
// ── Targeted pins: contracts a two-sided set comparison cannot see ──────────
+ // memories is SQL_ONLY_TOLERATED, so the shared-table comparison cannot
+ // catch fresh-install drift in its generated expression or matching index.
+ it('stores the same memory search lexemes on fresh installs and upgrades', () => {
+ for (const statement of memorySearchDdl) {
+ expect(normalize(INIT_SQL)).toContain(normalize(statement));
+ }
+ expect(sqlIndexes.get('memories').get('idx_memories_search_tsv')).toBe('using gin (search_tsv)');
+ });
+
// The operator-action ledger (#5594) and the human-activity timeline (#2150)
// rely on their dedupe index as the ON CONFLICT idempotency contract, and on
// the composites (#5715) to keep a source-scoped read off a full happened_at
diff --git a/server/lib/db/schema/README.md b/server/lib/db/schema/README.md
index dda0bbc440..58ceeab5c4 100644
--- a/server/lib/db/schema/README.md
+++ b/server/lib/db/schema/README.md
@@ -23,7 +23,7 @@ init-db.sql, each with a one-line reason.
| Module | Export(s) | Domain |
|---|---|---|
-| `core.js` | `coreDdl`, `restoreReceiptsDdl` | Memory sync columns + the `schema_migrations` version tracker + machine-local `restore_receipts` (snapshot restore replay receipts, #9725) |
+| `core.js` | `coreDdl`, `memorySearchDdl`, `restoreReceiptsDdl` | Memory search lexemes and sync columns + the `schema_migrations` version tracker + machine-local `restore_receipts` (snapshot restore replay receipts, #9725) |
| `tribe.js` | `tribeDdl` | Tribe CRM — people, network-scoped identities (`tribe_identities`, the durable-handle truth table behind Beeper linking), touchpoints, memory links (machine-local) |
| `humanActivity.js` | `humanActivityDdl` | Human-activity timeline event store (machine-local) |
| `post.js` | `postDdl` | MeatSpace POST normalized runs and attempts (machine-local) |
diff --git a/server/lib/db/schema/core.js b/server/lib/db/schema/core.js
index 0d0e4b8137..fb4d4146fd 100644
--- a/server/lib/db/schema/core.js
+++ b/server/lib/db/schema/core.js
@@ -18,6 +18,17 @@ export const restoreReceiptsDdl = [
)`,
];
+// Store the same lexemes the old expression index used so ranking does not
+// tokenize every matching memory body again on every agent/search request.
+// PostgreSQL backfills existing rows and maintains the value after text edits.
+export const memorySearchDdl = [
+ `ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_tsv tsvector
+ GENERATED ALWAYS AS (
+ to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))
+ ) STORED`,
+ `CREATE INDEX IF NOT EXISTS idx_memories_search_tsv ON memories USING gin (search_tsv)`,
+];
+
// Local history also captures current-row replacements arriving through federation.
export const memoryHistoryDdl = [
`ALTER TABLE memory_links ADD COLUMN IF NOT EXISTS id UUID NOT NULL DEFAULT gen_random_uuid()`,
@@ -79,6 +90,7 @@ END$$`,
];
export const coreDdl = [
+ ...memorySearchDdl,
// Deliberately no FK: the bridge repairs a purged target on its next sync.
`CREATE TABLE IF NOT EXISTS brain_memory_links (
bridge_key TEXT PRIMARY KEY,
diff --git a/server/scripts/db-migrations/017-memory-search-vector.js b/server/scripts/db-migrations/017-memory-search-vector.js
new file mode 100644
index 0000000000..cdb49a94b1
--- /dev/null
+++ b/server/scripts/db-migrations/017-memory-search-vector.js
@@ -0,0 +1,9 @@
+// Backfill stored lexemes before retiring the expression index. Keeping both
+// would tokenize the same text twice on writes. The boot DDL may already have
+// added the new column/index; these statements also work on a pre-upgrade table.
+import { memorySearchDdl } from '../../lib/db/schema/core.js';
+
+export async function up(client) {
+ for (const statement of memorySearchDdl) await client.query(statement);
+ await client.query('DROP INDEX IF EXISTS idx_memories_fts');
+}
diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sql
index cacfd8b346..b9ee75083a 100644
--- a/server/scripts/init-db.sql
+++ b/server/scripts/init-db.sql
@@ -81,11 +81,12 @@ CREATE INDEX IF NOT EXISTS idx_memories_embedding
ON memories USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
--- Full-text search index (replaces BM25)
-CREATE INDEX IF NOT EXISTS idx_memories_fts
- ON memories USING gin (
+-- Reuse stored lexemes for both matching and ranking. ADD also backfills upgrades.
+ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_tsv tsvector
+ GENERATED ALWAYS AS (
to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))
- );
+ ) STORED;
+CREATE INDEX IF NOT EXISTS idx_memories_search_tsv ON memories USING gin (search_tsv);
-- Filtered queries
CREATE INDEX IF NOT EXISTS idx_memories_status ON memories (status);
diff --git a/server/services/memoryDB.db.test.js b/server/services/memoryDB.db.test.js
index 33ac102af9..85e05d96bc 100644
--- a/server/services/memoryDB.db.test.js
+++ b/server/services/memoryDB.db.test.js
@@ -31,6 +31,7 @@ import { checkHealth, ensureSchema, close, query, withTransaction } from '../lib
import { requireDbOrSkip } from '../lib/dbTestGate.js';
import { mockNoPeers, mockTestIdentity, mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';
import { DEFAULT_MEMORY_CONFIG } from './memoryConfig.js';
+import { up as upgradeMemorySearch } from '../scripts/db-migrations/017-memory-search-vector.js';
// Keep every filesystem side effect this module graph can reach (notifications
// pruning on approve/reject, the instance registry) inside a temp dir.
@@ -422,6 +423,61 @@ describe.skipIf(!runDb)('memoryDB listing filters (#3447)', () => {
// Search
// ---------------------------------------------------------------------------
+describe.skipIf(!runDb)('stored memory search lexemes', () => {
+ it('upgrades populated legacy tables idempotently and retires their expression index', async () => {
+ await withTransaction(async (client) => {
+ // A transaction-local shadow exercises the old layout without altering
+ // the shared test schema (or depending on which migrations CI applied).
+ await client.query('SET LOCAL search_path = pg_temp');
+ await client.query(`CREATE TEMP TABLE memories (
+ id integer PRIMARY KEY, content text NOT NULL, summary text
+ ) ON COMMIT DROP`);
+ await client.query("INSERT INTO memories VALUES (1, 'river otters', NULL), (2, '', 'quasar')");
+ await client.query(`CREATE INDEX idx_memories_fts ON memories USING gin (
+ to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))
+ )`);
+ await upgradeMemorySearch(client);
+ await upgradeMemorySearch(client);
+
+ const { rows } = await client.query(`SELECT id, search_tsv::text AS lexemes,
+ search_tsv = to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')) AS unchanged
+ FROM memories ORDER BY id`);
+ expect(rows).toEqual([
+ { id: 1, lexemes: "'otter':2 'river':1", unchanged: true },
+ { id: 2, lexemes: "'quasar':1", unchanged: true },
+ ]);
+ const indexes = await client.query(`SELECT relname FROM pg_class
+ WHERE relnamespace = pg_my_temp_schema() AND relkind = 'i' ORDER BY relname`);
+ expect(indexes.rows.map(row => row.relname)).toEqual(['idx_memories_search_tsv', 'memories_pkey']);
+ });
+ });
+
+ it('preserves text ranking and filters while refreshing lexemes after content and summary edits', async () => {
+ await resetMemories();
+ const strongest = await memoryDB.createMemory({ type: 'fact', content: 'Otter otter otter.', summary: 'Otter', sourceAppId: 'brain' });
+ const weaker = await memoryDB.createMemory({ type: 'preference', content: 'An otter beside the river.', summary: 'River', sourceAppId: 'studio' });
+ await memoryDB.createMemory({ type: 'fact', content: 'Otter otter otter otter.', status: 'archived' });
+
+ const legacy = await query(`SELECT id FROM memories WHERE status = 'active'
+ AND to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')) @@ websearch_to_tsquery('english', 'otters')
+ ORDER BY ts_rank(to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')),
+ websearch_to_tsquery('english', 'otters')) DESC`);
+ const searched = await memoryDB.hybridSearchMemories('otters', null);
+ expect(legacy.rows.map(row => row.id)).toEqual([strongest.id, weaker.id]);
+ expect(searched.memories.map(row => row.id)).toEqual(legacy.rows.map(row => row.id));
+ expect(searched.memories.map(row => row.ftsRank)).toEqual([1, 2]);
+ const scoped = await memoryDB.hybridSearchMemories('otters', null, { appId: '__not_brain', types: ['preference'], limit: 1 });
+ expect(scoped.memories.map(row => row.id)).toEqual([weaker.id]);
+
+ await memoryDB.updateMemory(strongest.id, { content: 'Lighthouse.', summary: 'Coast' });
+ expect((await memoryDB.hybridSearchMemories('otter', null)).memories.map(row => row.id)).toEqual([weaker.id]);
+ expect((await memoryDB.hybridSearchMemories('lighthouse', null)).memories.map(row => row.id)).toEqual([strongest.id]);
+ await memoryDB.updateMemory(strongest.id, { summary: 'Aurora' });
+ expect((await memoryDB.hybridSearchMemories('aurora', null)).memories.map(row => row.id)).toEqual([strongest.id]);
+ expect((await memoryDB.hybridSearchMemories('coast', null)).memories).toEqual([]);
+ });
+});
+
describe.skipIf(!runDb)('memoryDB search (#3447)', () => {
let quasar; // VEC_A, app 'brain'
let zebrafish; // VEC_NEAR_A, app 'studio'
diff --git a/server/services/memoryDB.js b/server/services/memoryDB.js
index 4414cd6994..05af4b1da5 100644
--- a/server/services/memoryDB.js
+++ b/server/services/memoryDB.js
@@ -627,12 +627,10 @@ export async function hybridSearchMemories(queryText, queryEmbedding, options =
if (queryText) {
const ftsResult = await query(
`SELECT id, type, category, tags, summary, importance, created_at, status, source_app_id,
- ts_rank(to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')),
- websearch_to_tsquery('english', $${paramIdx})) AS fts_score
+ ts_rank(search_tsv, websearch_to_tsquery('english', $${paramIdx})) AS fts_score
FROM memories
WHERE ${filterWhere}
- AND to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))
- @@ websearch_to_tsquery('english', $${paramIdx})
+ AND search_tsv @@ websearch_to_tsquery('english', $${paramIdx})
ORDER BY fts_score DESC
LIMIT $${paramIdx + 1}`,
[...filterParams, queryText, fetchLimit]