Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reuse stored lexemes for hybrid memory search
Adam Eivyad63adc237b9
diff --git a/server/lib/db.ddlParity.test.js b/server/lib/db.ddlParity.test.jsindex 4cddee04f7..4cc0017edd 100644--- a/server/lib/db.ddlParity.test.js+++ b/server/lib/db.ddlParity.test.js@@ -26,6 +26,7 @@import { describe, it, expect } from 'vitest';import { FTS_PAYLOAD_FIELDS } from './catalogTypes.js';import { syncFeedDdl, syncFeedTables, buildSyncFeedTriggers } from './db/schema/syncFeed.js';+import { memorySearchDdl } from './db/schema/core.js';import { readFileSync, readdirSync } from 'node:fs';import { fileURLToPath } from 'node:url';import { dirname, join } from 'node:path';@@ -275,6 +276,15 @@ describe('DDL parity (init-db.sql ↔ db/schema ensureSchema)', () => {// ── Targeted pins: contracts a two-sided set comparison cannot see ──────────+ // memories is SQL_ONLY_TOLERATED, so the shared-table comparison cannot+ // catch fresh-install drift in its generated expression or matching index.+ it('stores the same memory search lexemes on fresh installs and upgrades', () => {+ for (const statement of memorySearchDdl) {+ expect(normalize(INIT_SQL)).toContain(normalize(statement));+ }+ expect(sqlIndexes.get('memories').get('idx_memories_search_tsv')).toBe('using gin (search_tsv)');+ });+// The operator-action ledger (#5594) and the human-activity timeline (#2150)// rely on their dedupe index as the ON CONFLICT idempotency contract, and on// the composites (#5715) to keep a source-scoped read off a full happened_atdiff --git a/server/lib/db/schema/README.md b/server/lib/db/schema/README.mdindex dda0bbc440..58ceeab5c4 100644--- a/server/lib/db/schema/README.md+++ b/server/lib/db/schema/README.md@@ -23,7 +23,7 @@ init-db.sql, each with a one-line reason.| Module | Export(s) | Domain ||---|---|---|-| `core.js` | `coreDdl`, `restoreReceiptsDdl` | Memory sync columns + the `schema_migrations` version tracker + machine-local `restore_receipts` (snapshot restore replay receipts, #9725) |+| `core.js` | `coreDdl`, `memorySearchDdl`, `restoreReceiptsDdl` | Memory search lexemes and sync columns + the `schema_migrations` version tracker + machine-local `restore_receipts` (snapshot restore replay receipts, #9725) || `tribe.js` | `tribeDdl` | Tribe CRM — people, network-scoped identities (`tribe_identities`, the durable-handle truth table behind Beeper linking), touchpoints, memory links (machine-local) || `humanActivity.js` | `humanActivityDdl` | Human-activity timeline event store (machine-local) || `post.js` | `postDdl` | MeatSpace POST normalized runs and attempts (machine-local) |diff --git a/server/lib/db/schema/core.js b/server/lib/db/schema/core.jsindex 0d0e4b8137..fb4d4146fd 100644--- a/server/lib/db/schema/core.js+++ b/server/lib/db/schema/core.js@@ -18,6 +18,17 @@ export const restoreReceiptsDdl = [)`,];+// Store the same lexemes the old expression index used so ranking does not+// tokenize every matching memory body again on every agent/search request.+// PostgreSQL backfills existing rows and maintains the value after text edits.+export const memorySearchDdl = [+ `ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_tsv tsvector+ GENERATED ALWAYS AS (+ to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))+ ) STORED`,+ `CREATE INDEX IF NOT EXISTS idx_memories_search_tsv ON memories USING gin (search_tsv)`,+];+// Local history also captures current-row replacements arriving through federation.export const memoryHistoryDdl = [`ALTER TABLE memory_links ADD COLUMN IF NOT EXISTS id UUID NOT NULL DEFAULT gen_random_uuid()`,@@ -79,6 +90,7 @@ END$$`,];export const coreDdl = [+ ...memorySearchDdl,// Deliberately no FK: the bridge repairs a purged target on its next sync.`CREATE TABLE IF NOT EXISTS brain_memory_links (bridge_key TEXT PRIMARY KEY,diff --git a/server/scripts/db-migrations/017-memory-search-vector.js b/server/scripts/db-migrations/017-memory-search-vector.jsnew file mode 100644index 0000000000..cdb49a94b1--- /dev/null+++ b/server/scripts/db-migrations/017-memory-search-vector.js@@ -0,0 +1,9 @@+// Backfill stored lexemes before retiring the expression index. Keeping both+// would tokenize the same text twice on writes. The boot DDL may already have+// added the new column/index; these statements also work on a pre-upgrade table.+import { memorySearchDdl } from '../../lib/db/schema/core.js';++export async function up(client) {+ for (const statement of memorySearchDdl) await client.query(statement);+ await client.query('DROP INDEX IF EXISTS idx_memories_fts');+}diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sqlindex cacfd8b346..b9ee75083a 100644--- a/server/scripts/init-db.sql+++ b/server/scripts/init-db.sql@@ -81,11 +81,12 @@ CREATE INDEX IF NOT EXISTS idx_memories_embeddingON memories USING hnsw (embedding vector_cosine_ops)WITH (m = 16, ef_construction = 64);--- Full-text search index (replaces BM25)-CREATE INDEX IF NOT EXISTS idx_memories_fts- ON memories USING gin (+-- Reuse stored lexemes for both matching and ranking. ADD also backfills upgrades.+ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_tsv tsvector+ GENERATED ALWAYS AS (to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))- );+ ) STORED;+CREATE INDEX IF NOT EXISTS idx_memories_search_tsv ON memories USING gin (search_tsv);-- Filtered queriesCREATE INDEX IF NOT EXISTS idx_memories_status ON memories (status);diff --git a/server/services/memoryDB.db.test.js b/server/services/memoryDB.db.test.jsindex 33ac102af9..85e05d96bc 100644--- a/server/services/memoryDB.db.test.js+++ b/server/services/memoryDB.db.test.js@@ -31,6 +31,7 @@ import { checkHealth, ensureSchema, close, query, withTransaction } from '../libimport { requireDbOrSkip } from '../lib/dbTestGate.js';import { mockNoPeers, mockTestIdentity, mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';import { DEFAULT_MEMORY_CONFIG } from './memoryConfig.js';+import { up as upgradeMemorySearch } from '../scripts/db-migrations/017-memory-search-vector.js';// Keep every filesystem side effect this module graph can reach (notifications// pruning on approve/reject, the instance registry) inside a temp dir.@@ -422,6 +423,61 @@ describe.skipIf(!runDb)('memoryDB listing filters (#3447)', () => {// Search// ---------------------------------------------------------------------------+describe.skipIf(!runDb)('stored memory search lexemes', () => {+ it('upgrades populated legacy tables idempotently and retires their expression index', async () => {+ await withTransaction(async (client) => {+ // A transaction-local shadow exercises the old layout without altering+ // the shared test schema (or depending on which migrations CI applied).+ await client.query('SET LOCAL search_path = pg_temp');+ await client.query(`CREATE TEMP TABLE memories (+ id integer PRIMARY KEY, content text NOT NULL, summary text+ ) ON COMMIT DROP`);+ await client.query("INSERT INTO memories VALUES (1, 'river otters', NULL), (2, '', 'quasar')");+ await client.query(`CREATE INDEX idx_memories_fts ON memories USING gin (+ to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))+ )`);+ await upgradeMemorySearch(client);+ await upgradeMemorySearch(client);++ const { rows } = await client.query(`SELECT id, search_tsv::text AS lexemes,+ search_tsv = to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')) AS unchanged+ FROM memories ORDER BY id`);+ expect(rows).toEqual([+ { id: 1, lexemes: "'otter':2 'river':1", unchanged: true },+ { id: 2, lexemes: "'quasar':1", unchanged: true },+ ]);+ const indexes = await client.query(`SELECT relname FROM pg_class+ WHERE relnamespace = pg_my_temp_schema() AND relkind = 'i' ORDER BY relname`);+ expect(indexes.rows.map(row => row.relname)).toEqual(['idx_memories_search_tsv', 'memories_pkey']);+ });+ });++ it('preserves text ranking and filters while refreshing lexemes after content and summary edits', async () => {+ await resetMemories();+ const strongest = await memoryDB.createMemory({ type: 'fact', content: 'Otter otter otter.', summary: 'Otter', sourceAppId: 'brain' });+ const weaker = await memoryDB.createMemory({ type: 'preference', content: 'An otter beside the river.', summary: 'River', sourceAppId: 'studio' });+ await memoryDB.createMemory({ type: 'fact', content: 'Otter otter otter otter.', status: 'archived' });++ const legacy = await query(`SELECT id FROM memories WHERE status = 'active'+ AND to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')) @@ websearch_to_tsquery('english', 'otters')+ ORDER BY ts_rank(to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')),+ websearch_to_tsquery('english', 'otters')) DESC`);+ const searched = await memoryDB.hybridSearchMemories('otters', null);+ expect(legacy.rows.map(row => row.id)).toEqual([strongest.id, weaker.id]);+ expect(searched.memories.map(row => row.id)).toEqual(legacy.rows.map(row => row.id));+ expect(searched.memories.map(row => row.ftsRank)).toEqual([1, 2]);+ const scoped = await memoryDB.hybridSearchMemories('otters', null, { appId: '__not_brain', types: ['preference'], limit: 1 });+ expect(scoped.memories.map(row => row.id)).toEqual([weaker.id]);++ await memoryDB.updateMemory(strongest.id, { content: 'Lighthouse.', summary: 'Coast' });+ expect((await memoryDB.hybridSearchMemories('otter', null)).memories.map(row => row.id)).toEqual([weaker.id]);+ expect((await memoryDB.hybridSearchMemories('lighthouse', null)).memories.map(row => row.id)).toEqual([strongest.id]);+ await memoryDB.updateMemory(strongest.id, { summary: 'Aurora' });+ expect((await memoryDB.hybridSearchMemories('aurora', null)).memories.map(row => row.id)).toEqual([strongest.id]);+ expect((await memoryDB.hybridSearchMemories('coast', null)).memories).toEqual([]);+ });+});+describe.skipIf(!runDb)('memoryDB search (#3447)', () => {let quasar; // VEC_A, app 'brain'let zebrafish; // VEC_NEAR_A, app 'studio'diff --git a/server/services/memoryDB.js b/server/services/memoryDB.jsindex 4414cd6994..05af4b1da5 100644--- a/server/services/memoryDB.js+++ b/server/services/memoryDB.js@@ -627,12 +627,10 @@ export async function hybridSearchMemories(queryText, queryEmbedding, options =if (queryText) {const ftsResult = await query(`SELECT id, type, category, tags, summary, importance, created_at, status, source_app_id,- ts_rank(to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, '')),- websearch_to_tsquery('english', $${paramIdx})) AS fts_score+ ts_rank(search_tsv, websearch_to_tsquery('english', $${paramIdx})) AS fts_scoreFROM memoriesWHERE ${filterWhere}- AND to_tsvector('english', coalesce(content, '') || ' ' || coalesce(summary, ''))- @@ websearch_to_tsquery('english', $${paramIdx})+ AND search_tsv @@ websearch_to_tsquery('english', $${paramIdx})ORDER BY fts_score DESCLIMIT $${paramIdx + 1}`,[...filterParams, queryText, fetchLimit]