Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: index exact message identities once per sync
Adam Eivye43a36f827a4
diff --git a/server/lib/README.md b/server/lib/README.mdindex 67369877f1..df63a9a0d9 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -830,7 +830,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `launchVideoValidation.js` | `validateLaunchVideoAssets` checks frozen launch composition text, storyboard reading time, duration, and poster position before rendering. |-| `messageBrowserIdentity.js` | Shared Outlook row extraction, canonical row/member ingestion identity, conservative legacy matching, detail mapping, and identity-safe browser action scripts; exact unique targets and confirmed row removal. |+| `messageBrowserIdentity.js` | Shared Outlook row extraction, canonical row/member ingestion identity, conservative legacy matching, per-sync exact identity indexing (`createBrowserMessageMatcher`), detail mapping, and identity-safe browser action scripts; exact unique targets and confirmed row removal. || `messageBrowserCompose.js` | Confirmed Outlook/Teams draft delivery through the authenticated browser tab, as pure helpers: `planDelivery` (deliverability + intended recipients/thread, run BEFORE a draft is claimed), `buildComposePhaseScript` (probe/open/fill/submit/discard CDP scripts, args as JSON), `classifySendOutcome` (confirmed only on a closed compose surface plus a fresh provider signal; anything else after the single submit click is `unknown`), `recipientsMatch`, `checkAccountIdentity`, and overridable `COMPOSE_SELECTOR_DEFAULTS`/`resolveComposeConfig`. Defaults are unvalidated against a live mailbox. || `messageTransport.js` | `MESSAGE_ACCOUNT_TRANSPORT` — the one table of what each message-account type delivers (`sendVia`, email-capable): `sendViaForAccountType`, `accountTypeCanSend`, and `pickEmailSenderAccount` (privacy email lanes prefer an API transport, never pick a Teams chat). |diff --git a/server/lib/messageBrowserIdentity.js b/server/lib/messageBrowserIdentity.jsindex 70f8643919..8f7c0ede23 100644--- a/server/lib/messageBrowserIdentity.js+++ b/server/lib/messageBrowserIdentity.js@@ -204,6 +204,59 @@ export function matchBrowserMessage(messages, incoming, batch = [incoming]) {const exact = messages.filter(message => messageIngestionIdentity(message) === key);if (exact.length === 1 && batch.filter(message => messageIngestionIdentity(message) === key).length === 1) return exact[0];if (exact.length) return null;+ return matchCompatibleMessage(messages, incoming, batch);+}++/**+ * One sync's exact-match index. The incoming batch stays unchanged; callers+ * must refresh a cached record after mutating its identity. Keep every match+ * in each bucket: duplicate identities are ambiguous, never last-write-wins.+ * Legacy compatibility still uses the conservative scan below.+ */+export function createBrowserMessageMatcher(messages, batch) {+ const byIdentity = new Map();+ const knownKeys = new Map();+ const batchCounts = new Map();+ const add = (key, message) => {+ const bucket = byIdentity.get(key) || [];+ bucket.push(message);+ byIdentity.set(key, bucket);+ };+ for (const message of messages) {+ const key = messageIngestionIdentity(message);+ knownKeys.set(message, key);+ add(key, message);+ }+ for (const message of batch) {+ const key = messageIngestionIdentity(message);+ batchCounts.set(key, (batchCounts.get(key) || 0) + 1);+ }+ return {+ match(incoming) {+ const key = messageIngestionIdentity(incoming);+ const exact = byIdentity.get(key);+ if (exact?.length) return exact.length === 1 && batchCounts.get(key) === 1 ? exact[0] : null;+ return matchCompatibleMessage(messages, incoming, batch);+ },+ refresh(message) {+ const key = messageIngestionIdentity(message);+ if (knownKeys.has(message) && knownKeys.get(message) === key) return;+ let copies = 1;+ if (knownKeys.has(message)) {+ const oldKey = knownKeys.get(message);+ const bucket = byIdentity.get(oldKey);+ const remaining = bucket.filter(item => item !== message);+ copies = bucket.length - remaining.length;+ if (remaining.length) byIdentity.set(oldKey, remaining);+ else byIdentity.delete(oldKey);+ }+ knownKeys.set(message, key);+ for (let i = 0; i < copies; i++) add(key, message);+ },+ };+}++function matchCompatibleMessage(messages, incoming, batch) {const compatible = (message, candidate) => {const memberEvidence = message.providerRowId && message.providerRowId === candidate.providerRowId&& candidate.browserMessageKind === 'member'diff --git a/server/services/messageSync.js b/server/services/messageSync.jsindex a7dc804a10..d08d9a9f3e 100644--- a/server/services/messageSync.js+++ b/server/services/messageSync.js@@ -1,6 +1,6 @@import { messageLogError } from '../lib/messageLogError.js';-import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';+import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, createBrowserMessageMatcher, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';import { join } from 'path';import { atomicWrite, ensureDir, filterBySearch as genericFilterBySearch, PATHS, safeDate, safeJSONParse, UUID_RE, tryReadFile } from '../lib/fileUtils.js';import { getAccount, updateSyncStatus, markSentIngested, updateSendAsAliases } from './messageAccounts.js';@@ -230,9 +230,10 @@ export async function syncAccount(accountId, io, options = {}) {// Preserve the DB identity of unambiguous pre-upgrade cache entries.retainLegacyActivityIdentities(cache.messages);+ const matcher = createBrowserMessageMatcher(cache.messages, newMessages);const uniqueNew = [];for (const msg of newMessages) {- const existing = matchBrowserMessage(cache.messages, msg, newMessages);+ const existing = matcher.match(msg);if (!msg.externalId || !existing) {uniqueNew.push(msg);} else {@@ -258,6 +259,7 @@ export async function syncAccount(accountId, io, options = {}) {}// Set threadId if newly availableif (msg.threadId && !existing.threadId) existing.threadId = msg.threadId;+ matcher.refresh(existing);}}cache.messages.push(...uniqueNew);diff --git a/server/services/messageSync.test.js b/server/services/messageSync.test.jsindex 234ba41ac7..8a6b880bc7 100644--- a/server/services/messageSync.test.js+++ b/server/services/messageSync.test.js@@ -628,6 +628,46 @@ describe('syncAccount', () => {]);});+ it('replays a full cached sync with linear identity reads and preserves local IDs and evaluations', async () => {+ const count = 500;+ const messages = Array.from({ length: count }, (_, i) => ({+ id: `local-${i}`, externalId: `summary-${i}`, providerRowId: `row-${i}`,+ browserMessageKind: 'member', date: '2026-01-01T12:00:00Z',+ from: { email: '[email protected]' }, evaluation: { action: 'review' }, activityIdentity: `activity-${i}`,+ }));+ let dateReads = 0;+ const incoming = messages.map(({ evaluation: _evaluation, ...message }) => ({+ ...message, id: `incoming-${message.id}`, isRead: true,+ get date() { dateReads++; return message.date; },+ }));+ getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });+ readFile.mockResolvedValue(JSON.stringify({ messages }));+ syncPlaywright.mockResolvedValue({ status: 'success', messages: incoming });++ const result = await syncAccount(VALID_UUID, mockIo);++ expect(result).toMatchObject({ newMessages: 0, total: count });+ expect(atomicWrite.mock.calls[0][1].messages).toEqual(messages.map(message => ({ ...message, isRead: true })));+ // Count actual identity input reads rather than a machine-dependent deadline.+ // Re-scanning the incoming batch for every message takes >250,000 reads.+ expect(dateReads).toBeLessThanOrEqual(count * 8);+ });++ it('reindexes a legacy row when a full body makes it a member during the same sync', async () => {+ const identity = { externalId: 'summary', providerRowId: 'row', date: '2026-01-01T12:00:00Z', from: { email: '[email protected]' } };+ getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });+ readFile.mockResolvedValue(JSON.stringify({ messages: [{ ...identity, id: 'local', threadId: 'thread', bodyFull: false }] }));+ syncPlaywright.mockResolvedValue({ status: 'success', messages: [+ { ...identity, id: 'row-update', bodyFull: true, bodyText: 'First body' },+ { ...identity, id: 'member-update', externalId: 'member-summary', browserMessageKind: 'member', bodyFull: true, bodyText: 'Member body' },+ ] });++ expect(await syncAccount(VALID_UUID, mockIo)).toMatchObject({ newMessages: 0, total: 1 });+ expect(atomicWrite.mock.calls[0][1].messages).toEqual([expect.objectContaining({+ id: 'local', externalId: 'member-summary', browserMessageKind: 'member', bodyText: 'Member body',+ })]);+ });+it('retains a known row ID when a later unambiguous extraction omits it', async () => {getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });readFile.mockResolvedValue(JSON.stringify({ messages: [{ id: 'local', externalId: 'summary', providerRowId: 'stable-row' }] }));