Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: index exact message identities once per sync

Adam Eivye43a36f827a4
diff --git a/server/lib/README.md b/server/lib/README.md
index 67369877f1..df63a9a0d9 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -830,7 +830,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `launchVideoValidation.js` | `validateLaunchVideoAssets` checks frozen launch composition text, storyboard reading time, duration, and poster position before rendering. |
-| `messageBrowserIdentity.js` | Shared Outlook row extraction, canonical row/member ingestion identity, conservative legacy matching, detail mapping, and identity-safe browser action scripts; exact unique targets and confirmed row removal. |
+| `messageBrowserIdentity.js` | Shared Outlook row extraction, canonical row/member ingestion identity, conservative legacy matching, per-sync exact identity indexing (`createBrowserMessageMatcher`), detail mapping, and identity-safe browser action scripts; exact unique targets and confirmed row removal. |
| `messageBrowserCompose.js` | Confirmed Outlook/Teams draft delivery through the authenticated browser tab, as pure helpers: `planDelivery` (deliverability + intended recipients/thread, run BEFORE a draft is claimed), `buildComposePhaseScript` (probe/open/fill/submit/discard CDP scripts, args as JSON), `classifySendOutcome` (confirmed only on a closed compose surface plus a fresh provider signal; anything else after the single submit click is `unknown`), `recipientsMatch`, `checkAccountIdentity`, and overridable `COMPOSE_SELECTOR_DEFAULTS`/`resolveComposeConfig`. Defaults are unvalidated against a live mailbox. |
| `messageTransport.js` | `MESSAGE_ACCOUNT_TRANSPORT` — the one table of what each message-account type delivers (`sendVia`, email-capable): `sendViaForAccountType`, `accountTypeCanSend`, and `pickEmailSenderAccount` (privacy email lanes prefer an API transport, never pick a Teams chat). |
diff --git a/server/lib/messageBrowserIdentity.js b/server/lib/messageBrowserIdentity.js
index 70f8643919..8f7c0ede23 100644
--- a/server/lib/messageBrowserIdentity.js
+++ b/server/lib/messageBrowserIdentity.js
@@ -204,6 +204,59 @@ export function matchBrowserMessage(messages, incoming, batch = [incoming]) {
const exact = messages.filter(message => messageIngestionIdentity(message) === key);
if (exact.length === 1 && batch.filter(message => messageIngestionIdentity(message) === key).length === 1) return exact[0];
if (exact.length) return null;
+ return matchCompatibleMessage(messages, incoming, batch);
+}
+
+/**
+ * One sync's exact-match index. The incoming batch stays unchanged; callers
+ * must refresh a cached record after mutating its identity. Keep every match
+ * in each bucket: duplicate identities are ambiguous, never last-write-wins.
+ * Legacy compatibility still uses the conservative scan below.
+ */
+export function createBrowserMessageMatcher(messages, batch) {
+ const byIdentity = new Map();
+ const knownKeys = new Map();
+ const batchCounts = new Map();
+ const add = (key, message) => {
+ const bucket = byIdentity.get(key) || [];
+ bucket.push(message);
+ byIdentity.set(key, bucket);
+ };
+ for (const message of messages) {
+ const key = messageIngestionIdentity(message);
+ knownKeys.set(message, key);
+ add(key, message);
+ }
+ for (const message of batch) {
+ const key = messageIngestionIdentity(message);
+ batchCounts.set(key, (batchCounts.get(key) || 0) + 1);
+ }
+ return {
+ match(incoming) {
+ const key = messageIngestionIdentity(incoming);
+ const exact = byIdentity.get(key);
+ if (exact?.length) return exact.length === 1 && batchCounts.get(key) === 1 ? exact[0] : null;
+ return matchCompatibleMessage(messages, incoming, batch);
+ },
+ refresh(message) {
+ const key = messageIngestionIdentity(message);
+ if (knownKeys.has(message) && knownKeys.get(message) === key) return;
+ let copies = 1;
+ if (knownKeys.has(message)) {
+ const oldKey = knownKeys.get(message);
+ const bucket = byIdentity.get(oldKey);
+ const remaining = bucket.filter(item => item !== message);
+ copies = bucket.length - remaining.length;
+ if (remaining.length) byIdentity.set(oldKey, remaining);
+ else byIdentity.delete(oldKey);
+ }
+ knownKeys.set(message, key);
+ for (let i = 0; i < copies; i++) add(key, message);
+ },
+ };
+}
+
+function matchCompatibleMessage(messages, incoming, batch) {
const compatible = (message, candidate) => {
const memberEvidence = message.providerRowId && message.providerRowId === candidate.providerRowId
&& candidate.browserMessageKind === 'member'
diff --git a/server/services/messageSync.js b/server/services/messageSync.js
index a7dc804a10..d08d9a9f3e 100644
--- a/server/services/messageSync.js
+++ b/server/services/messageSync.js
@@ -1,6 +1,6 @@
import { messageLogError } from '../lib/messageLogError.js';
-import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';
+import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, createBrowserMessageMatcher, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';
import { join } from 'path';
import { atomicWrite, ensureDir, filterBySearch as genericFilterBySearch, PATHS, safeDate, safeJSONParse, UUID_RE, tryReadFile } from '../lib/fileUtils.js';
import { getAccount, updateSyncStatus, markSentIngested, updateSendAsAliases } from './messageAccounts.js';
@@ -230,9 +230,10 @@ export async function syncAccount(accountId, io, options = {}) {
// Preserve the DB identity of unambiguous pre-upgrade cache entries.
retainLegacyActivityIdentities(cache.messages);
+ const matcher = createBrowserMessageMatcher(cache.messages, newMessages);
const uniqueNew = [];
for (const msg of newMessages) {
- const existing = matchBrowserMessage(cache.messages, msg, newMessages);
+ const existing = matcher.match(msg);
if (!msg.externalId || !existing) {
uniqueNew.push(msg);
} else {
@@ -258,6 +259,7 @@ export async function syncAccount(accountId, io, options = {}) {
}
// Set threadId if newly available
if (msg.threadId && !existing.threadId) existing.threadId = msg.threadId;
+ matcher.refresh(existing);
}
}
cache.messages.push(...uniqueNew);
diff --git a/server/services/messageSync.test.js b/server/services/messageSync.test.js
index 234ba41ac7..8a6b880bc7 100644
--- a/server/services/messageSync.test.js
+++ b/server/services/messageSync.test.js
@@ -628,6 +628,46 @@ describe('syncAccount', () => {
]);
});
+ it('replays a full cached sync with linear identity reads and preserves local IDs and evaluations', async () => {
+ const count = 500;
+ const messages = Array.from({ length: count }, (_, i) => ({
+ id: `local-${i}`, externalId: `summary-${i}`, providerRowId: `row-${i}`,
+ browserMessageKind: 'member', date: '2026-01-01T12:00:00Z',
+ from: { email: '[email protected]' }, evaluation: { action: 'review' }, activityIdentity: `activity-${i}`,
+ }));
+ let dateReads = 0;
+ const incoming = messages.map(({ evaluation: _evaluation, ...message }) => ({
+ ...message, id: `incoming-${message.id}`, isRead: true,
+ get date() { dateReads++; return message.date; },
+ }));
+ getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });
+ readFile.mockResolvedValue(JSON.stringify({ messages }));
+ syncPlaywright.mockResolvedValue({ status: 'success', messages: incoming });
+
+ const result = await syncAccount(VALID_UUID, mockIo);
+
+ expect(result).toMatchObject({ newMessages: 0, total: count });
+ expect(atomicWrite.mock.calls[0][1].messages).toEqual(messages.map(message => ({ ...message, isRead: true })));
+ // Count actual identity input reads rather than a machine-dependent deadline.
+ // Re-scanning the incoming batch for every message takes >250,000 reads.
+ expect(dateReads).toBeLessThanOrEqual(count * 8);
+ });
+
+ it('reindexes a legacy row when a full body makes it a member during the same sync', async () => {
+ const identity = { externalId: 'summary', providerRowId: 'row', date: '2026-01-01T12:00:00Z', from: { email: '[email protected]' } };
+ getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });
+ readFile.mockResolvedValue(JSON.stringify({ messages: [{ ...identity, id: 'local', threadId: 'thread', bodyFull: false }] }));
+ syncPlaywright.mockResolvedValue({ status: 'success', messages: [
+ { ...identity, id: 'row-update', bodyFull: true, bodyText: 'First body' },
+ { ...identity, id: 'member-update', externalId: 'member-summary', browserMessageKind: 'member', bodyFull: true, bodyText: 'Member body' },
+ ] });
+
+ expect(await syncAccount(VALID_UUID, mockIo)).toMatchObject({ newMessages: 0, total: 1 });
+ expect(atomicWrite.mock.calls[0][1].messages).toEqual([expect.objectContaining({
+ id: 'local', externalId: 'member-summary', browserMessageKind: 'member', bodyText: 'Member body',
+ })]);
+ });
+
it('retains a known row ID when a later unambiguous extraction omits it', async () => {
getAccount.mockResolvedValue({ id: VALID_UUID, type: 'outlook', enabled: true });
readFile.mockResolvedValue(JSON.stringify({ messages: [{ id: 'local', externalId: 'summary', providerRowId: 'stable-row' }] }));