Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: reuse compact inbox summaries across unchanged pages

Adam Eivyaa05f3b9dc75
diff --git a/server/routes/messageInboxRead.js b/server/routes/messageInboxRead.js
index 5b8988bef0..c89f216be7 100644
--- a/server/routes/messageInboxRead.js
+++ b/server/routes/messageInboxRead.js
@@ -14,33 +14,11 @@ export const messageInboxRead = asyncHandler(async (req, res) => {
const result = await messageSync.getMessages({
accountId,
search,
+ summary: req.query.summary === 'true',
limit: parsedLimit,
offset: parsedOffset
});
- // Summary is opt-in so older clients and automation retain the full-record
- // contract. Search and pagination above still run against the full cache.
- res.json(req.query.summary === 'true' ? {
- ...result,
- messages: result.messages.map(message => ({
- id: message.id,
- accountId: message.accountId,
- externalId: message.externalId,
- threadId: message.threadId,
- subject: message.subject,
- from: message.from && { name: message.from.name, email: message.from.email },
- date: message.date,
- source: message.source,
- isRead: message.isRead,
- isUnread: message.isUnread,
- isPinned: message.isPinned,
- isFlagged: message.isFlagged,
- preview: typeof message.bodyText === 'string' ? message.bodyText.slice(0, 100) : '',
- evaluation: message.evaluation && {
- action: message.evaluation.action,
- priority: message.evaluation.priority,
- },
- })),
- } : result);
+ res.json(result);
});
diff --git a/server/routes/messageInboxRead.test.js b/server/routes/messageInboxRead.test.js
new file mode 100644
index 0000000000..4f7484f618
--- /dev/null
+++ b/server/routes/messageInboxRead.test.js
@@ -0,0 +1,178 @@
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+import { mkdir, readFile, rm, stat, utimes, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { makePathsProxy, lazyTempDataRoot, cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';
+import { request } from '../lib/testHelper.js';
+import { ACCOUNT_IDS, messageFixture } from '../../scripts/perf/collectionFixtureData.js';
+
+const permissions = vi.hoisted(() => ({ denied: false }));
+vi.mock('node:fs/promises', async original => {
+ const actual = await original();
+ return { ...actual, access: async (...args) => {
+ if (permissions.denied) throw Object.assign(new Error('Synthetic access denial'), { code: 'EACCES' });
+ return actual.access(...args);
+ } };
+});
+vi.mock('../lib/fileUtils.js', async original => {
+ const actual = await original();
+ const { PATHS } = makePathsProxy(actual, { dataRoot: () => lazyTempDataRoot('portos-inbox-summary-') });
+ return { ...actual, PATHS, tryReadFile: vi.fn((...args) => permissions.denied ? null : actual.tryReadFile(...args)) };
+});
+
+import { PATHS, atomicWrite, tryReadFile } from '../lib/fileUtils.js';
+import { errorMiddleware } from '../lib/errorHandler.js';
+import { deleteCache, getMessages, removeMessageFromCache, updateMessageEvaluations } from '../services/messageSync.js';
+import { messageInboxRead, messageDetailRead } from './messageInboxRead.js';
+
+const cacheDir = join(PATHS.messages, 'cache');
+const cachePath = id => join(cacheDir, id + '.json');
+const app = express();
+app.get('/inbox', messageInboxRead);
+app.get('/:accountId/:messageId', messageDetailRead);
+app.use(errorMiddleware);
+const inbox = query => request(app).get('/inbox?summary=true' + (query ? '&' + query : ''));
+const seed = async (count = 30) => {
+ for (const [account, id] of ACCOUNT_IDS.entries()) {
+ await writeFile(cachePath(id), JSON.stringify({ messages: Array.from({ length: count }, (_, i) => messageFixture(id, i + account * count)) }));
+ }
+};
+
+beforeEach(async () => {
+ permissions.denied = false;
+ vi.mocked(tryReadFile).mockClear();
+ await rm(PATHS.messages, { recursive: true, force: true });
+ await mkdir(cacheDir, { recursive: true });
+});
+afterAll(cleanupTempDataRoots);
+
+describe('inbox summary reads through the production HTTP/service boundary', () => {
+ it('shares first reads and reuses compact rows across pages without hydrating bodies', async () => {
+ await seed();
+ const [first, duplicate] = await Promise.all([inbox(), inbox()]);
+ const second = await inbox('offset=50');
+ expect(first.status).toBe(200);
+ expect(duplicate.body).toEqual(first.body);
+ expect(first.body.total).toBe(60);
+ expect(first.body.messages).toHaveLength(50);
+ expect(second.body.messages).toHaveLength(10);
+ expect(new Set([...first.body.messages, ...second.body.messages].map(message => message.id)).size).toBe(60);
+ expect(tryReadFile).toHaveBeenCalledTimes(2);
+ expect(JSON.stringify(first.body).length).toBeLessThan(26000);
+ expect(first.body.messages[0].preview).toHaveLength(100);
+ expect(first.body.messages[0]).not.toHaveProperty('bodyText');
+ expect(first.body.messages[0]).not.toHaveProperty('bodyHtml');
+ expect(first.body.messages[0].evaluation).not.toHaveProperty('reasoning');
+ });
+
+ it('keeps full-body search, legacy reads, lazy details and account filters intact', async () => {
+ const id = ACCOUNT_IDS[0];
+ const message = { ...messageFixture(id, 1), bodyText: 'x'.repeat(500) + 'unique-body-match',
+ from: { name: 'Example Sender', email: '[email protected]', detail: 'private detail' } };
+ await writeFile(cachePath(id), JSON.stringify({ messages: [message] }));
+ await inbox();
+ const found = await inbox('search=unique-body-match&accountId=' + id);
+ expect(found.body).toMatchObject({ total: 1, messages: [{ id: message.id, preview: 'x'.repeat(100) }] });
+ expect(found.body.messages[0].from).toEqual({ name: 'Example Sender', email: '[email protected]' });
+ expect(found.body.messages[0].evaluation).toEqual({ action: 'review', priority: 'normal' });
+ expect((await request(app).get('/inbox')).body.messages).toEqual([message]);
+ expect((await request(app).get(`/${id}/${message.id}`)).body).toEqual(message);
+ expect((await inbox('accountId=' + ACCOUNT_IDS[1])).body).toEqual({ messages: [], total: 0 });
+ expect(tryReadFile).toHaveBeenCalledTimes(5);
+ });
+
+ it('observes same-size edits, external replacements, evaluation writes and deletions immediately', async () => {
+ const id = ACCOUNT_IDS[0];
+ let message = { ...messageFixture(id, 1), bodyText: 'Old' };
+ const save = () => writeFile(cachePath(id), JSON.stringify({ messages: [message] }));
+ await save();
+ expect((await inbox()).body.messages[0].preview).toBe('Old');
+ const before = await stat(cachePath(id));
+ message = { ...message, bodyText: 'New' };
+ await save();
+ await utimes(cachePath(id), before.atime, before.mtime);
+ expect((await inbox()).body.messages[0].preview).toBe('New');
+ message = { ...message, bodyText: 'Replacement' };
+ await atomicWrite(cachePath(id), { messages: [message] });
+ expect((await inbox()).body.messages[0].preview).toBe('Replacement');
+ await updateMessageEvaluations({ [message.id]: { action: 'keep', priority: 'low', reasoning: 'Full detail' } });
+ expect((await inbox()).body.messages[0].evaluation).toEqual({ action: 'keep', priority: 'low' });
+ await removeMessageFromCache(id, message.id);
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ await deleteCache(id);
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ });
+
+ it('does not publish a stale first read over a concurrent committed evaluation', async () => {
+ const id = ACCOUNT_IDS[0];
+ const message = messageFixture(id, 1);
+ await writeFile(cachePath(id), JSON.stringify({ messages: [message] }));
+ let started;
+ let release;
+ const reading = new Promise(resolve => { started = resolve; });
+ const held = new Promise(resolve => { release = resolve; });
+ vi.mocked(tryReadFile).mockImplementationOnce(async file => {
+ const content = await readFile(file, 'utf8');
+ started();
+ await held;
+ return content;
+ });
+ const first = getMessages({ accountId: id, summary: true });
+ await reading;
+ const joined = getMessages({ accountId: id, summary: true });
+ try {
+ await updateMessageEvaluations({ [message.id]: { action: 'keep', priority: 'high' } });
+ } finally { release(); }
+ expect((await first).messages[0].evaluation).toEqual({ action: 'keep', priority: 'high' });
+ expect(await joined).toEqual(await first);
+ expect((await inbox()).body.messages[0].evaluation).toEqual({ action: 'keep', priority: 'high' });
+ expect(tryReadFile).toHaveBeenCalledTimes(3);
+ });
+
+ it('never serves old summaries after unreadable, corrupt, missing or empty caches', async () => {
+ const id = ACCOUNT_IDS[0];
+ const restore = () => writeFile(cachePath(id), JSON.stringify({ messages: [messageFixture(id, 1)] }));
+ await restore();
+ expect((await inbox()).body.total).toBe(1);
+ permissions.denied = true;
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ permissions.denied = false;
+ expect((await inbox()).body.total).toBe(1);
+ await writeFile(cachePath(id), '{invalid');
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ await restore();
+ expect((await inbox()).body.total).toBe(1);
+ await rm(cachePath(id));
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ await writeFile(cachePath(id), JSON.stringify({ messages: [] }));
+ await inbox();
+ vi.mocked(tryReadFile).mockClear();
+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });
+ expect(tryReadFile).not.toHaveBeenCalled();
+ });
+
+ it.skipIf(process.env.PORTOS_INBOX_BENCHMARK !== '1')('measures repeated 4000-message HTTP pages without timing gates', async () => {
+ await seed(2000);
+ const bytes = (await Promise.all(ACCOUNT_IDS.map(async id => (await stat(cachePath(id))).size))).reduce((a, b) => a + b);
+ const measure = async summary => {
+ vi.mocked(tryReadFile).mockClear();
+ const durations = [];
+ for (let page = 0; page < 8; page++) {
+ const start = performance.now();
+ const response = await request(app).get(`/inbox?summary=${summary}&offset=${page * 50}`);
+ expect(response.body.total).toBe(4000);
+ expect(response.body.messages).toHaveLength(50);
+ durations.push(performance.now() - start);
+ }
+ return { reads: vi.mocked(tryReadFile).mock.calls.length,
+ medianWarmMs: durations.slice(1).sort((a, b) => a - b)[3] };
+ };
+ const legacy = await measure(false);
+ const summary = await measure(true);
+ expect(legacy.reads).toBe(16);
+ expect(summary.reads).toBe(2);
+ process.stdout.write(JSON.stringify({ synthetic: true, messages: 4000, pageLimit: 50, pages: 8,
+ bytesPerFullRead: bytes, legacy, summary, eliminatedReadBytes: bytes * 7 }) + '\n');
+ });
+});
diff --git a/server/routes/messages.test.js b/server/routes/messages.test.js
index a09ad70f97..99f8c23c0e 100644
--- a/server/routes/messages.test.js
+++ b/server/routes/messages.test.js
@@ -366,39 +366,6 @@ describe('Messages Routes', () => {
expect(response.body.total).toBe(1);
});
- it('keeps large bodies out of summary pages while preserving legacy and detail reads', async () => {
- const message = {
- id: 'msg-large', accountId: VALID_UUID, subject: 'Synthetic large message',
- from: { name: 'Example Sender', email: '[email protected]', extra: 'detail' },
- bodyText: 'Synthetic message body. '.repeat(10000),
- bodyHtml: '<p>Synthetic HTML</p>'.repeat(10000),
- attachments: [{ content: 'synthetic attachment'.repeat(1000) }],
- evaluation: { action: 'reply', priority: 'high', reasoning: 'detail'.repeat(1000) },
- };
- messageSync.getMessages.mockResolvedValue({ messages: Array(50).fill(message), total: 5000 });
- messageSync.getMessage.mockResolvedValue(message);
-
- const summary = await request(app).get('/api/messages/inbox?summary=true&search=body&offset=50');
- expect(summary.status).toBe(200);
- expect(summary.body.total).toBe(5000);
- expect(summary.body.messages).toHaveLength(50);
- expect(summary.body.messages[0]).toEqual({
- id: message.id, accountId: VALID_UUID, subject: message.subject,
- from: { name: 'Example Sender', email: '[email protected]' },
- preview: message.bodyText.slice(0, 100),
- evaluation: { action: 'reply', priority: 'high' },
- });
- expect(JSON.stringify(summary.body).length).toBeLessThan(25000);
- expect(messageSync.getMessages).toHaveBeenCalledWith({
- accountId: undefined, search: 'body', limit: 50, offset: 50,
- });
- const legacy = await request(app).get('/api/messages/inbox');
- expect(legacy.body.messages[0]).toEqual(message);
- const detail = await request(app).get(`/api/messages/${VALID_UUID}/${message.id}`);
- expect(detail.status).toBe(200);
- expect(detail.body).toEqual(message);
- });
-
it('should pass accountId filter', async () => {
messageSync.getMessages.mockResolvedValue({ messages: [], total: 0 });
diff --git a/server/services/messageSync.js b/server/services/messageSync.js
index d08d9a9f3e..2b8af4ed68 100644
--- a/server/services/messageSync.js
+++ b/server/services/messageSync.js
@@ -2,6 +2,8 @@ import { messageLogError } from '../lib/messageLogError.js';
import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, createBrowserMessageMatcher, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';
import { join } from 'path';
+import { access, stat } from 'node:fs/promises';
+import { constants } from 'node:fs';
import { atomicWrite, ensureDir, filterBySearch as genericFilterBySearch, PATHS, safeDate, safeJSONParse, UUID_RE, tryReadFile } from '../lib/fileUtils.js';
import { getAccount, updateSyncStatus, markSentIngested, updateSendAsAliases } from './messageAccounts.js';
import { defaultSyncModeForAccountType, syncModesForAccountType } from '../lib/messageTransport.js';
@@ -9,6 +11,7 @@ import { getLocalParts } from '../lib/timezone.js';
import { getUserTimezone } from './userTimezone.js';
import { v4 as uuidv4 } from '../lib/uuid.js';
import { createKeyCachedQueue } from '../lib/createKeyCachedQueue.js';
+import { createBoundedStateMap } from '../lib/boundedStateMap.js';
const CACHE_DIR = join(PATHS.messages, 'cache');
// `syncLocks` is a boolean re-entrancy guard that rejects a *duplicate* in-flight
@@ -23,6 +26,8 @@ const syncLocks = new Map();
// makes each one await the previous and re-read the freshest persisted state
// inside its serialized region — mirroring `issueWriteTail` in pipeline/issues.js.
const accountWriteQueue = createKeyCachedQueue();
+const summarySnapshots = createBoundedStateMap({ maxSize: 8 });
+const summaryReads = new Map();
function queueAccountWrite(accountId, work) {
return accountWriteQueue(accountId, work);
}
@@ -84,40 +89,121 @@ async function loadCache(accountId) {
return parsed;
}
+function messageSummary(message) {
+ return {
+ id: message.id,
+ accountId: message.accountId,
+ externalId: message.externalId,
+ threadId: message.threadId,
+ subject: message.subject,
+ from: message.from && { name: message.from.name, email: message.from.email },
+ date: message.date,
+ source: message.source,
+ isRead: message.isRead,
+ isUnread: message.isUnread,
+ isPinned: message.isPinned,
+ isFlagged: message.isFlagged,
+ // Detach the short preview so V8 cannot retain the full body behind it.
+ preview: typeof message.bodyText === 'string' ? message.bodyText.slice(0, 100).split('').join('') : '',
+ evaluation: message.evaluation && {
+ action: message.evaluation.action,
+ priority: message.evaluation.priority,
+ },
+ };
+}
+
+async function cacheRevision(filePath) {
+ try {
+ const info = await stat(filePath, { bigint: true });
+ await access(filePath, constants.R_OK);
+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(':');
+ } catch {
+ // Match loadCache's missing/unreadable fallback, without serving old rows.
+ return null;
+ }
+}
+
+function invalidateSummary(accountId) {
+ summarySnapshots.delete(accountId);
+ const pending = summaryReads.get(accountId);
+ if (pending) pending.generation++;
+}
+
+function loadSummaryCache(accountId) {
+ if (!UUID_RE.test(accountId)) throw new Error(`Invalid accountId: ${accountId}`);
+ const current = summaryReads.get(accountId);
+ if (current) return current.promise;
+ const pending = { generation: 0 };
+ pending.promise = (async () => {
+ const filePath = join(CACHE_DIR, `${accountId}.json`);
+ // No TTL freshness window: local writes invalidate immediately, while
+ // restore/external replacements and permission changes change the revision.
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const generation = pending.generation;
+ const before = await cacheRevision(filePath);
+ const snapshot = summarySnapshots.get(accountId);
+ if (before !== null && snapshot?.revision === before && generation === pending.generation) {
+ return snapshot.cache;
+ }
+ summarySnapshots.delete(accountId);
+ const full = await loadCache(accountId);
+ const after = await cacheRevision(filePath);
+ if (before !== after || generation !== pending.generation) continue;
+ const messages = full.messages.map(message => {
+ const row = messageSummary(message);
+ if (row.from) Object.freeze(row.from);
+ if (row.evaluation) Object.freeze(row.evaluation);
+ return Object.freeze(row);
+ });
+ const cache = { messages: Object.freeze(messages) };
+ if (after !== null) summarySnapshots.set(accountId, { revision: after, cache });
+ return cache;
+ }
+ throw new Error('Message cache changed repeatedly while reading its summary snapshot');
+ })().finally(() => { summaryReads.delete(accountId); });
+ summaryReads.set(accountId, pending);
+ return pending.promise;
+}
+
async function saveCache(accountId, cache) {
const filePath = join(CACHE_DIR, `${accountId}.json`);
await atomicWrite(filePath, cache);
+ invalidateSummary(accountId);
}
export async function getMessages(options = {}) {
- const { accountId, search, unevaluatedOnly, messageIds, limit = 50, offset = 0 } = options;
+ const { accountId, search, unevaluatedOnly, messageIds, summary = false, limit = 50, offset = 0 } = options;
+ const readCache = summary && !search ? loadSummaryCache : loadCache;
+ let result;
// If specific account, just load that cache
if (accountId) {
- const cache = await loadCache(accountId);
+ const cache = await readCache(accountId);
let messages = cache.messages.map(m => ({ ...m, accountId: m.accountId || accountId }));
messages = filterMessages(messages, { search, unevaluatedOnly, messageIds });
- return {
+ result = {
messages: messages.sort((a, b) => safeDate(b.date) - safeDate(a.date)).slice(offset, offset + limit),
total: messages.length
};
+ } else {
+ // Otherwise aggregate across all account caches
+ await ensureDir(CACHE_DIR);
+ const { readdir } = await import('fs/promises');
+ const files = await readdir(CACHE_DIR).catch(() => []);
+ const accountIds = files
+ .filter(file => file.endsWith('.json'))
+ .map(file => file.replace('.json', ''))
+ .filter(id => UUID_RE.test(id));
+ // Load each account's cache in parallel rather than serializing one disk read
+ // per account before we can aggregate the combined inbox.
+ const caches = await Promise.all(
+ accountIds.map(async id => ({ id, cache: await readCache(id) }))
+ );
+ // Bound the cross-account merge to `accounts × (offset+limit)` heads instead
+ // of spreading + globally sorting every message across every account (#2540).
+ result = aggregatePagedMessages(caches, { search, unevaluatedOnly, messageIds, limit, offset });
}
-
- // Otherwise aggregate across all account caches
- await ensureDir(CACHE_DIR);
- const { readdir } = await import('fs/promises');
- const files = await readdir(CACHE_DIR).catch(() => []);
- const accountIds = files
- .filter(file => file.endsWith('.json'))
- .map(file => file.replace('.json', ''))
- .filter(id => UUID_RE.test(id));
- // Load each account's cache in parallel rather than serializing one disk read
- // per account before we can aggregate the combined inbox.
- const caches = await Promise.all(
- accountIds.map(async id => ({ id, cache: await loadCache(id) }))
- );
- // Bound the cross-account merge to `accounts × (offset+limit)` heads instead
- // of spreading + globally sorting every message across every account (#2540).
- return aggregatePagedMessages(caches, { search, unevaluatedOnly, messageIds, limit, offset });
+ // Search keeps its full-body semantics; only the selected page is projected.
+ return summary && search ? { ...result, messages: result.messages.map(messageSummary) } : result;
}
/** Remove only a confirmed action target, serialized with sync/refresh writes. */
@@ -140,6 +226,7 @@ export async function deleteCache(accountId) {
await unlink(filePath).catch(err => {
if (err.code !== 'ENOENT') throw err;
});
+ invalidateSummary(accountId);
console.log(`🗑️ Message cache deleted for account ${accountId}`);
});
}