Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: reuse compact inbox summaries across unchanged pages
Adam Eivyaa05f3b9dc75
diff --git a/server/routes/messageInboxRead.js b/server/routes/messageInboxRead.jsindex 5b8988bef0..c89f216be7 100644--- a/server/routes/messageInboxRead.js+++ b/server/routes/messageInboxRead.js@@ -14,33 +14,11 @@ export const messageInboxRead = asyncHandler(async (req, res) => {const result = await messageSync.getMessages({accountId,search,+ summary: req.query.summary === 'true',limit: parsedLimit,offset: parsedOffset});- // Summary is opt-in so older clients and automation retain the full-record- // contract. Search and pagination above still run against the full cache.- res.json(req.query.summary === 'true' ? {- ...result,- messages: result.messages.map(message => ({- id: message.id,- accountId: message.accountId,- externalId: message.externalId,- threadId: message.threadId,- subject: message.subject,- from: message.from && { name: message.from.name, email: message.from.email },- date: message.date,- source: message.source,- isRead: message.isRead,- isUnread: message.isUnread,- isPinned: message.isPinned,- isFlagged: message.isFlagged,- preview: typeof message.bodyText === 'string' ? message.bodyText.slice(0, 100) : '',- evaluation: message.evaluation && {- action: message.evaluation.action,- priority: message.evaluation.priority,- },- })),- } : result);+ res.json(result);});diff --git a/server/routes/messageInboxRead.test.js b/server/routes/messageInboxRead.test.jsnew file mode 100644index 0000000000..4f7484f618--- /dev/null+++ b/server/routes/messageInboxRead.test.js@@ -0,0 +1,178 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { mkdir, readFile, rm, stat, utimes, writeFile } from 'node:fs/promises';+import { join } from 'node:path';+import { performance } from 'node:perf_hooks';+import { makePathsProxy, lazyTempDataRoot, cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';+import { request } from '../lib/testHelper.js';+import { ACCOUNT_IDS, messageFixture } from '../../scripts/perf/collectionFixtureData.js';++const permissions = vi.hoisted(() => ({ denied: false }));+vi.mock('node:fs/promises', async original => {+ const actual = await original();+ return { ...actual, access: async (...args) => {+ if (permissions.denied) throw Object.assign(new Error('Synthetic access denial'), { code: 'EACCES' });+ return actual.access(...args);+ } };+});+vi.mock('../lib/fileUtils.js', async original => {+ const actual = await original();+ const { PATHS } = makePathsProxy(actual, { dataRoot: () => lazyTempDataRoot('portos-inbox-summary-') });+ return { ...actual, PATHS, tryReadFile: vi.fn((...args) => permissions.denied ? null : actual.tryReadFile(...args)) };+});++import { PATHS, atomicWrite, tryReadFile } from '../lib/fileUtils.js';+import { errorMiddleware } from '../lib/errorHandler.js';+import { deleteCache, getMessages, removeMessageFromCache, updateMessageEvaluations } from '../services/messageSync.js';+import { messageInboxRead, messageDetailRead } from './messageInboxRead.js';++const cacheDir = join(PATHS.messages, 'cache');+const cachePath = id => join(cacheDir, id + '.json');+const app = express();+app.get('/inbox', messageInboxRead);+app.get('/:accountId/:messageId', messageDetailRead);+app.use(errorMiddleware);+const inbox = query => request(app).get('/inbox?summary=true' + (query ? '&' + query : ''));+const seed = async (count = 30) => {+ for (const [account, id] of ACCOUNT_IDS.entries()) {+ await writeFile(cachePath(id), JSON.stringify({ messages: Array.from({ length: count }, (_, i) => messageFixture(id, i + account * count)) }));+ }+};++beforeEach(async () => {+ permissions.denied = false;+ vi.mocked(tryReadFile).mockClear();+ await rm(PATHS.messages, { recursive: true, force: true });+ await mkdir(cacheDir, { recursive: true });+});+afterAll(cleanupTempDataRoots);++describe('inbox summary reads through the production HTTP/service boundary', () => {+ it('shares first reads and reuses compact rows across pages without hydrating bodies', async () => {+ await seed();+ const [first, duplicate] = await Promise.all([inbox(), inbox()]);+ const second = await inbox('offset=50');+ expect(first.status).toBe(200);+ expect(duplicate.body).toEqual(first.body);+ expect(first.body.total).toBe(60);+ expect(first.body.messages).toHaveLength(50);+ expect(second.body.messages).toHaveLength(10);+ expect(new Set([...first.body.messages, ...second.body.messages].map(message => message.id)).size).toBe(60);+ expect(tryReadFile).toHaveBeenCalledTimes(2);+ expect(JSON.stringify(first.body).length).toBeLessThan(26000);+ expect(first.body.messages[0].preview).toHaveLength(100);+ expect(first.body.messages[0]).not.toHaveProperty('bodyText');+ expect(first.body.messages[0]).not.toHaveProperty('bodyHtml');+ expect(first.body.messages[0].evaluation).not.toHaveProperty('reasoning');+ });++ it('keeps full-body search, legacy reads, lazy details and account filters intact', async () => {+ const id = ACCOUNT_IDS[0];+ const message = { ...messageFixture(id, 1), bodyText: 'x'.repeat(500) + 'unique-body-match',+ from: { name: 'Example Sender', email: '[email protected]', detail: 'private detail' } };+ await writeFile(cachePath(id), JSON.stringify({ messages: [message] }));+ await inbox();+ const found = await inbox('search=unique-body-match&accountId=' + id);+ expect(found.body).toMatchObject({ total: 1, messages: [{ id: message.id, preview: 'x'.repeat(100) }] });+ expect(found.body.messages[0].from).toEqual({ name: 'Example Sender', email: '[email protected]' });+ expect(found.body.messages[0].evaluation).toEqual({ action: 'review', priority: 'normal' });+ expect((await request(app).get('/inbox')).body.messages).toEqual([message]);+ expect((await request(app).get(`/${id}/${message.id}`)).body).toEqual(message);+ expect((await inbox('accountId=' + ACCOUNT_IDS[1])).body).toEqual({ messages: [], total: 0 });+ expect(tryReadFile).toHaveBeenCalledTimes(5);+ });++ it('observes same-size edits, external replacements, evaluation writes and deletions immediately', async () => {+ const id = ACCOUNT_IDS[0];+ let message = { ...messageFixture(id, 1), bodyText: 'Old' };+ const save = () => writeFile(cachePath(id), JSON.stringify({ messages: [message] }));+ await save();+ expect((await inbox()).body.messages[0].preview).toBe('Old');+ const before = await stat(cachePath(id));+ message = { ...message, bodyText: 'New' };+ await save();+ await utimes(cachePath(id), before.atime, before.mtime);+ expect((await inbox()).body.messages[0].preview).toBe('New');+ message = { ...message, bodyText: 'Replacement' };+ await atomicWrite(cachePath(id), { messages: [message] });+ expect((await inbox()).body.messages[0].preview).toBe('Replacement');+ await updateMessageEvaluations({ [message.id]: { action: 'keep', priority: 'low', reasoning: 'Full detail' } });+ expect((await inbox()).body.messages[0].evaluation).toEqual({ action: 'keep', priority: 'low' });+ await removeMessageFromCache(id, message.id);+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ await deleteCache(id);+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ });++ it('does not publish a stale first read over a concurrent committed evaluation', async () => {+ const id = ACCOUNT_IDS[0];+ const message = messageFixture(id, 1);+ await writeFile(cachePath(id), JSON.stringify({ messages: [message] }));+ let started;+ let release;+ const reading = new Promise(resolve => { started = resolve; });+ const held = new Promise(resolve => { release = resolve; });+ vi.mocked(tryReadFile).mockImplementationOnce(async file => {+ const content = await readFile(file, 'utf8');+ started();+ await held;+ return content;+ });+ const first = getMessages({ accountId: id, summary: true });+ await reading;+ const joined = getMessages({ accountId: id, summary: true });+ try {+ await updateMessageEvaluations({ [message.id]: { action: 'keep', priority: 'high' } });+ } finally { release(); }+ expect((await first).messages[0].evaluation).toEqual({ action: 'keep', priority: 'high' });+ expect(await joined).toEqual(await first);+ expect((await inbox()).body.messages[0].evaluation).toEqual({ action: 'keep', priority: 'high' });+ expect(tryReadFile).toHaveBeenCalledTimes(3);+ });++ it('never serves old summaries after unreadable, corrupt, missing or empty caches', async () => {+ const id = ACCOUNT_IDS[0];+ const restore = () => writeFile(cachePath(id), JSON.stringify({ messages: [messageFixture(id, 1)] }));+ await restore();+ expect((await inbox()).body.total).toBe(1);+ permissions.denied = true;+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ permissions.denied = false;+ expect((await inbox()).body.total).toBe(1);+ await writeFile(cachePath(id), '{invalid');+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ await restore();+ expect((await inbox()).body.total).toBe(1);+ await rm(cachePath(id));+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ await writeFile(cachePath(id), JSON.stringify({ messages: [] }));+ await inbox();+ vi.mocked(tryReadFile).mockClear();+ expect((await inbox()).body).toEqual({ messages: [], total: 0 });+ expect(tryReadFile).not.toHaveBeenCalled();+ });++ it.skipIf(process.env.PORTOS_INBOX_BENCHMARK !== '1')('measures repeated 4000-message HTTP pages without timing gates', async () => {+ await seed(2000);+ const bytes = (await Promise.all(ACCOUNT_IDS.map(async id => (await stat(cachePath(id))).size))).reduce((a, b) => a + b);+ const measure = async summary => {+ vi.mocked(tryReadFile).mockClear();+ const durations = [];+ for (let page = 0; page < 8; page++) {+ const start = performance.now();+ const response = await request(app).get(`/inbox?summary=${summary}&offset=${page * 50}`);+ expect(response.body.total).toBe(4000);+ expect(response.body.messages).toHaveLength(50);+ durations.push(performance.now() - start);+ }+ return { reads: vi.mocked(tryReadFile).mock.calls.length,+ medianWarmMs: durations.slice(1).sort((a, b) => a - b)[3] };+ };+ const legacy = await measure(false);+ const summary = await measure(true);+ expect(legacy.reads).toBe(16);+ expect(summary.reads).toBe(2);+ process.stdout.write(JSON.stringify({ synthetic: true, messages: 4000, pageLimit: 50, pages: 8,+ bytesPerFullRead: bytes, legacy, summary, eliminatedReadBytes: bytes * 7 }) + '\n');+ });+});diff --git a/server/routes/messages.test.js b/server/routes/messages.test.jsindex a09ad70f97..99f8c23c0e 100644--- a/server/routes/messages.test.js+++ b/server/routes/messages.test.js@@ -366,39 +366,6 @@ describe('Messages Routes', () => {expect(response.body.total).toBe(1);});- it('keeps large bodies out of summary pages while preserving legacy and detail reads', async () => {- const message = {- id: 'msg-large', accountId: VALID_UUID, subject: 'Synthetic large message',- from: { name: 'Example Sender', email: '[email protected]', extra: 'detail' },- bodyText: 'Synthetic message body. '.repeat(10000),- bodyHtml: '<p>Synthetic HTML</p>'.repeat(10000),- attachments: [{ content: 'synthetic attachment'.repeat(1000) }],- evaluation: { action: 'reply', priority: 'high', reasoning: 'detail'.repeat(1000) },- };- messageSync.getMessages.mockResolvedValue({ messages: Array(50).fill(message), total: 5000 });- messageSync.getMessage.mockResolvedValue(message);-- const summary = await request(app).get('/api/messages/inbox?summary=true&search=body&offset=50');- expect(summary.status).toBe(200);- expect(summary.body.total).toBe(5000);- expect(summary.body.messages).toHaveLength(50);- expect(summary.body.messages[0]).toEqual({- id: message.id, accountId: VALID_UUID, subject: message.subject,- from: { name: 'Example Sender', email: '[email protected]' },- preview: message.bodyText.slice(0, 100),- evaluation: { action: 'reply', priority: 'high' },- });- expect(JSON.stringify(summary.body).length).toBeLessThan(25000);- expect(messageSync.getMessages).toHaveBeenCalledWith({- accountId: undefined, search: 'body', limit: 50, offset: 50,- });- const legacy = await request(app).get('/api/messages/inbox');- expect(legacy.body.messages[0]).toEqual(message);- const detail = await request(app).get(`/api/messages/${VALID_UUID}/${message.id}`);- expect(detail.status).toBe(200);- expect(detail.body).toEqual(message);- });-it('should pass accountId filter', async () => {messageSync.getMessages.mockResolvedValue({ messages: [], total: 0 });diff --git a/server/services/messageSync.js b/server/services/messageSync.jsindex d08d9a9f3e..2b8af4ed68 100644--- a/server/services/messageSync.js+++ b/server/services/messageSync.js@@ -2,6 +2,8 @@ import { messageLogError } from '../lib/messageLogError.js';import { browserSummaryId, browserDetailIdentities, messageIngestionIdentity, matchBrowserMessage, createBrowserMessageMatcher, retainLegacyActivityIdentities } from '../lib/messageBrowserIdentity.js';import { join } from 'path';+import { access, stat } from 'node:fs/promises';+import { constants } from 'node:fs';import { atomicWrite, ensureDir, filterBySearch as genericFilterBySearch, PATHS, safeDate, safeJSONParse, UUID_RE, tryReadFile } from '../lib/fileUtils.js';import { getAccount, updateSyncStatus, markSentIngested, updateSendAsAliases } from './messageAccounts.js';import { defaultSyncModeForAccountType, syncModesForAccountType } from '../lib/messageTransport.js';@@ -9,6 +11,7 @@ import { getLocalParts } from '../lib/timezone.js';import { getUserTimezone } from './userTimezone.js';import { v4 as uuidv4 } from '../lib/uuid.js';import { createKeyCachedQueue } from '../lib/createKeyCachedQueue.js';+import { createBoundedStateMap } from '../lib/boundedStateMap.js';const CACHE_DIR = join(PATHS.messages, 'cache');// `syncLocks` is a boolean re-entrancy guard that rejects a *duplicate* in-flight@@ -23,6 +26,8 @@ const syncLocks = new Map();// makes each one await the previous and re-read the freshest persisted state// inside its serialized region — mirroring `issueWriteTail` in pipeline/issues.js.const accountWriteQueue = createKeyCachedQueue();+const summarySnapshots = createBoundedStateMap({ maxSize: 8 });+const summaryReads = new Map();function queueAccountWrite(accountId, work) {return accountWriteQueue(accountId, work);}@@ -84,40 +89,121 @@ async function loadCache(accountId) {return parsed;}+function messageSummary(message) {+ return {+ id: message.id,+ accountId: message.accountId,+ externalId: message.externalId,+ threadId: message.threadId,+ subject: message.subject,+ from: message.from && { name: message.from.name, email: message.from.email },+ date: message.date,+ source: message.source,+ isRead: message.isRead,+ isUnread: message.isUnread,+ isPinned: message.isPinned,+ isFlagged: message.isFlagged,+ // Detach the short preview so V8 cannot retain the full body behind it.+ preview: typeof message.bodyText === 'string' ? message.bodyText.slice(0, 100).split('').join('') : '',+ evaluation: message.evaluation && {+ action: message.evaluation.action,+ priority: message.evaluation.priority,+ },+ };+}++async function cacheRevision(filePath) {+ try {+ const info = await stat(filePath, { bigint: true });+ await access(filePath, constants.R_OK);+ return [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(':');+ } catch {+ // Match loadCache's missing/unreadable fallback, without serving old rows.+ return null;+ }+}++function invalidateSummary(accountId) {+ summarySnapshots.delete(accountId);+ const pending = summaryReads.get(accountId);+ if (pending) pending.generation++;+}++function loadSummaryCache(accountId) {+ if (!UUID_RE.test(accountId)) throw new Error(`Invalid accountId: ${accountId}`);+ const current = summaryReads.get(accountId);+ if (current) return current.promise;+ const pending = { generation: 0 };+ pending.promise = (async () => {+ const filePath = join(CACHE_DIR, `${accountId}.json`);+ // No TTL freshness window: local writes invalidate immediately, while+ // restore/external replacements and permission changes change the revision.+ for (let attempt = 0; attempt < 3; attempt++) {+ const generation = pending.generation;+ const before = await cacheRevision(filePath);+ const snapshot = summarySnapshots.get(accountId);+ if (before !== null && snapshot?.revision === before && generation === pending.generation) {+ return snapshot.cache;+ }+ summarySnapshots.delete(accountId);+ const full = await loadCache(accountId);+ const after = await cacheRevision(filePath);+ if (before !== after || generation !== pending.generation) continue;+ const messages = full.messages.map(message => {+ const row = messageSummary(message);+ if (row.from) Object.freeze(row.from);+ if (row.evaluation) Object.freeze(row.evaluation);+ return Object.freeze(row);+ });+ const cache = { messages: Object.freeze(messages) };+ if (after !== null) summarySnapshots.set(accountId, { revision: after, cache });+ return cache;+ }+ throw new Error('Message cache changed repeatedly while reading its summary snapshot');+ })().finally(() => { summaryReads.delete(accountId); });+ summaryReads.set(accountId, pending);+ return pending.promise;+}+async function saveCache(accountId, cache) {const filePath = join(CACHE_DIR, `${accountId}.json`);await atomicWrite(filePath, cache);+ invalidateSummary(accountId);}export async function getMessages(options = {}) {- const { accountId, search, unevaluatedOnly, messageIds, limit = 50, offset = 0 } = options;+ const { accountId, search, unevaluatedOnly, messageIds, summary = false, limit = 50, offset = 0 } = options;+ const readCache = summary && !search ? loadSummaryCache : loadCache;+ let result;// If specific account, just load that cacheif (accountId) {- const cache = await loadCache(accountId);+ const cache = await readCache(accountId);let messages = cache.messages.map(m => ({ ...m, accountId: m.accountId || accountId }));messages = filterMessages(messages, { search, unevaluatedOnly, messageIds });- return {+ result = {messages: messages.sort((a, b) => safeDate(b.date) - safeDate(a.date)).slice(offset, offset + limit),total: messages.length};+ } else {+ // Otherwise aggregate across all account caches+ await ensureDir(CACHE_DIR);+ const { readdir } = await import('fs/promises');+ const files = await readdir(CACHE_DIR).catch(() => []);+ const accountIds = files+ .filter(file => file.endsWith('.json'))+ .map(file => file.replace('.json', ''))+ .filter(id => UUID_RE.test(id));+ // Load each account's cache in parallel rather than serializing one disk read+ // per account before we can aggregate the combined inbox.+ const caches = await Promise.all(+ accountIds.map(async id => ({ id, cache: await readCache(id) }))+ );+ // Bound the cross-account merge to `accounts × (offset+limit)` heads instead+ // of spreading + globally sorting every message across every account (#2540).+ result = aggregatePagedMessages(caches, { search, unevaluatedOnly, messageIds, limit, offset });}-- // Otherwise aggregate across all account caches- await ensureDir(CACHE_DIR);- const { readdir } = await import('fs/promises');- const files = await readdir(CACHE_DIR).catch(() => []);- const accountIds = files- .filter(file => file.endsWith('.json'))- .map(file => file.replace('.json', ''))- .filter(id => UUID_RE.test(id));- // Load each account's cache in parallel rather than serializing one disk read- // per account before we can aggregate the combined inbox.- const caches = await Promise.all(- accountIds.map(async id => ({ id, cache: await loadCache(id) }))- );- // Bound the cross-account merge to `accounts × (offset+limit)` heads instead- // of spreading + globally sorting every message across every account (#2540).- return aggregatePagedMessages(caches, { search, unevaluatedOnly, messageIds, limit, offset });+ // Search keeps its full-body semantics; only the selected page is projected.+ return summary && search ? { ...result, messages: result.messages.map(messageSummary) } : result;}/** Remove only a confirmed action target, serialized with sync/refresh writes. */@@ -140,6 +226,7 @@ export async function deleteCache(accountId) {await unlink(filePath).catch(err => {if (err.code !== 'ENOENT') throw err;});+ invalidateSummary(accountId);console.log(`🗑️ Message cache deleted for account ${accountId}`);});}