Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(messages): preserve Gmail reply message and thread identity (#10716)
[._.]/ Adam Eivycdc7b6a71053
diff --git a/server/routes/messageGmailReply.test.js b/server/routes/messageGmailReply.test.jsnew file mode 100644index 0000000000..bc302ffb03--- /dev/null+++ b/server/routes/messageGmailReply.test.js@@ -0,0 +1,160 @@+import { beforeEach, beforeAll, afterAll, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { join } from 'path';+import { request } from '../lib/testHelper.js';++// Exercise HTTP creation/approval/send through the real store, sender and Gmail+// adapter. Only storage, account/cache lookup and the external provider are doubled.+const doubles = vi.hoisted(() => ({+ files: new Map(), original: null, metadata: vi.fn(), submit: vi.fn(), auth: { getAccessToken: vi.fn() }+}));+vi.mock('../lib/fileUtils.js', () => ({+ PATHS: { messages: '/mock/messages' },+ UUID_RE: /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i,+ ensureDir: async () => {},+ readJSONFileStrict: async (path, fallback) => ({+ ok: true, value: doubles.files.has(path) ? JSON.parse(doubles.files.get(path)) : fallback+ }),+ tryReadFile: async path => doubles.files.get(path) ?? null,+ atomicWrite: async (path, data) => { doubles.files.set(path, JSON.stringify(data)); },+ safeJSONParse: JSON.parse+}));+vi.mock('../services/messageAccounts.js', () => ({+ getAccount: async id => ({ id, type: 'gmail' }), listAccounts: async () => []+}));+vi.mock('../services/messageSync.js', () => ({+ getMessage: async (accountId, id) => doubles.original?.id === id && doubles.original.accountId === accountId ? doubles.original : null+}));+vi.mock('../services/googleAuth.js', () => ({ getAuthenticatedClient: async () => doubles.auth }));+vi.mock('@googleapis/gmail', () => ({ gmail: () => ({ users: { messages: { get: doubles.metadata, send: doubles.submit } } }) }));+vi.mock('../services/messagePlaywrightSync.js', () => ({ getSelectors: vi.fn(), updateSelectors: vi.fn(), testSelectors: vi.fn(), launchProvider: vi.fn() }));+vi.mock('../services/messageEvaluator.js', () => ({ evaluateMessages: vi.fn(), generateReplyBody: vi.fn() }));+vi.mock('../services/messageActions.js', () => ({ executeAction: vi.fn() }));+vi.mock('../services/messageTriageRules.js', () => ({ listRules: vi.fn(), deleteRule: vi.fn() }));+vi.mock('../services/messageTokenExtractor.js', () => ({ getToken: vi.fn(), getTokenStatus: vi.fn(), testApi: vi.fn(), clearTokenCache: vi.fn() }));++import messagesRoutes from './messages.js';+import { errorEvents } from '../lib/errorHandler.js';+import { getDraft, updateDraft } from '../services/messageDrafts.js';++const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111';+const OTHER_ACCOUNT_ID = '22222222-2222-4222-8222-222222222222';+const LOCAL_ID = '33333333-3333-4333-8333-333333333333';+const observeError = () => {};+beforeAll(() => errorEvents.on('error', observeError));+afterAll(() => errorEvents.off('error', observeError));+let app;+const metadata = () => ({ data: {+ id: 'gmail-message-1', threadId: 'gmail-thread-1', payload: { headers: [+ { name: 'Message-ID', value: '<[email protected]>' },+ { name: 'References', value: '<[email protected]>\r\n <[email protected]>' },+ { name: 'Subject', value: 'Example conversation' }+ ] }+} });++beforeEach(() => {+ vi.resetAllMocks();+ doubles.files.clear();+ doubles.original = {+ id: LOCAL_ID, accountId: ACCOUNT_ID, apiId: 'gmail-message-1',+ threadId: 'conv-local-hash', conversationId: 'gmail-thread-1'+ // Legacy cache: no persisted RFC Message-ID or References.+ };+ doubles.metadata.mockResolvedValue(metadata());+ doubles.auth.getAccessToken.mockResolvedValue({ token: 'example-token' });+ doubles.submit.mockResolvedValue({ data: { id: 'gmail-reply-1' } });+ app = express();+ app.use(express.json());+ app.use('/api/messages', messagesRoutes);+});++async function approvedReply(overrides = {}) {+ const created = await request(app).post('/api/messages/drafts').send({+ accountId: ACCOUNT_ID, replyToMessageId: LOCAL_ID, threadId: 'conv-local-hash',+ to: ['[email protected]'], subject: 'Re: Example conversation', body: 'Example reply', ...overrides+ });+ expect(created.status).toBe(201);+ const { id } = created.body;+ expect((await request(app).post(`/api/messages/drafts/${id}/approve`)).status).toBe(200);+ return id;+}++describe('HTTP Gmail reply identity', () => {+ it('resolves legacy local IDs once, preserves the RFC chain and submits the provider thread and subject', async () => {+ const id = await approvedReply();+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).body).toEqual({ success: true });+ expect((await getDraft(id)).status).toBe('sent');+ expect(doubles.metadata).toHaveBeenCalledTimes(1);+ expect(doubles.metadata).toHaveBeenCalledWith({+ userId: 'me', id: 'gmail-message-1', format: 'metadata',+ metadataHeaders: ['Message-ID', 'References', 'In-Reply-To', 'Subject']+ }, { timeout: 10_000, retry: false });+ expect(doubles.submit).toHaveBeenCalledTimes(1);+ const [{ requestBody }, options] = doubles.submit.mock.calls[0];+ expect(requestBody.threadId).toBe('gmail-thread-1');+ expect(options).toEqual({ retry: false });+ const mime = Buffer.from(requestBody.raw, 'base64url').toString('utf8');+ expect(mime).toContain('In-Reply-To: <[email protected]>\r\n');+ expect(mime).toContain('References: <[email protected]> <[email protected]> <[email protected]>\r\n');+ expect(mime).toContain('Subject: Example conversation\r\n');+ expect(mime).toContain('\r\n\r\nExample reply');+ expect(mime).not.toContain(LOCAL_ID);+ expect(mime).not.toContain('conv-local-hash');+ expect((await getDraft(id)).replyToMessageId).toBe(LOCAL_ID);+ });++ it('fills an empty reply subject and uses the parent In-Reply-To when References are absent', async () => {+ const data = metadata();+ data.data.payload.headers = data.data.payload.headers.filter(h => h.name !== 'References');+ data.data.payload.headers.push({ name: 'In-Reply-To', value: '<[email protected]>' });+ doubles.metadata.mockResolvedValue(data);+ const id = await approvedReply({ subject: '', threadId: null });+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).status).toBe(200);+ const mime = Buffer.from(doubles.submit.mock.calls[0][0].requestBody.raw, 'base64url').toString('utf8');+ expect(mime).toContain('References: <[email protected]> <[email protected]>\r\n');+ expect(mime).toContain('Subject: Example conversation\r\n');+ });++ it.each([+ ['missing original', () => { doubles.original = null; }, {}, 'REPLY_TARGET_NOT_FOUND'],+ ['wrong account', () => {}, { accountId: OTHER_ACCOUNT_ID }, 'REPLY_TARGET_NOT_FOUND'],+ ['local thread mismatch', () => {}, { threadId: 'other-local-thread' }, 'THREAD_MISMATCH'],+ ['metadata lookup failure', () => { doubles.metadata.mockRejectedValue(new Error('Example lookup failure')); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],+ ['provider identity mismatch', () => { const data = metadata(); data.data.id = 'other-gmail-message'; doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],+ ['provider thread mismatch', () => { const data = metadata(); data.data.threadId = 'other-gmail-thread'; doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],+ ['missing RFC identity', () => { const data = metadata(); data.data.payload.headers = data.data.payload.headers.filter(h => h.name !== 'Message-ID'); doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_INVALID'],+ ['incompatible subject', () => {}, { subject: 'Different conversation' }, 'GMAIL_REPLY_SUBJECT_MISMATCH']+ ])('refuses %s without submission, approval consumption or send attempts', async (_label, arrange, overrides, code) => {+ arrange();+ const id = await approvedReply(overrides);+ const response = await request(app).post(`/api/messages/drafts/${id}/send`);+ expect(response.status).toBeGreaterThanOrEqual(400);+ expect(response.body).toMatchObject({ code, error: expect.any(String) });+ expect(await getDraft(id)).toMatchObject({ status: 'approved', sendAttempts: [] });+ expect(doubles.submit).not.toHaveBeenCalled();+ });++ it('revalidates the claimed subject if it changes while provider metadata is loading', async () => {+ const id = await approvedReply();+ doubles.metadata.mockImplementationOnce(async () => {+ await updateDraft(id, { subject: 'Different conversation' });+ return metadata();+ });+ const response = await request(app).post(`/api/messages/drafts/${id}/send`);+ expect(response.body.code).toBe('GMAIL_REPLY_SUBJECT_MISMATCH');+ expect(await getDraft(id)).toMatchObject({ status: 'approved', subject: 'Different conversation', sendAttempts: [] });+ expect(doubles.submit).not.toHaveBeenCalled();+ });++ it('sends new messages without metadata lookup or threading fields', async () => {+ const id = await approvedReply({ replyToMessageId: null, threadId: null, subject: 'New message' });+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).status).toBe(200);+ expect(doubles.metadata).not.toHaveBeenCalled();+ const body = doubles.submit.mock.calls[0][0].requestBody;+ expect(Object.keys(body)).toEqual(['raw']);+ const mime = Buffer.from(body.raw, 'base64url').toString('utf8');+ expect(mime).toContain('Subject: New message\r\n');+ expect(mime).not.toMatch(/In-Reply-To:|References:/);+ expect(doubles.files.has(join('/mock/messages', 'drafts.json'))).toBe(true);+ });+});diff --git a/server/services/messageDrafts.js b/server/services/messageDrafts.jsindex 9f0d6c60e7..12609be825 100644--- a/server/services/messageDrafts.js+++ b/server/services/messageDrafts.js@@ -142,7 +142,7 @@ export async function updateDraft(id, updates) {// The eligibility check and transition share the same queue as every draft edit.// Provider I/O happens after this promise resolves, never while holding the queue.-export async function claimDraftForSend(id) {+export async function claimDraftForSend(id, validateClaim) {return queueWrite(async () => {const drafts = await loadDrafts();const draft = drafts.find(d => d.id === id);@@ -150,6 +150,7 @@ export async function claimDraftForSend(id) {if (draft.status !== 'approved') {throw new ServerError('Draft must be approved and not already sending or sent', { status: 409, code: 'DRAFT_STATE_CONFLICT' });}+ validateClaim?.(draft);// Keep uncertain delivery blocked after a crash; never silently retry it.draft.status = 'sending';draft.updatedAt = new Date().toISOString();diff --git a/server/services/messageGmailSync.js b/server/services/messageGmailSync.jsindex 3904b589ff..6f0a3fa590 100644--- a/server/services/messageGmailSync.js+++ b/server/services/messageGmailSync.js@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from '../lib/uuid.js';import crypto from 'crypto';import { getAuthenticatedClient } from './googleAuth.js';import { htmlToText as sharedHtmlToText } from '../lib/htmlToText.js';+import { ServerError } from '../lib/errorHandler.js';function makeExternalId(gmailId) {return 'api-gmail-' + crypto.createHash('md5').update(gmailId).digest('hex').slice(0, 12);@@ -127,7 +128,9 @@ export async function collectMessageIds(passes, listFn, { onProgress } = {}) {}function getHeader(headers, name) {- return headers?.find(h => h.name.toLowerCase() === name.toLowerCase())?.value || '';+ if (!Array.isArray(headers)) return '';+ const value = headers.find(h => typeof h?.name === 'string' && h.name.toLowerCase() === name.toLowerCase())?.value;+ return typeof value === 'string' ? value : '';}function decodeBase64Url(str) {@@ -399,13 +402,82 @@ const GMAIL_DELIVERY_UNKNOWN = {// is a timeout, which is exactly the ambiguity this classification exists for.const isDefinitiveRefusal = (status) => Number.isInteger(status) && status >= 400 && status < 500 && status !== 408;+const replyRefusal = (code, error, status = 422) => ({ refusal: { success: false, status, code, error } });+const replySubject = subject => String(subject || '').replace(/^(?:\s*re:\s*)+/i, '').trim();+const unfoldHeader = value => value.replace(/\r?\n[ \t]+/g, ' ').trim();++// Resolve legacy cached messages through their Gmail API identity. Never persist+// RFC metadata or interpret a PortOS UUID/thread hash as a provider identity.+export async function prepareGmailReply(account, draft, original) {+ if (!original || original.id !== draft.replyToMessageId || original.accountId !== account.id) {+ return replyRefusal('REPLY_TARGET_NOT_FOUND', 'The original message is unavailable in this account. Sync this account and select the reply target again.', 404);+ }+ if (draft.threadId && draft.threadId !== original.threadId) {+ return replyRefusal('THREAD_MISMATCH', 'The reply target does not belong to the selected conversation. Select the original message again.');+ }+ if (!original.apiId) {+ return replyRefusal('GMAIL_REPLY_METADATA_UNAVAILABLE', 'The original message has no Gmail identity. Sync it through the Gmail API before replying.');+ }+ const auth = await getAuthenticatedClient();+ if (!auth) return replyRefusal('GMAIL_NOT_CONFIGURED', 'Google OAuth is not configured. Connect Google before sending this reply.', 502);+ const gmailClient = gmail({ version: 'v1', auth });+ const metadata = await gmailClient.users.messages.get({+ userId: 'me', id: original.apiId, format: 'metadata',+ metadataHeaders: ['Message-ID', 'References', 'In-Reply-To', 'Subject']+ }, { timeout: 10_000, retry: false }).then(res => res.data, () => null);+ if (!metadata || metadata.id !== original.apiId || !metadata.threadId+ || (original.conversationId && metadata.threadId !== original.conversationId)) {+ return replyRefusal('GMAIL_REPLY_METADATA_UNAVAILABLE', 'Gmail could not verify the original message and conversation. Sync the account or reconnect Google, then try again.', 502);+ }+ const headers = metadata.payload?.headers;+ const messageId = unfoldHeader(getHeader(headers, 'Message-ID'));+ const subject = unfoldHeader(getHeader(headers, 'Subject'));+ if (!/^<[^<>\s@]+@[^<>\s@]+>$/.test(messageId) || /[\r\n]/.test(subject)) {+ return replyRefusal('GMAIL_REPLY_METADATA_INVALID', 'The original message lacks valid reply headers. Select a different message or send a new email.');+ }+ // RFC 2822: use the parent's References, falling back to its single+ // In-Reply-To when References is absent, then append the parent's Message-ID.+ const parentReferences = getHeader(headers, 'References');+ const referenceIds = unfoldHeader(parentReferences || getHeader(headers, 'In-Reply-To')).match(/<[^<>\s@]+@[^<>\s@]+>/g) || [];+ const references = parentReferences ? referenceIds : (referenceIds.length === 1 ? referenceIds : []);+ const delivery = Object.freeze({+ accountId: account.id, replyToMessageId: original.id, localThreadId: draft.threadId,+ messageId, references: [...references, messageId].join(' '), threadId: metadata.threadId, subject+ });+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);+ return refusal ? { refusal } : { delivery };+}++function gmailReplyDraftRefusal(account, draft, delivery) {+ if (!delivery || delivery.accountId !== account.id || draft.accountId !== account.id || draft.sendVia !== 'api'+ || delivery.replyToMessageId !== draft.replyToMessageId+ || delivery.localThreadId !== draft.threadId) {+ return replyRefusal('GMAIL_REPLY_NOT_PREPARED', 'The Gmail reply target was not verified. Select the original message and try again.').refusal;+ }+ if (draft.subject && replySubject(draft.subject) !== replySubject(delivery.subject)) {+ return replyRefusal('GMAIL_REPLY_SUBJECT_MISMATCH', 'A reply must keep the original conversation subject. Restore the original subject or send a new email.').refusal;+ }+ return null;+}++// Called inside the draft store's claim queue, so an edit while metadata is+// loading cannot consume approval or send an incompatible reply subject.+export function assertGmailReplyDraft(account, draft, delivery) {+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);+ if (refusal) throw new ServerError(refusal.error, { status: refusal.status, code: refusal.code });+}+/*** Send email via Gmail API.* @param {object} account - Account config* @param {object} draft - Draft with to, cc, subject, body* @returns {{ success: boolean, deliveryUnknown?: boolean, error?: string }}*/-export async function sendGmail(account, draft) {+export async function sendGmail(account, draft, delivery = null) {+ if (draft.replyToMessageId) {+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);+ if (refusal) return refusal;+ }const auth = await getAuthenticatedClient();if (!auth) {return { success: false, error: 'Google OAuth not configured', status: 502, code: 'GMAIL_NOT_CONFIGURED' };@@ -425,7 +497,7 @@ export async function sendGmail(account, draft) {const toLine = Array.isArray(draft.to) ? draft.to.join(', ') : draft.to;const lines = [`To: ${toLine}`,- `Subject: ${draft.subject || ''}`,+ `Subject: ${delivery?.subject ?? draft.subject ?? ''}`,'Content-Type: text/plain; charset=utf-8','MIME-Version: 1.0'];@@ -433,10 +505,9 @@ export async function sendGmail(account, draft) {const ccLine = Array.isArray(draft.cc) ? draft.cc.join(', ') : draft.cc;lines.push(`Cc: ${ccLine}`);}- if (draft.replyToMessageId && draft.threadId) {- // For replies, set In-Reply-To and References headers- lines.push(`In-Reply-To: ${draft.replyToMessageId}`);- lines.push(`References: ${draft.replyToMessageId}`);+ if (draft.replyToMessageId) {+ lines.push(`In-Reply-To: ${delivery.messageId}`);+ lines.push(`References: ${delivery.references}`);}lines.push('', draft.body || '');@@ -446,7 +517,7 @@ export async function sendGmail(account, draft) {// after a lost acknowledgement is a duplicate message.const sendError = await gmailClient.users.messages.send({userId: 'me',- requestBody: { raw }+ requestBody: { raw, ...(draft.replyToMessageId ? { threadId: delivery.threadId } : {}) }}, { retry: false }).then(() => null, err => err);if (sendError) {diff --git a/server/services/messageSender.js b/server/services/messageSender.jsindex 685f01ca6c..87902b7866 100644--- a/server/services/messageSender.js+++ b/server/services/messageSender.js@@ -41,19 +41,28 @@ export async function sendDraft(draftId, io) {}let delivery = null;+ let validateClaim;if (draft.sendVia === 'playwright') {const prepared = await prepareBrowserDelivery(account, draft);if (prepared.refusal) return prepared.refusal;delivery = prepared.delivery;+ } else if (draft.replyToMessageId) {+ const { getMessage } = await import('./messageSync.js');+ const { prepareGmailReply, assertGmailReplyDraft } = await import('./messageGmailSync.js');+ const original = await getMessage(account.id, draft.replyToMessageId);+ const prepared = await prepareGmailReply(account, draft, original);+ if (prepared.refusal) return prepared.refusal;+ delivery = prepared.delivery;+ validateClaim = claimed => assertGmailReplyDraft(account, claimed, delivery);}- draft = await claimDraftForSend(draftId);+ draft = await claimDraftForSend(draftId, validateClaim);console.log(`📧 Sending draft ${draft.id} via ${draft.sendVia}`);const dispatch = async () => {if (draft.sendVia === 'api') {const { sendGmail } = await import('./messageGmailSync.js');- return sendGmail(account, draft);+ return delivery ? sendGmail(account, draft, delivery) : sendGmail(account, draft);}const { sendPlaywright } = await import('./messagePlaywrightSync.js');return sendPlaywright(account, draft, delivery);