Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(messages): preserve Gmail reply message and thread identity (#10716)

[._.]/ Adam Eivycdc7b6a71053
diff --git a/server/routes/messageGmailReply.test.js b/server/routes/messageGmailReply.test.js
new file mode 100644
index 0000000000..bc302ffb03
--- /dev/null
+++ b/server/routes/messageGmailReply.test.js
@@ -0,0 +1,160 @@
+import { beforeEach, beforeAll, afterAll, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+import { join } from 'path';
+import { request } from '../lib/testHelper.js';
+
+// Exercise HTTP creation/approval/send through the real store, sender and Gmail
+// adapter. Only storage, account/cache lookup and the external provider are doubled.
+const doubles = vi.hoisted(() => ({
+ files: new Map(), original: null, metadata: vi.fn(), submit: vi.fn(), auth: { getAccessToken: vi.fn() }
+}));
+vi.mock('../lib/fileUtils.js', () => ({
+ PATHS: { messages: '/mock/messages' },
+ UUID_RE: /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i,
+ ensureDir: async () => {},
+ readJSONFileStrict: async (path, fallback) => ({
+ ok: true, value: doubles.files.has(path) ? JSON.parse(doubles.files.get(path)) : fallback
+ }),
+ tryReadFile: async path => doubles.files.get(path) ?? null,
+ atomicWrite: async (path, data) => { doubles.files.set(path, JSON.stringify(data)); },
+ safeJSONParse: JSON.parse
+}));
+vi.mock('../services/messageAccounts.js', () => ({
+ getAccount: async id => ({ id, type: 'gmail' }), listAccounts: async () => []
+}));
+vi.mock('../services/messageSync.js', () => ({
+ getMessage: async (accountId, id) => doubles.original?.id === id && doubles.original.accountId === accountId ? doubles.original : null
+}));
+vi.mock('../services/googleAuth.js', () => ({ getAuthenticatedClient: async () => doubles.auth }));
+vi.mock('@googleapis/gmail', () => ({ gmail: () => ({ users: { messages: { get: doubles.metadata, send: doubles.submit } } }) }));
+vi.mock('../services/messagePlaywrightSync.js', () => ({ getSelectors: vi.fn(), updateSelectors: vi.fn(), testSelectors: vi.fn(), launchProvider: vi.fn() }));
+vi.mock('../services/messageEvaluator.js', () => ({ evaluateMessages: vi.fn(), generateReplyBody: vi.fn() }));
+vi.mock('../services/messageActions.js', () => ({ executeAction: vi.fn() }));
+vi.mock('../services/messageTriageRules.js', () => ({ listRules: vi.fn(), deleteRule: vi.fn() }));
+vi.mock('../services/messageTokenExtractor.js', () => ({ getToken: vi.fn(), getTokenStatus: vi.fn(), testApi: vi.fn(), clearTokenCache: vi.fn() }));
+
+import messagesRoutes from './messages.js';
+import { errorEvents } from '../lib/errorHandler.js';
+import { getDraft, updateDraft } from '../services/messageDrafts.js';
+
+const ACCOUNT_ID = '11111111-1111-4111-8111-111111111111';
+const OTHER_ACCOUNT_ID = '22222222-2222-4222-8222-222222222222';
+const LOCAL_ID = '33333333-3333-4333-8333-333333333333';
+const observeError = () => {};
+beforeAll(() => errorEvents.on('error', observeError));
+afterAll(() => errorEvents.off('error', observeError));
+let app;
+const metadata = () => ({ data: {
+ id: 'gmail-message-1', threadId: 'gmail-thread-1', payload: { headers: [
+ { name: 'Message-ID', value: '<[email protected]>' },
+ { name: 'References', value: '<[email protected]>\r\n <[email protected]>' },
+ { name: 'Subject', value: 'Example conversation' }
+ ] }
+} });
+
+beforeEach(() => {
+ vi.resetAllMocks();
+ doubles.files.clear();
+ doubles.original = {
+ id: LOCAL_ID, accountId: ACCOUNT_ID, apiId: 'gmail-message-1',
+ threadId: 'conv-local-hash', conversationId: 'gmail-thread-1'
+ // Legacy cache: no persisted RFC Message-ID or References.
+ };
+ doubles.metadata.mockResolvedValue(metadata());
+ doubles.auth.getAccessToken.mockResolvedValue({ token: 'example-token' });
+ doubles.submit.mockResolvedValue({ data: { id: 'gmail-reply-1' } });
+ app = express();
+ app.use(express.json());
+ app.use('/api/messages', messagesRoutes);
+});
+
+async function approvedReply(overrides = {}) {
+ const created = await request(app).post('/api/messages/drafts').send({
+ accountId: ACCOUNT_ID, replyToMessageId: LOCAL_ID, threadId: 'conv-local-hash',
+ to: ['[email protected]'], subject: 'Re: Example conversation', body: 'Example reply', ...overrides
+ });
+ expect(created.status).toBe(201);
+ const { id } = created.body;
+ expect((await request(app).post(`/api/messages/drafts/${id}/approve`)).status).toBe(200);
+ return id;
+}
+
+describe('HTTP Gmail reply identity', () => {
+ it('resolves legacy local IDs once, preserves the RFC chain and submits the provider thread and subject', async () => {
+ const id = await approvedReply();
+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).body).toEqual({ success: true });
+ expect((await getDraft(id)).status).toBe('sent');
+ expect(doubles.metadata).toHaveBeenCalledTimes(1);
+ expect(doubles.metadata).toHaveBeenCalledWith({
+ userId: 'me', id: 'gmail-message-1', format: 'metadata',
+ metadataHeaders: ['Message-ID', 'References', 'In-Reply-To', 'Subject']
+ }, { timeout: 10_000, retry: false });
+ expect(doubles.submit).toHaveBeenCalledTimes(1);
+ const [{ requestBody }, options] = doubles.submit.mock.calls[0];
+ expect(requestBody.threadId).toBe('gmail-thread-1');
+ expect(options).toEqual({ retry: false });
+ const mime = Buffer.from(requestBody.raw, 'base64url').toString('utf8');
+ expect(mime).toContain('In-Reply-To: <[email protected]>\r\n');
+ expect(mime).toContain('References: <[email protected]> <[email protected]> <[email protected]>\r\n');
+ expect(mime).toContain('Subject: Example conversation\r\n');
+ expect(mime).toContain('\r\n\r\nExample reply');
+ expect(mime).not.toContain(LOCAL_ID);
+ expect(mime).not.toContain('conv-local-hash');
+ expect((await getDraft(id)).replyToMessageId).toBe(LOCAL_ID);
+ });
+
+ it('fills an empty reply subject and uses the parent In-Reply-To when References are absent', async () => {
+ const data = metadata();
+ data.data.payload.headers = data.data.payload.headers.filter(h => h.name !== 'References');
+ data.data.payload.headers.push({ name: 'In-Reply-To', value: '<[email protected]>' });
+ doubles.metadata.mockResolvedValue(data);
+ const id = await approvedReply({ subject: '', threadId: null });
+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).status).toBe(200);
+ const mime = Buffer.from(doubles.submit.mock.calls[0][0].requestBody.raw, 'base64url').toString('utf8');
+ expect(mime).toContain('References: <[email protected]> <[email protected]>\r\n');
+ expect(mime).toContain('Subject: Example conversation\r\n');
+ });
+
+ it.each([
+ ['missing original', () => { doubles.original = null; }, {}, 'REPLY_TARGET_NOT_FOUND'],
+ ['wrong account', () => {}, { accountId: OTHER_ACCOUNT_ID }, 'REPLY_TARGET_NOT_FOUND'],
+ ['local thread mismatch', () => {}, { threadId: 'other-local-thread' }, 'THREAD_MISMATCH'],
+ ['metadata lookup failure', () => { doubles.metadata.mockRejectedValue(new Error('Example lookup failure')); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],
+ ['provider identity mismatch', () => { const data = metadata(); data.data.id = 'other-gmail-message'; doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],
+ ['provider thread mismatch', () => { const data = metadata(); data.data.threadId = 'other-gmail-thread'; doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_UNAVAILABLE'],
+ ['missing RFC identity', () => { const data = metadata(); data.data.payload.headers = data.data.payload.headers.filter(h => h.name !== 'Message-ID'); doubles.metadata.mockResolvedValue(data); }, {}, 'GMAIL_REPLY_METADATA_INVALID'],
+ ['incompatible subject', () => {}, { subject: 'Different conversation' }, 'GMAIL_REPLY_SUBJECT_MISMATCH']
+ ])('refuses %s without submission, approval consumption or send attempts', async (_label, arrange, overrides, code) => {
+ arrange();
+ const id = await approvedReply(overrides);
+ const response = await request(app).post(`/api/messages/drafts/${id}/send`);
+ expect(response.status).toBeGreaterThanOrEqual(400);
+ expect(response.body).toMatchObject({ code, error: expect.any(String) });
+ expect(await getDraft(id)).toMatchObject({ status: 'approved', sendAttempts: [] });
+ expect(doubles.submit).not.toHaveBeenCalled();
+ });
+
+ it('revalidates the claimed subject if it changes while provider metadata is loading', async () => {
+ const id = await approvedReply();
+ doubles.metadata.mockImplementationOnce(async () => {
+ await updateDraft(id, { subject: 'Different conversation' });
+ return metadata();
+ });
+ const response = await request(app).post(`/api/messages/drafts/${id}/send`);
+ expect(response.body.code).toBe('GMAIL_REPLY_SUBJECT_MISMATCH');
+ expect(await getDraft(id)).toMatchObject({ status: 'approved', subject: 'Different conversation', sendAttempts: [] });
+ expect(doubles.submit).not.toHaveBeenCalled();
+ });
+
+ it('sends new messages without metadata lookup or threading fields', async () => {
+ const id = await approvedReply({ replyToMessageId: null, threadId: null, subject: 'New message' });
+ expect((await request(app).post(`/api/messages/drafts/${id}/send`)).status).toBe(200);
+ expect(doubles.metadata).not.toHaveBeenCalled();
+ const body = doubles.submit.mock.calls[0][0].requestBody;
+ expect(Object.keys(body)).toEqual(['raw']);
+ const mime = Buffer.from(body.raw, 'base64url').toString('utf8');
+ expect(mime).toContain('Subject: New message\r\n');
+ expect(mime).not.toMatch(/In-Reply-To:|References:/);
+ expect(doubles.files.has(join('/mock/messages', 'drafts.json'))).toBe(true);
+ });
+});
diff --git a/server/services/messageDrafts.js b/server/services/messageDrafts.js
index 9f0d6c60e7..12609be825 100644
--- a/server/services/messageDrafts.js
+++ b/server/services/messageDrafts.js
@@ -142,7 +142,7 @@ export async function updateDraft(id, updates) {
// The eligibility check and transition share the same queue as every draft edit.
// Provider I/O happens after this promise resolves, never while holding the queue.
-export async function claimDraftForSend(id) {
+export async function claimDraftForSend(id, validateClaim) {
return queueWrite(async () => {
const drafts = await loadDrafts();
const draft = drafts.find(d => d.id === id);
@@ -150,6 +150,7 @@ export async function claimDraftForSend(id) {
if (draft.status !== 'approved') {
throw new ServerError('Draft must be approved and not already sending or sent', { status: 409, code: 'DRAFT_STATE_CONFLICT' });
}
+ validateClaim?.(draft);
// Keep uncertain delivery blocked after a crash; never silently retry it.
draft.status = 'sending';
draft.updatedAt = new Date().toISOString();
diff --git a/server/services/messageGmailSync.js b/server/services/messageGmailSync.js
index 3904b589ff..6f0a3fa590 100644
--- a/server/services/messageGmailSync.js
+++ b/server/services/messageGmailSync.js
@@ -9,6 +9,7 @@ import { v4 as uuidv4 } from '../lib/uuid.js';
import crypto from 'crypto';
import { getAuthenticatedClient } from './googleAuth.js';
import { htmlToText as sharedHtmlToText } from '../lib/htmlToText.js';
+import { ServerError } from '../lib/errorHandler.js';
function makeExternalId(gmailId) {
return 'api-gmail-' + crypto.createHash('md5').update(gmailId).digest('hex').slice(0, 12);
@@ -127,7 +128,9 @@ export async function collectMessageIds(passes, listFn, { onProgress } = {}) {
}
function getHeader(headers, name) {
- return headers?.find(h => h.name.toLowerCase() === name.toLowerCase())?.value || '';
+ if (!Array.isArray(headers)) return '';
+ const value = headers.find(h => typeof h?.name === 'string' && h.name.toLowerCase() === name.toLowerCase())?.value;
+ return typeof value === 'string' ? value : '';
}
function decodeBase64Url(str) {
@@ -399,13 +402,82 @@ const GMAIL_DELIVERY_UNKNOWN = {
// is a timeout, which is exactly the ambiguity this classification exists for.
const isDefinitiveRefusal = (status) => Number.isInteger(status) && status >= 400 && status < 500 && status !== 408;
+const replyRefusal = (code, error, status = 422) => ({ refusal: { success: false, status, code, error } });
+const replySubject = subject => String(subject || '').replace(/^(?:\s*re:\s*)+/i, '').trim();
+const unfoldHeader = value => value.replace(/\r?\n[ \t]+/g, ' ').trim();
+
+// Resolve legacy cached messages through their Gmail API identity. Never persist
+// RFC metadata or interpret a PortOS UUID/thread hash as a provider identity.
+export async function prepareGmailReply(account, draft, original) {
+ if (!original || original.id !== draft.replyToMessageId || original.accountId !== account.id) {
+ return replyRefusal('REPLY_TARGET_NOT_FOUND', 'The original message is unavailable in this account. Sync this account and select the reply target again.', 404);
+ }
+ if (draft.threadId && draft.threadId !== original.threadId) {
+ return replyRefusal('THREAD_MISMATCH', 'The reply target does not belong to the selected conversation. Select the original message again.');
+ }
+ if (!original.apiId) {
+ return replyRefusal('GMAIL_REPLY_METADATA_UNAVAILABLE', 'The original message has no Gmail identity. Sync it through the Gmail API before replying.');
+ }
+ const auth = await getAuthenticatedClient();
+ if (!auth) return replyRefusal('GMAIL_NOT_CONFIGURED', 'Google OAuth is not configured. Connect Google before sending this reply.', 502);
+ const gmailClient = gmail({ version: 'v1', auth });
+ const metadata = await gmailClient.users.messages.get({
+ userId: 'me', id: original.apiId, format: 'metadata',
+ metadataHeaders: ['Message-ID', 'References', 'In-Reply-To', 'Subject']
+ }, { timeout: 10_000, retry: false }).then(res => res.data, () => null);
+ if (!metadata || metadata.id !== original.apiId || !metadata.threadId
+ || (original.conversationId && metadata.threadId !== original.conversationId)) {
+ return replyRefusal('GMAIL_REPLY_METADATA_UNAVAILABLE', 'Gmail could not verify the original message and conversation. Sync the account or reconnect Google, then try again.', 502);
+ }
+ const headers = metadata.payload?.headers;
+ const messageId = unfoldHeader(getHeader(headers, 'Message-ID'));
+ const subject = unfoldHeader(getHeader(headers, 'Subject'));
+ if (!/^<[^<>\s@]+@[^<>\s@]+>$/.test(messageId) || /[\r\n]/.test(subject)) {
+ return replyRefusal('GMAIL_REPLY_METADATA_INVALID', 'The original message lacks valid reply headers. Select a different message or send a new email.');
+ }
+ // RFC 2822: use the parent's References, falling back to its single
+ // In-Reply-To when References is absent, then append the parent's Message-ID.
+ const parentReferences = getHeader(headers, 'References');
+ const referenceIds = unfoldHeader(parentReferences || getHeader(headers, 'In-Reply-To')).match(/<[^<>\s@]+@[^<>\s@]+>/g) || [];
+ const references = parentReferences ? referenceIds : (referenceIds.length === 1 ? referenceIds : []);
+ const delivery = Object.freeze({
+ accountId: account.id, replyToMessageId: original.id, localThreadId: draft.threadId,
+ messageId, references: [...references, messageId].join(' '), threadId: metadata.threadId, subject
+ });
+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);
+ return refusal ? { refusal } : { delivery };
+}
+
+function gmailReplyDraftRefusal(account, draft, delivery) {
+ if (!delivery || delivery.accountId !== account.id || draft.accountId !== account.id || draft.sendVia !== 'api'
+ || delivery.replyToMessageId !== draft.replyToMessageId
+ || delivery.localThreadId !== draft.threadId) {
+ return replyRefusal('GMAIL_REPLY_NOT_PREPARED', 'The Gmail reply target was not verified. Select the original message and try again.').refusal;
+ }
+ if (draft.subject && replySubject(draft.subject) !== replySubject(delivery.subject)) {
+ return replyRefusal('GMAIL_REPLY_SUBJECT_MISMATCH', 'A reply must keep the original conversation subject. Restore the original subject or send a new email.').refusal;
+ }
+ return null;
+}
+
+// Called inside the draft store's claim queue, so an edit while metadata is
+// loading cannot consume approval or send an incompatible reply subject.
+export function assertGmailReplyDraft(account, draft, delivery) {
+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);
+ if (refusal) throw new ServerError(refusal.error, { status: refusal.status, code: refusal.code });
+}
+
/**
* Send email via Gmail API.
* @param {object} account - Account config
* @param {object} draft - Draft with to, cc, subject, body
* @returns {{ success: boolean, deliveryUnknown?: boolean, error?: string }}
*/
-export async function sendGmail(account, draft) {
+export async function sendGmail(account, draft, delivery = null) {
+ if (draft.replyToMessageId) {
+ const refusal = gmailReplyDraftRefusal(account, draft, delivery);
+ if (refusal) return refusal;
+ }
const auth = await getAuthenticatedClient();
if (!auth) {
return { success: false, error: 'Google OAuth not configured', status: 502, code: 'GMAIL_NOT_CONFIGURED' };
@@ -425,7 +497,7 @@ export async function sendGmail(account, draft) {
const toLine = Array.isArray(draft.to) ? draft.to.join(', ') : draft.to;
const lines = [
`To: ${toLine}`,
- `Subject: ${draft.subject || ''}`,
+ `Subject: ${delivery?.subject ?? draft.subject ?? ''}`,
'Content-Type: text/plain; charset=utf-8',
'MIME-Version: 1.0'
];
@@ -433,10 +505,9 @@ export async function sendGmail(account, draft) {
const ccLine = Array.isArray(draft.cc) ? draft.cc.join(', ') : draft.cc;
lines.push(`Cc: ${ccLine}`);
}
- if (draft.replyToMessageId && draft.threadId) {
- // For replies, set In-Reply-To and References headers
- lines.push(`In-Reply-To: ${draft.replyToMessageId}`);
- lines.push(`References: ${draft.replyToMessageId}`);
+ if (draft.replyToMessageId) {
+ lines.push(`In-Reply-To: ${delivery.messageId}`);
+ lines.push(`References: ${delivery.references}`);
}
lines.push('', draft.body || '');
@@ -446,7 +517,7 @@ export async function sendGmail(account, draft) {
// after a lost acknowledgement is a duplicate message.
const sendError = await gmailClient.users.messages.send({
userId: 'me',
- requestBody: { raw }
+ requestBody: { raw, ...(draft.replyToMessageId ? { threadId: delivery.threadId } : {}) }
}, { retry: false }).then(() => null, err => err);
if (sendError) {
diff --git a/server/services/messageSender.js b/server/services/messageSender.js
index 685f01ca6c..87902b7866 100644
--- a/server/services/messageSender.js
+++ b/server/services/messageSender.js
@@ -41,19 +41,28 @@ export async function sendDraft(draftId, io) {
}
let delivery = null;
+ let validateClaim;
if (draft.sendVia === 'playwright') {
const prepared = await prepareBrowserDelivery(account, draft);
if (prepared.refusal) return prepared.refusal;
delivery = prepared.delivery;
+ } else if (draft.replyToMessageId) {
+ const { getMessage } = await import('./messageSync.js');
+ const { prepareGmailReply, assertGmailReplyDraft } = await import('./messageGmailSync.js');
+ const original = await getMessage(account.id, draft.replyToMessageId);
+ const prepared = await prepareGmailReply(account, draft, original);
+ if (prepared.refusal) return prepared.refusal;
+ delivery = prepared.delivery;
+ validateClaim = claimed => assertGmailReplyDraft(account, claimed, delivery);
}
- draft = await claimDraftForSend(draftId);
+ draft = await claimDraftForSend(draftId, validateClaim);
console.log(`📧 Sending draft ${draft.id} via ${draft.sendVia}`);
const dispatch = async () => {
if (draft.sendVia === 'api') {
const { sendGmail } = await import('./messageGmailSync.js');
- return sendGmail(account, draft);
+ return delivery ? sendGmail(account, draft, delivery) : sendGmail(account, draft);
}
const { sendPlaywright } = await import('./messagePlaywrightSync.js');
return sendPlaywright(account, draft, delivery);