Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: require complete Google MCP event pages before pruning calendar cache (#10869)
Adam Eivy10690aaa4260
diff --git a/server/services/calendarGoogleSync.js b/server/services/calendarGoogleSync.jsindex 7a3ad195d9..1a399cff37 100644--- a/server/services/calendarGoogleSync.js+++ b/server/services/calendarGoogleSync.js@@ -83,7 +83,8 @@ export function getSyncDateRange(pastDays = 7, futureDays = 30) {* `options.prune` (default true) removes cached events for this subcalendar* that the incoming batch didn't mention — correct only when `rawEvents` is the* COMPLETE set for the range. A caller working from a possibly-truncated payload- * (a CLI that exited non-zero mid-stream) must pass `prune: false`, or the+ * (a CLI that exited non-zero mid-stream, or a source that still had event+ * pages left) must pass `prune: false`, or the* missing tail reads as "these events were deleted upstream" and destroys real* calendar data. `options.status` labels the resulting sync for the UI.*@@ -218,10 +219,11 @@ export async function pushSyncEvents(accountId, calendarId, calendarName, rawEve* what a complete Google response actually means, matching what the direct-API* mapper emits for the same reason.*- * Only a COMPLETE payload earns this. A partial one (the CLI died mid-stream)- * is relayed untouched, so a truncated event that lost its conference fields- * reads as "unknown" rather than "cleared" — the same rule that already stops a- * partial payload from driving a prune. Clearing a link is destructive too.+ * Only a COMPLETE calendar earns this (see `incompleteReason`). A partial one —+ * the CLI died mid-stream, or the source still had event pages — is relayed+ * untouched, so a truncated event that lost its conference fields reads as+ * "unknown" rather than "cleared", the same rule that stops it from driving a+ * prune. Clearing a link is destructive too.*/function withExplicitConferenceFields(event) {return { ...event, organizer: event?.organizer ?? null, attendees: event?.attendees ?? [], hangoutLink: event?.hangoutLink ?? null, conferenceData: event?.conferenceData ?? null };@@ -261,10 +263,12 @@ async function runMcpSyncAccount(accountId, io) {${calendarList}-For EACH calendar, call gcal_list_events with the calendarId, timeMin, and timeMax. Use maxResults=250.+For EACH calendar, call gcal_list_events with the calendarId, timeMin, and timeMax. Use maxResults=250. If a response carries a nextPageToken, call gcal_list_events again with the same calendarId, timeMin and timeMax plus pageToken set to that token, and repeat until a response carries no nextPageToken. Concatenate the events of every page into one array for that calendar.After fetching ALL calendars, output ONLY a single JSON object (no markdown fences, no explanation) with this exact structure:-{"calendars":[{"calendarId":"...","calendarName":"...","events":[...raw events from gcal_list_events response...]}]}+{"calendars":[{"calendarId":"...","calendarName":"...","timeMin":"...","timeMax":"...","complete":true,"nextPageToken":null,"events":[...raw events from every page...]}]}++Per calendar, "timeMin" and "timeMax" echo the exact values you queried with. Set "complete" to true ONLY when you fetched every page and the final response carried no nextPageToken; otherwise set "complete" to false and "nextPageToken" to the last token you still hold. Never claim completeness you did not verify.Include the full events arrays as returned by gcal_list_events, with every field each event carries — do NOT abbreviate, summarize, or drop fields. In particular keep "conferenceData" and "hangoutLink" exactly as returned, and omit them only when the event itself does not have them. Output NOTHING else — just the JSON.`;@@ -275,8 +279,8 @@ Include the full events arrays as returned by gcal_list_events, with every field// JSON may be cut off mid-array, so every calendar it does describe is// treated as incomplete. Upsert what arrived, but never prune from it —// absent events mean "the CLI died", not "deleted upstream".- const { partial, stderrTail } = result;- const status = partial ? 'partial' : 'success';+ const { partial: cliPartial, stderrTail } = result;+ const cliReason = stderrTail || `CLI exited with code ${result.exitCode}`;// Parse Claude's output and push eventsconst parsed = parseCalendarJson(result.output.split(prompt).join(''));@@ -287,6 +291,15 @@ Include the full events arrays as returned by gcal_list_events, with every fieldthrow new ServerError(`Failed to parse calendar data from Claude response${reason}`, { status: 502 });}+ // A zero exit proves the CLI finished, NOT that the source handed over every+ // page: each calendar must vouch for itself (#10869). Only a calendar that+ // does earns the prune and the authoritative metadata clear.+ const requestedIds = new Map(enabledCalendars.map(sc => [sc.calendarId, sc.name]));+ const returnedCounts = new Map();+ for (const cal of parsed.calendars) returnedCounts.set(cal.calendarId, (returnedCounts.get(cal.calendarId) || 0) + 1);+ const context = { requestedIds, duplicateIds: new Set([...returnedCounts].filter(([, n]) => n > 1).map(([id]) => id)), timeMin, timeMax };+ const incompleteReasons = [];+let totalNew = 0;let totalUpdated = 0;let totalPruned = 0;@@ -294,20 +307,35 @@ Include the full events arrays as returned by gcal_list_events, with every fieldfor (const cal of parsed.calendars) {if (!cal.calendarId || !Array.isArray(cal.events)) continue;+ const calendarName = cal.calendarName || cal.calendarId;+ const why = cliPartial ? 'the CLI did not exit cleanly' : incompleteReason(cal, context);+ if (why && !cliPartial) incompleteReasons.push(`"${calendarName}": ${why}`);+ const complete = !why;const syncResult = await pushSyncEvents(accountId,cal.calendarId,- cal.calendarName || cal.calendarId,- partial ? cal.events : cal.events.map(withExplicitConferenceFields),+ calendarName,+ complete ? cal.events.map(withExplicitConferenceFields) : cal.events,null,- { prune: !partial, status, dateRange: { pastDate, futureDate } },+ { prune: complete, status: complete ? 'success' : 'partial', dateRange: { pastDate, futureDate } },);totalNew += syncResult.newEvents;totalUpdated += syncResult.updated;totalPruned += syncResult.pruned;- results.push({ calendarId: cal.calendarId, calendarName: cal.calendarName, ...syncResult });+ results.push({ calendarId: cal.calendarId, calendarName: cal.calendarName, complete, ...(why ? { reason: why } : {}), ...syncResult });+ }++ // A requested calendar the response never described was not reconciled at all.+ if (!cliPartial) {+ for (const [id, name] of requestedIds) {+ if (!returnedCounts.has(id)) incompleteReasons.push(`"${name || id}": missing from the response`);+ }}+ const partial = cliPartial || incompleteReasons.length > 0;+ const status = partial ? 'partial' : 'success';+ const reason = cliPartial ? cliReason : summarizeReasons(incompleteReasons);+await updateSyncStatus(accountId, status);io?.emit('calendar:sync:completed', {accountId,@@ -316,10 +344,10 @@ Include the full events arrays as returned by gcal_list_events, with every fieldpruned: totalPruned,status,method: 'mcp',- ...(partial ? { reason: stderrTail || `CLI exited with code ${result.exitCode}` } : {}),+ ...(partial ? { reason } : {}),});if (partial) {- console.warn(`⚠️ MCP sync PARTIAL for ${account.name} (exit ${result.exitCode}): ${totalNew} new, ${totalUpdated} updated, prune skipped across ${results.length} calendars`);+ console.warn(`⚠️ MCP sync PARTIAL for ${account.name}: ${reason} — ${totalNew} new, ${totalUpdated} updated, ${totalPruned} pruned across ${results.length} calendars`);} else {console.log(`📅 MCP sync complete for ${account.name}: ${totalNew} new, ${totalUpdated} updated, ${totalPruned} pruned across ${results.length} calendars`);}@@ -330,7 +358,7 @@ Include the full events arrays as returned by gcal_list_events, with every fieldpruned: totalPruned,calendars: results,status,- ...(partial ? { reason: stderrTail || `CLI exited with code ${result.exitCode}` } : {}),+ ...(partial ? { reason } : {}),};};@@ -342,6 +370,31 @@ Include the full events arrays as returned by gcal_list_events, with every field});}+const hasPageToken = token => token !== undefined && token !== null && token !== '';+const sameInstant = (a, b) => typeof a === 'string' && Date.parse(a) === Date.parse(b);++/**+ * Why one returned calendar cannot be treated as the COMPLETE event set for the+ * requested window, or null when it can (#10869). The CLI exiting 0 only proves+ * the process finished; Google paginates, so a first page alone also exits 0.+ * Only an explicit `complete: true`, no remaining `nextPageToken`, a calendar we+ * asked for (once) and an echo of the requested window together license a prune.+ * Anything missing, legacy or contradictory degrades to upsert-only.+ */+function incompleteReason(cal, { requestedIds, duplicateIds, timeMin, timeMax }) {+ if (!requestedIds.has(cal.calendarId)) return 'calendar was not requested';+ if (duplicateIds.has(cal.calendarId)) return 'calendar was returned more than once';+ if (hasPageToken(cal.nextPageToken)) return 'the source still has more event pages';+ if (cal.complete !== true) return cal.complete === false ? 'the source reported the event list as incomplete' : 'no completion marker';+ if (!sameInstant(cal.timeMin, timeMin) || !sameInstant(cal.timeMax, timeMax)) return 'window metadata is missing or differs from the request';+ return null;+}++function summarizeReasons(reasons) {+ const shown = reasons.slice(0, 3).join('; ');+ return reasons.length > 3 ? `${shown}; and ${reasons.length - 3} more` : shown;+}+const isCalendarPayload = value => Array.isArray(value?.calendars) && value.calendars.every(cal =>typeof cal?.calendarId === 'string' && cal.calendarId.trim() && Array.isArray(cal.events)&& cal.events.every(event => event !== null && typeof event === 'object' && !Array.isArray(event)));diff --git a/server/services/calendarGoogleSync.test.js b/server/services/calendarGoogleSync.test.jsindex 973bee75a2..0679427703 100644--- a/server/services/calendarGoogleSync.test.js+++ b/server/services/calendarGoogleSync.test.js@@ -69,6 +69,21 @@ const rawEvent = (id, summary) => ({const savedCache = () => saveCache.mock.calls.at(-1)[1];+// The completion contract (#10869): a zero-exit reply only licenses a prune for a+// calendar that echoes the requested window, carries `complete: true` and has no+// remaining page token. The window is read back out of the prompt the service+// actually sent, so these fixtures cannot drift from what it asked for.+const windowFromPrompt = (prompt) => {+ const [, timeMin, timeMax] = prompt.match(/date range (\d{4}-[\w:.-]+Z) to (\d{4}-[\w:.-]+Z)/);+ return { timeMin, timeMax };+};+// `undefined` overrides drop a field from the JSON, simulating a legacy envelope.+const completeCalendar = (prompt, cal) => ({ calendarName: 'Work', complete: true, nextPageToken: null, ...windowFromPrompt(prompt), ...cal });+const cliReply = (calendars, cli = {}) => async ({ prompt }) => ({+ text: JSON.stringify({ calendars: calendars.map((cal) => completeCalendar(prompt, cal)) }),+ exitCode: 0, partial: false, stderrTail: '', ...cli,+});+describe('mcpSyncAccount CLI exit-status agreement (#5302)', () => {beforeEach(() => {vi.clearAllMocks();@@ -77,12 +92,7 @@ describe('mcpSyncAccount CLI exit-status agreement (#5302)', () => {});it('prunes cache-only events on a clean exit 0 and records success', async () => {- runCliProviderPrompt.mockResolvedValue({- text: JSON.stringify({ calendars: [{ calendarId: CAL_ID, calendarName: 'Work', events: [rawEvent('upstream-2', 'Design review')] }] }),- exitCode: 0,- partial: false,- stderrTail: '',- });+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [rawEvent('upstream-2', 'Design review')] }]));const result = await mcpSyncAccount(ACCOUNT_ID, null);@@ -196,7 +206,7 @@ describe('MCP transcript extraction', () => {const event = rawEvent('new-event', 'Review [draft] {v2}');event.description = 'Keep ```json {"example":[]} ``` in the notes.';runCliProviderPrompt.mockImplementation(async ({ prompt }) => ({- text: `${prompt}\nMetadata: {"request":"complete"}\n${prompt}\nActual: \`\`\`json\n${JSON.stringify({ calendars: [{ calendarId: CAL_ID, events: [event] }] })}\n\`\`\``,+ text: `${prompt}\nMetadata: {"request":"complete"}\n${prompt}\nActual: \`\`\`json\n${JSON.stringify({ calendars: [completeCalendar(prompt, { calendarId: CAL_ID, events: [event] })] })}\n\`\`\``,exitCode: 0, partial: false,}));@@ -207,12 +217,13 @@ describe('MCP transcript extraction', () => {});it('accepts an empty event snapshot and an empty calendar response', async () => {- runCliProviderPrompt.mockResolvedValue({ text: JSON.stringify({ calendars: [{ calendarId: CAL_ID, events: [] }] }), exitCode: 0 });+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [] }]));expect((await mcpSyncAccount(ACCOUNT_ID, null)).pruned).toBe(1);expect(savedCache().events).toEqual([]);saveCache.mockClear();runCliProviderPrompt.mockResolvedValue({ text: '{"calendars":[]}', exitCode: 0 });- expect((await mcpSyncAccount(ACCOUNT_ID, null)).calendars).toEqual([]);+ // Nothing was described, so nothing was reconciled: not a success.+ expect(await mcpSyncAccount(ACCOUNT_ID, null)).toMatchObject({ calendars: [], status: 'partial' });expect(saveCache).not.toHaveBeenCalled();});@@ -245,6 +256,123 @@ describe('MCP transcript extraction', () => {});});+// #10869: a zero CLI exit proves the process finished, not that the source+// handed over every page. These pin the per-calendar completion contract at the+// service boundary, where a wrong answer silently deletes real cached events and+// reports the sync as a success.+describe('Google MCP page completeness (#10869)', () => {+ const OLD_ROOM = 'https://meet.example.com/old-room';+ const TWO_CALENDARS = '[email protected]';++ beforeEach(() => {+ vi.clearAllMocks();+ getAccount.mockResolvedValue(account());+ loadCache.mockResolvedValue({ events: cachedEvents() });+ });++ it('asks the model to drain every page and echo the window and completion state', async () => {+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [] }]));+ await mcpSyncAccount(ACCOUNT_ID, null);+ const { prompt } = runCliProviderPrompt.mock.calls.at(-1)[0];+ expect(prompt).toMatch(/maxResults=250/);+ expect(prompt).toMatch(/pageToken set to that token/);+ expect(prompt).toMatch(/"complete":true,"nextPageToken":null/);+ });++ it('reconciles a fully drained multi-page calendar: upserts every page and prunes only the genuine deletion', async () => {+ const firstPage = Array.from({ length: 3 }, (_, i) => rawEvent(`page1-${i}`, `Page one ${i}`));+ const secondPage = [rawEvent('page2-0', 'Page two 0')];+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [...firstPage, ...secondPage] }]));++ const result = await mcpSyncAccount(ACCOUNT_ID, null);++ expect(result).toMatchObject({ status: 'success', newEvents: 4, pruned: 1 });+ expect(result.reason).toBeUndefined();+ expect(savedCache().events.map((e) => e.title).sort()).toEqual(['Page one 0', 'Page one 1', 'Page one 2', 'Page two 0']);+ });++ // The synthetic reproduction from the issue: one received event, a token that+ // says more exist, a different event already cached in the window.+ it.each([+ ['a remaining nextPageToken', { nextPageToken: 'synthetic-next-page' }, 'more event pages'],+ ['a legacy envelope with no completion fields', { complete: undefined, nextPageToken: undefined, timeMin: undefined, timeMax: undefined }, 'no completion marker'],+ ['an explicit complete:false', { complete: false }, 'incomplete'],+ ['complete:true but no window echo', { timeMin: undefined, timeMax: undefined }, 'window metadata'],+ ['a window that contradicts the request', { timeMax: '2000-01-01T00:00:00.000Z' }, 'window metadata'],+ ['a calendar that was never requested', { calendarId: '[email protected]' }, 'not requested'],+ ])('upserts but never prunes on %s, and reports a partial result', async (_name, override, reasonFragment) => {+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [rawEvent('upstream-2', 'Design review')], ...override }]));+ const io = { emit: vi.fn() };++ const result = await mcpSyncAccount(ACCOUNT_ID, io);++ expect(result).toMatchObject({ status: 'partial', pruned: 0, reason: expect.stringContaining(reasonFragment) });+ expect(updateSyncStatus).toHaveBeenLastCalledWith(ACCOUNT_ID, 'partial');+ expect(io.emit).toHaveBeenCalledWith('calendar:sync:completed', expect.objectContaining({+ status: 'partial', pruned: 0, reason: expect.stringContaining(reasonFragment),+ }));+ // The cached event the response never mentioned survives alongside the new one.+ expect(savedCache().events.map((e) => e.title).sort()).toEqual(['Design review', 'Standing 1:1']);+ });++ it('keeps cached conference and participant metadata that an incomplete page omits', async () => {+ const full = {+ ...rawEvent('upstream-move', 'Weekly sync'),+ conferenceData: { entryPoints: [{ entryPointType: 'video', uri: OLD_ROOM }] },+ organizer: { email: '[email protected]' },+ attendees: [{ email: '[email protected]', self: true, responseStatus: 'declined' }],+ };+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [full] }]));+ await mcpSyncAccount(ACCOUNT_ID, null);+ loadCache.mockResolvedValue({ events: [savedCache().events.at(-1)] });++ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, nextPageToken: 'synthetic-next-page', events: [rawEvent('upstream-move', 'Weekly sync')] }]));+ const result = await mcpSyncAccount(ACCOUNT_ID, null);++ expect(result.status).toBe('partial');+ expect(savedCache().events[0]).toMatchObject({+ meetingUrl: OLD_ROOM, organizer: { email: '[email protected]' }, myStatus: 'declined',+ });+ });++ it('treats a calendar returned twice as contradictory instead of trusting either copy', async () => {+ runCliProviderPrompt.mockImplementation(cliReply([+ { calendarId: CAL_ID, events: [rawEvent('upstream-2', 'Design review')] },+ { calendarId: CAL_ID, events: [rawEvent('upstream-3', 'Retro')] },+ ]));++ const result = await mcpSyncAccount(ACCOUNT_ID, null);++ expect(result).toMatchObject({ status: 'partial', pruned: 0, reason: expect.stringContaining('more than once') });+ expect(savedCache().events.map((e) => e.title)).toContain('Standing 1:1');+ });++ it('judges each calendar on its own: the complete one prunes, the one the reply never described is reported', async () => {+ getAccount.mockResolvedValue({+ ...account(),+ subcalendars: [...account().subcalendars, { calendarId: TWO_CALENDARS, name: 'Team', enabled: true, dormant: false }],+ });+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [rawEvent('upstream-2', 'Design review')] }]));++ const result = await mcpSyncAccount(ACCOUNT_ID, null);++ expect(result).toMatchObject({ status: 'partial', pruned: 1, reason: expect.stringContaining('"Team": missing from the response') });+ expect(result.calendars).toEqual([expect.objectContaining({ calendarId: CAL_ID, complete: true, pruned: 1 })]);+ expect(savedCache().events.map((e) => e.title)).toEqual(['Design review']);+ });++ it('a non-zero CLI exit stays partial even when the envelope claims completeness', async () => {+ runCliProviderPrompt.mockImplementation(cliReply(+ [{ calendarId: CAL_ID, events: [rawEvent('upstream-2', 'Design review')] }],+ { exitCode: 1, partial: true, stderrTail: 'stream aborted' },+ ));++ const result = await mcpSyncAccount(ACCOUNT_ID, null);++ expect(result).toMatchObject({ status: 'partial', pruned: 0, reason: 'stream aborted' });+ });+});+// #6289: a Google event's join link is projected into a single cached// `meetingUrl`. The regressions these pin are (a) caching something that is not// a usable video link, and (b) an older producer that omits the conference@@ -257,12 +385,7 @@ describe('meetingUrl projection (#6289)', () => {});const resync = async (raw) => {- runCliProviderPrompt.mockResolvedValue({- text: JSON.stringify({ calendars: [{ calendarId: CAL_ID, calendarName: 'Work', events: [raw] }] }),- exitCode: 0,- partial: false,- stderrTail: '',- });+ runCliProviderPrompt.mockImplementation(cliReply([{ calendarId: CAL_ID, events: [raw] }]));return mcpSyncAccount(ACCOUNT_ID, null);};@@ -537,11 +660,11 @@ describe('Google participant snapshot reconciliation (#8838)', () => {attendees: [{ email: '[email protected]', self: true, responseStatus: 'declined' }],}]);loadCache.mockResolvedValue(savedCache());- const text = JSON.stringify({ calendars: [{ calendarId: CAL_ID, calendarName: 'Work', events: [rawEvent('identity', 'Updated')] }] });- runCliProviderPrompt.mockResolvedValue({ text, exitCode: 1, partial: true });+ const reply = cliReply([{ calendarId: CAL_ID, events: [rawEvent('identity', 'Updated')] }]);+ runCliProviderPrompt.mockImplementation(reply);+ runCliProviderPrompt.mockImplementationOnce(async (args) => ({ ...(await reply(args)), exitCode: 1, partial: true }));await mcpSyncAccount(ACCOUNT_ID, null);expect(savedCache().events[0].myStatus).toBe('declined');- runCliProviderPrompt.mockResolvedValue({ text, exitCode: 0, partial: false });await mcpSyncAccount(ACCOUNT_ID, null);expect(savedCache().events[0]).toMatchObject({ organizer: null, attendees: [] });expect(savedCache().events[0].myStatus).toBeUndefined();