Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(graphql): reconnect a dead GraphQL socket when the app wakes (#9207)

Wolf Mermelsteinf5a69df54bbe
diff --git a/apps/web/AGENTS.md b/apps/web/AGENTS.md
index cd04ecf16e..09a39799d4 100644
--- a/apps/web/AGENTS.md
+++ b/apps/web/AGENTS.md
@@ -398,6 +398,11 @@ resolved session ID; keep the existing `AgentSessionProvider` subscription to
behind ordered writes and never replay alerts when the cache resumes. Link
cache queue/apply spans to receipt and mark listener failures without stopping
the remaining callbacks.
+- The Soup GraphQL websocket pings when the app returns to the foreground or
+ comes back online, and terminates a socket that does not answer within five
+ seconds (close 4499, retried). A phone can wake to a socket that reads open but
+ is dead; the reconnect is what tells followers such as agent sessions
+ (`subscribeGraphqlSoupReconnected`) to refetch what they missed.
- Drain browser cache engines when their last visible consumer hides, then park
without retiring clients; queue background work until a visible owner reopens
storage, retaining accepted mutation settlements even if their caller leaves.
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup-websocket.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup-websocket.ts
index da389c5599..eec4abafe5 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup-websocket.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup-websocket.ts
@@ -48,6 +48,7 @@ const RETRYABLE_WEBSOCKET_CLOSE_CODES = new Set([
1013, // try again later
1014, // bad gateway
4408, // connection initialisation timeout
+ 4499, // terminated by the client, as a socket that stopped answering
4504, // connection acknowledgement timeout
]);
@@ -113,6 +114,81 @@ export function waitToRetryGraphqlSoupWebSocket(
});
}
+/** How long a socket checked by {@link createGraphqlSoupWebSocketLiveness} has to answer. */
+export const GRAPHQL_SOUP_WEBSOCKET_PONG_TIMEOUT_MS = 5_000;
+
+type LivenessSocket = { readonly readyState: number; send(data: string): void };
+
+/** graphql-ws reports the socket it built with `webSocketImpl` as `unknown`. */
+function isLivenessSocket(value: unknown): value is LivenessSocket {
+ return (
+ value !== null &&
+ typeof value === 'object' &&
+ 'readyState' in value &&
+ typeof value.readyState === 'number' &&
+ 'send' in value &&
+ typeof value.send === 'function'
+ );
+}
+
+/**
+ * Pings the connected socket when the app returns to the foreground or
+ * regains connectivity, and terminates it when the server does not answer.
+ *
+ * A phone that sleeps with the app open can wake to a socket the network
+ * dropped without a close frame. It still reads as open, so graphql-ws never
+ * reconnects it: subscriptions hear nothing, and nothing that refetches on a
+ * reconnect runs. Terminating it lets graphql-ws reconnect, and the reconnect
+ * tells every follower to refetch what it missed.
+ */
+export function createGraphqlSoupWebSocketLiveness(terminate: () => void): {
+ connected(socket: unknown): void;
+ pong(received: boolean): void;
+ closed(): void;
+} {
+ let socket: LivenessSocket | undefined;
+ let stopListening: (() => void) | undefined;
+ let waiting:
+ | { timeout: ReturnType<typeof setTimeout>; span: Span }
+ | undefined;
+
+ const settle = (outcome: 'answered' | 'closed' | 'terminated') => {
+ if (!waiting) return;
+ clearTimeout(waiting.timeout);
+ waiting.span.setAttr('websocket.outcome', outcome);
+ waiting.span.end();
+ waiting = undefined;
+ };
+ const check = () => {
+ if (!socket || socket.readyState !== WebSocket.OPEN || waiting) return;
+ const span = Telemetry.span('graphql_soup.websocket.liveness_check');
+ socket.send(JSON.stringify({ type: 'ping' }));
+ waiting = {
+ span,
+ timeout: setTimeout(() => {
+ settle('terminated');
+ terminate();
+ }, GRAPHQL_SOUP_WEBSOCKET_PONG_TIMEOUT_MS),
+ };
+ };
+
+ return {
+ connected(next) {
+ socket = isLivenessSocket(next) ? next : undefined;
+ stopListening ??= onReconnectOpportunity(check);
+ },
+ pong(received) {
+ if (received) settle('answered');
+ },
+ closed() {
+ socket = undefined;
+ settle('closed');
+ stopListening?.();
+ stopListening = undefined;
+ },
+ };
+}
+
/** Converts a DSS HTTP origin into its Soup GraphQL websocket endpoint. */
export function buildGraphqlSoupWebSocketUrl(
dssHost: string,
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
index 3d6a6bd089..05a683d013 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.test.ts
@@ -494,6 +494,11 @@ vi.mock('./graphql-soup-websocket', () => ({
shouldRetryGraphqlSoupWebSocket: () => false,
waitToRetryGraphqlSoupWebSocket: vi.fn(),
createGraphqlSoupWebSocketUrlResolver: () => () => 'ws://dss.test',
+ createGraphqlSoupWebSocketLiveness: () => ({
+ connected: vi.fn(),
+ pong: vi.fn(),
+ closed: vi.fn(),
+ }),
createGraphqlSoupSubscriptionsLifecycle: () => ({
replace: mocks.replaceSubscriptions,
dispose: vi.fn(() => mocks.recordSubscriptionDisposal()),
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
index e6eee43490..f61214551a 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-soup.ts
@@ -111,6 +111,7 @@ import { shouldRetryGraphqlMutation } from './graphql-mutation-retry';
import { notificationReceiptExchange } from './graphql-notification-receipt';
import {
createGraphqlSoupSubscriptionsLifecycle,
+ createGraphqlSoupWebSocketLiveness,
createGraphqlSoupWebSocketUrlResolver,
publishNotificationPatch,
SOUP_GRAPHQL_WEBSOCKET_RETRY_ATTEMPTS,
@@ -422,7 +423,8 @@ function createGraphqlSoupWebSocketClient(
getApiToken: getMacroApiToken,
refreshCookieAuth: fetchToken,
});
- return createGraphqlWsClient({
+ const liveness = createGraphqlSoupWebSocketLiveness(() => client.terminate());
+ const client = createGraphqlWsClient({
url: resolveWebSocketUrl,
connectionParams: () =>
Telemetry.span('graphql.websocket.connection_params', async (span) => {
@@ -441,8 +443,13 @@ function createGraphqlSoupWebSocketClient(
retryWait: waitToRetryGraphqlSoupWebSocket,
on: {
connecting: () => onState('connecting'),
- closed: () => onState('disconnected'),
- connected: () => {
+ closed: () => {
+ liveness.closed();
+ onState('disconnected');
+ },
+ pong: liveness.pong,
+ connected: (socket) => {
+ liveness.connected(socket);
onState('connected');
onConnected();
connections += 1;
@@ -452,6 +459,7 @@ function createGraphqlSoupWebSocketClient(
},
shouldRetry: shouldRetryGraphqlSoupWebSocket,
});
+ return client;
}
function graphqlSoupSubscriptionExchange(websocketClient: GraphqlWsClient) {
diff --git a/apps/web/src/lib/service-clients/service-storage/graphql-ws-recovery.test.ts b/apps/web/src/lib/service-clients/service-storage/graphql-ws-recovery.test.ts
index 3452fba927..d1703e17d9 100644
--- a/apps/web/src/lib/service-clients/service-storage/graphql-ws-recovery.test.ts
+++ b/apps/web/src/lib/service-clients/service-storage/graphql-ws-recovery.test.ts
@@ -1,6 +1,6 @@
import { createClient } from 'graphql-ws';
import { err, ok } from 'neverthrow';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
type ReplayEnvelope = { seq: number; message: { type: string; data: unknown } };
type ConnectArgs = {
@@ -13,6 +13,11 @@ type ConnectArgs = {
const plugin = vi.hoisted(() => ({
connects: [] as ConnectArgs[],
received: [] as Array<{ id: number; type: string; payload: unknown }>,
+ // Connections the network dropped without a close frame: still open on the
+ // client, never answered.
+ silent: new Set<number>(),
+ // Subscriptions answer once and stay open instead of completing.
+ live: false,
}));
vi.mock('@tauri-apps/api/core', () => ({
@@ -24,6 +29,9 @@ vi.mock('@tauri-apps/api/core', () => ({
plugin.connects.push(args as ConnectArgs);
return plugin.connects.length;
}
+ // Nothing reached the native side that the webview did not hear.
+ if (command === 'plugin:websocket|recover')
+ return { messages: [], gap: false };
if (command !== 'plugin:websocket|send') return;
const id = args.id as number;
const message = args.message as { type: string; data: unknown };
@@ -33,6 +41,7 @@ vi.mock('@tauri-apps/api/core', () => ({
}
const frame = JSON.parse(message.data as string);
plugin.received.push({ id, type: frame.type, payload: frame.payload });
+ if (plugin.silent.has(id)) return;
const channel = plugin.connects[id - 1]?.onMessage;
let seq = plugin.received.length;
const reply = (value: object) =>
@@ -43,16 +52,19 @@ vi.mock('@tauri-apps/api/core', () => ({
})
);
if (frame.type === 'connection_init') reply({ type: 'connection_ack' });
+ if (frame.type === 'ping') reply({ type: 'pong' });
if (frame.type === 'subscribe') {
reply({ id: frame.id, type: 'next', payload: { data: { ping: id } } });
- reply({ id: frame.id, type: 'complete' });
+ if (!plugin.live) reply({ id: frame.id, type: 'complete' });
}
}),
}));
import { TauriWebSocketWrapper } from '@macro-inc/collaboration/websocket/platform/tauri-websocket';
import {
+ createGraphqlSoupWebSocketLiveness,
createGraphqlSoupWebSocketUrlResolver,
+ GRAPHQL_SOUP_WEBSOCKET_PONG_TIMEOUT_MS,
SOUP_GRAPHQL_WEBSOCKET_RETRY_ATTEMPTS,
shouldRetryGraphqlSoupWebSocket,
} from './graphql-soup-websocket';
@@ -91,9 +103,15 @@ function cookieClient(refreshCookieAuth: () => Promise<unknown>) {
beforeEach(() => {
plugin.connects.length = 0;
plugin.received.length = 0;
+ plugin.silent.clear();
+ plugin.live = false;
vi.spyOn(console, 'log').mockImplementation(() => {});
});
+afterEach(() => {
+ vi.useRealTimers();
+});
+
describe('GraphQL websocket auth recovery', () => {
it('retries a cookie refresh that failed during an outage', async () => {
const refresh = vi
@@ -156,3 +174,91 @@ describe('GraphQL websocket over the native transport', () => {
await client.dispose();
});
});
+
+describe('GraphQL websocket liveness on resume', () => {
+ // Wired as createGraphqlSoupWebSocketClient wires it.
+ function livenessClient() {
+ const connected: boolean[] = [];
+ const liveness = createGraphqlSoupWebSocketLiveness(() =>
+ client.terminate()
+ );
+ const client = createClient({
+ url: 'wss://dss.test/items/soup/graphql/ws',
+ webSocketImpl: TauriWebSocketWrapper,
+ retryAttempts: SOUP_GRAPHQL_WEBSOCKET_RETRY_ATTEMPTS,
+ retryWait: async () => {},
+ shouldRetry: shouldRetryGraphqlSoupWebSocket,
+ on: {
+ closed: () => liveness.closed(),
+ pong: liveness.pong,
+ connected: (socket, _payload, wasRetry) => {
+ liveness.connected(socket);
+ connected.push(wasRetry);
+ },
+ },
+ });
+ return { client, connected };
+ }
+
+ function follow(client: ReturnType<typeof createClient>) {
+ const values: unknown[] = [];
+ const errors: unknown[] = [];
+ const unsubscribe = client.subscribe(
+ { query: 'subscription { ping }' },
+ {
+ next: (value) => values.push(value),
+ error: (error) => errors.push(error),
+ complete: () => undefined,
+ }
+ );
+ return { values, errors, unsubscribe };
+ }
+
+ function returnToForeground() {
+ document.dispatchEvent(new Event('visibilitychange'));
+ }
+
+ it('reconnects a socket that stopped answering while the app slept', async () => {
+ vi.useFakeTimers();
+ plugin.live = true;
+ const { client, connected } = livenessClient();
+ const subscription = follow(client);
+ await vi.waitFor(() =>
+ expect(subscription.values).toEqual([{ data: { ping: 1 } }])
+ );
+
+ plugin.silent.add(1);
+ returnToForeground();
+ expect(plugin.received.at(-1)?.type).toBe('ping');
+ await vi.advanceTimersByTimeAsync(GRAPHQL_SOUP_WEBSOCKET_PONG_TIMEOUT_MS);
+
+ await vi.waitFor(() =>
+ expect(subscription.values).toEqual([
+ { data: { ping: 1 } },
+ { data: { ping: 2 } },
+ ])
+ );
+ expect(connected).toEqual([false, true]);
+ expect(subscription.errors).toEqual([]);
+ subscription.unsubscribe();
+ await client.dispose();
+ });
+
+ it('keeps a socket that answers', async () => {
+ vi.useFakeTimers();
+ plugin.live = true;
+ const { client, connected } = livenessClient();
+ const subscription = follow(client);
+ await vi.waitFor(() =>
+ expect(subscription.values).toEqual([{ data: { ping: 1 } }])
+ );
+
+ returnToForeground();
+ await vi.advanceTimersByTimeAsync(GRAPHQL_SOUP_WEBSOCKET_PONG_TIMEOUT_MS);
+
+ expect(plugin.connects).toHaveLength(1);
+ expect(connected).toEqual([false]);
+ subscription.unsubscribe();
+ await client.dispose();
+ });
+});