Skip to content

The open-source app everyone uses to manage agents at work

refactor(server): isolate provider webhook lifecycle processing (#16232)

Dotta875943ac83e7
diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts
index 11beb2174e..37c2ce57b8 100644
--- a/server/src/services/chat-channels.ts
+++ b/server/src/services/chat-channels.ts
@@ -1,3 +1,4 @@
+import { initializeGitHubWebhookRuntime } from "./chat-channels/github/webhook-runtime.js";
import { githubInboundTaskTitle, githubInboundCommentText, githubAssignedAgentRequiresLowTrust, retainGitHubDeliveryAuthority, retainGitHubReviewReceipt } from "./chat-channels/github/task-preparation.js";
import { applyChatTaskReviewContainment } from "./chat-channels/inbound/task-trust.js";
import { photonControlTargetsConversation, photonReplyMetadata, recordPhotonInboundActivity } from "./chat-channels/imessage-photon/messages.js";
@@ -96,7 +97,7 @@ import { createChatPublicationReplacement } from "./chat-channels/publication-re
import { createChatIdentityLinks } from "./chat-channels/identity-links.js";
import { createChatResourceReconciliation } from "./chat-channels/resource-reconciliation.js";
-import { slackResourceLabelIsFallback } from "./chat-channels/slack/resources.js";
+import { scheduleSlackLifecycleResourceHydration } from "./chat-channels/slack/resources.js";
import {
MAX_INBOUND_TEXT,
@@ -108,8 +109,7 @@ import {
microsoftTeamsLifecycleEventFromPayload, teamsOrphanFollowUp, teamsAttachmentReferenceText,
} from "./chat-channels/microsoft-teams/messages.js";
import {
- type GitHubLifecycleEvent,
- githubLifecycleEventFromRequest,
+ githubSupplementalLifecycleFromRequest,
} from "./chat-channels/github/messages.js";
import { nextChatActionAt } from "./chat-action-work.js";
import type { ScheduledChatAction } from "./chat-work-notifications.js";
@@ -217,9 +217,6 @@ import {
readRegisteredDiscordCommandRegistration,
} from "./chat-discord-command-registration-store.js";
-import {
- parseSlackSessionStop,
-} from "./chat-slack-sessions.js";
import {
parseChatProviderLifecycle,
@@ -889,7 +886,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
});
const { handleTelegramSlashCommand } = createTelegramCommands({ runtime, runtimeFor, handleSdkMessage });
const { stageGitHubWebhookIngress, githubIngressAcceptedResponse, processGitHubWebhookIngress, processFailedGitHubWebhookDeliveries, processPendingGitHubWebhookIngress } = createGitHubWebhookIngress(db, { endpointRecord, runtimeContextForRecord, resolveCredentialRefs, runtimeGeneration, credentialFingerprint, getWebhookPublicBaseUrl, redactError, handleWebhook, runtimeCallbackEndpoint, fetchImpl, persistence, withCredentialMutationLease, isShuttingDown: () => shuttingDown, get githubWebhookReplayBarrier() { return options.githubWebhookReplayBarrier; }, supportedWebhookEvents: SUPPORTED_GITHUB_WEBHOOK_EVENTS });
- const { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops } = createSlackSessionStop(db, { runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, nonDirectDestinationAllowed, stageAuthorizedTaskControlPublication, getPublicBaseUrl, get heartbeat() { return options.heartbeat; }, redactError });
+ const { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops, handleSlackSessionStopPayload } = createSlackSessionStop(db, { scheduleMessageProcessing, get deferWebhookProcessing() { return options.deferWebhookProcessing; }, runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, nonDirectDestinationAllowed, stageAuthorizedTaskControlPublication, getPublicBaseUrl, get heartbeat() { return options.heartbeat; }, redactError });
const { stageTelegramMaintenance, processTelegramMaintenance, processPendingTelegramMaintenance, processPendingTelegramActions, processTelegramSubscriptionDiscovery, scheduleTelegramSubscriptionDiscovery, hasPendingTelegramSubscriptionDiscovery, hasTelegramStopSubscription, recordTelegramStopSubscription } = createTelegramMaintenance(db, { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, resolveCredentials, clearCredentials, runtimeCallbackEndpoint, getWebhookPublicBaseUrl, fetchImpl, runtimeGeneration, credentialFingerprint, redactError, credentialCheckTimeoutMs: PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS });
const { reconcileGitHubWebhookRepository, reconcileProviderResourceRows, listResources, listGitHubRepositories, replaceResources, toggleAllGitHubRepositories, hasNewerProcessedProviderLifecycleEffect, applyProviderAvailability } = createChatResourceReconciliation(db, { endpointRecord, withCredentialMutationLease, runtimeGeneration, maxErrorText: MAX_ERROR_TEXT });
const { applyGitHubInstallationLifecycle } = createGitHubInstallationLifecycle(db, {
@@ -2979,46 +2976,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
effect,
runtimeContext,
);
- if (
- applied &&
- endpoint.provider === "slack" &&
- effect.kind === "resource" &&
- effect.availability === "available" &&
- slackResourceLabelIsFallback(effect.providerResourceId, effect.label)
- ) {
- const stillAvailable = await db
- .select({ id: chatEndpointResources.id })
- .from(chatEndpointResources)
- .where(
- and(
- eq(chatEndpointResources.endpointId, endpoint.id),
- eq(chatEndpointResources.type, "channel"),
- eq(
- chatEndpointResources.providerResourceId,
- effect.providerResourceId,
- ),
- eq(chatEndpointResources.availability, "available"),
- ),
- )
- .limit(1)
- .then((rows) => rows.length > 0);
- if (stillAvailable) {
- scheduleMessageProcessing(async () => {
- await hydrateSlackResourceLabel(
- endpoint.id,
- effect.providerResourceId,
- ).catch((error) => {
- logger.warn(
- {
- endpointId: endpoint.id,
- providerResourceId: effect.providerResourceId,
- error: redactError(error),
- },
- "could not hydrate a newly available Slack channel label",
- );
- });
- });
- }
+ if (applied && endpoint.provider === "slack") {
+ await scheduleSlackLifecycleResourceHydration(db, endpoint, effect, { scheduleMessageProcessing, hydrateSlackResourceLabel, redactError });
}
}
}
@@ -3085,36 +3044,9 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
const githubInspection = provider === "github" ? request.clone() : null;
recordChatWebhookStage("runtime_requested");
const runtimePromise = runtimeFor(endpoint);
- let endpointRuntime: ChatEndpointRuntime;
- if (githubResponseDeadlineAt !== null) {
- let timeout: ReturnType<typeof setTimeout> | null = null;
- try {
- const outcome = await Promise.race([
- runtimePromise.then((value) => ({ completed: true as const, value })),
- new Promise<{ completed: false; value?: never }>((resolve) => {
- timeout = setTimeout(
- () => resolve({ completed: false }),
- Math.max(0, githubResponseDeadlineAt - Date.now()),
- );
- timeout.unref?.();
- }),
- ]);
- if (!outcome.completed) {
- return retryableGitHubWebhookResponse();
- }
- endpointRuntime = outcome.value;
- } catch (error) {
- logger.warn(
- { endpointId: endpoint.id, error: redactError(error) },
- "GitHub runtime initialization failed before durable webhook receipt",
- );
- return retryableGitHubWebhookResponse();
- } finally {
- if (timeout) clearTimeout(timeout);
- }
- } else {
- endpointRuntime = await runtimePromise;
- }
+ const initialized = await initializeGitHubWebhookRuntime(endpoint, runtimePromise, githubResponseDeadlineAt, redactError);
+ if (initialized.response) return initialized.response;
+ const endpointRuntime = initialized.runtime;
recordChatWebhookStage("runtime_ready");
const runtimeContext = runtimeContexts.get(endpointRuntime as object);
if (!runtimeContext) {
@@ -3155,26 +3087,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
endpointRuntime.acceptsProviderScope(lifecyclePayload)
) {
if (endpoint.provider === "slack") {
- const stop = parseSlackSessionStop(
- lifecyclePayload,
- endpoint.providerAccountId,
- );
- if (stop) {
- const actionId = await stageSlackSessionStop(
- endpoint,
- stop,
- runtimeContext,
- );
- if (actionId) {
- if (options.deferWebhookProcessing === true) {
- scheduleMessageProcessing(() =>
- processSlackSessionStop(actionId),
- );
- } else {
- await processSlackSessionStop(actionId);
- }
- }
- }
+ await handleSlackSessionStopPayload(endpoint, lifecyclePayload, runtimeContext);
}
const effects = parseChatProviderLifecycle({
provider: endpoint.provider,
@@ -3191,13 +3104,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
await recordSlackWebhookCallback(endpoint, request, slackCallbackInspection, runtimeContext);
}
if (githubInspection && response.ok) {
- let lifecycle: GitHubLifecycleEvent | null = null;
- try {
- lifecycle = await githubLifecycleEventFromRequest(githubInspection);
- } catch {
- // Ignore only malformed or unsupported supplemental payloads. Once an
- // event is recognized, its durable persistence failure must escape.
- }
+ const lifecycle = await githubSupplementalLifecycleFromRequest(githubInspection);
if (lifecycle) {
await recordLifecycleDelivery(
{
diff --git a/server/src/services/chat-channels/github/messages.ts b/server/src/services/chat-channels/github/messages.ts
index 800d806d26..0a1c0e1933 100644
--- a/server/src/services/chat-channels/github/messages.ts
+++ b/server/src/services/chat-channels/github/messages.ts
@@ -130,3 +130,14 @@ export async function githubLifecycleEventFromRequest(
threadId,
};
}
+
+export async function githubSupplementalLifecycleFromRequest(githubInspection: Request): Promise<GitHubLifecycleEvent | null> {
+ let lifecycle: GitHubLifecycleEvent | null = null;
+ try {
+ lifecycle = await githubLifecycleEventFromRequest(githubInspection);
+ } catch {
+ // Ignore only malformed or unsupported supplemental payloads. Once an
+ // event is recognized, its durable persistence failure must escape.
+ }
+ return lifecycle;
+}
diff --git a/server/src/services/chat-channels/github/webhook-runtime.ts b/server/src/services/chat-channels/github/webhook-runtime.ts
new file mode 100644
index 0000000000..7ac1b98270
--- /dev/null
+++ b/server/src/services/chat-channels/github/webhook-runtime.ts
@@ -0,0 +1,43 @@
+import type { ChatEndpointRuntime } from "../../voice/voice-runtime.js";
+import type { EndpointRow } from "../runtime-types.js";
+import { logger } from "../../../middleware/logger.js";
+import { retryableGitHubWebhookResponse } from "./webhook-authentication.js";
+
+export async function initializeGitHubWebhookRuntime(
+ endpoint: EndpointRow,
+ runtimePromise: Promise<ChatEndpointRuntime>,
+ githubResponseDeadlineAt: number | null,
+ redactError: (error: unknown) => string,
+): Promise<{ response: Response; runtime?: never } | { response?: never; runtime: ChatEndpointRuntime }> {
+ let endpointRuntime: ChatEndpointRuntime;
+ if (githubResponseDeadlineAt !== null) {
+ let timeout: ReturnType<typeof setTimeout> | null = null;
+ try {
+ const outcome = await Promise.race([
+ runtimePromise.then((value) => ({ completed: true as const, value })),
+ new Promise<{ completed: false; value?: never }>((resolve) => {
+ timeout = setTimeout(
+ () => resolve({ completed: false }),
+ Math.max(0, githubResponseDeadlineAt - Date.now()),
+ );
+ timeout.unref?.();
+ }),
+ ]);
+ if (!outcome.completed) {
+ return { response: retryableGitHubWebhookResponse(), runtime: undefined };
+ }
+ endpointRuntime = outcome.value;
+ } catch (error) {
+ logger.warn(
+ { endpointId: endpoint.id, error: redactError(error) },
+ "GitHub runtime initialization failed before durable webhook receipt",
+ );
+ return { response: retryableGitHubWebhookResponse(), runtime: undefined };
+ } finally {
+ if (timeout) clearTimeout(timeout);
+ }
+ } else {
+ endpointRuntime = await runtimePromise;
+ }
+ return { runtime: endpointRuntime };
+}
diff --git a/server/src/services/chat-channels/slack/resources.ts b/server/src/services/chat-channels/slack/resources.ts
index 1d7773f750..c135134824 100644
--- a/server/src/services/chat-channels/slack/resources.ts
+++ b/server/src/services/chat-channels/slack/resources.ts
@@ -1,3 +1,9 @@
+import { chatEndpointResources, type Db } from "@paperclipai/db";
+import { and, eq } from "drizzle-orm";
+import type { EndpointRow } from "../runtime-types.js";
+import type { ChatProviderLifecycleEffect } from "../../chat-provider-lifecycle.js";
+import { logger } from "../../../middleware/logger.js";
+
export function slackResourceLabelIsFallback(
providerResourceId: string,
label: string,
@@ -8,3 +14,49 @@ export function slackResourceLabelIsFallback(
candidate === `slack:${providerResourceId}`
);
}
+
+export async function scheduleSlackLifecycleResourceHydration(db: Db, endpoint: EndpointRow, effect: ChatProviderLifecycleEffect, options: {
+ scheduleMessageProcessing: (task: () => Promise<void>) => void;
+ hydrateSlackResourceLabel: (endpointId: string, resourceId: string) => Promise<unknown>;
+ redactError: (error: unknown) => string;
+}): Promise<void> {
+ if (
+ effect.kind === "resource" &&
+ effect.availability === "available" &&
+ slackResourceLabelIsFallback(effect.providerResourceId, effect.label)
+ ) {
+ const stillAvailable = await db
+ .select({ id: chatEndpointResources.id })
+ .from(chatEndpointResources)
+ .where(
+ and(
+ eq(chatEndpointResources.endpointId, endpoint.id),
+ eq(chatEndpointResources.type, "channel"),
+ eq(
+ chatEndpointResources.providerResourceId,
+ effect.providerResourceId,
+ ),
+ eq(chatEndpointResources.availability, "available"),
+ ),
+ )
+ .limit(1)
+ .then((rows) => rows.length > 0);
+ if (stillAvailable) {
+ options.scheduleMessageProcessing(async () => {
+ await options.hydrateSlackResourceLabel(
+ endpoint.id,
+ effect.providerResourceId,
+ ).catch((error) => {
+ logger.warn(
+ {
+ endpointId: endpoint.id,
+ providerResourceId: effect.providerResourceId,
+ error: options.redactError(error),
+ },
+ "could not hydrate a newly available Slack channel label",
+ );
+ });
+ });
+ }
+ }
+}
diff --git a/server/src/services/chat-channels/slack/session-stop.ts b/server/src/services/chat-channels/slack/session-stop.ts
index bb2e071e7c..aef5c73136 100644
--- a/server/src/services/chat-channels/slack/session-stop.ts
+++ b/server/src/services/chat-channels/slack/session-stop.ts
@@ -45,7 +45,7 @@ import {
import { projectSafeChatPublication } from "../../chat-publication-projection.js";
import {
- type SlackSessionStop,
+ parseSlackSessionStop, type SlackSessionStop,
} from "../../chat-slack-sessions.js";
import {
@@ -121,6 +121,8 @@ export function slackSessionStopPayload(
};
}
type SlackSessionStopOptions = {
+ scheduleMessageProcessing: (task: () => Promise<void>) => void;
+ readonly deferWebhookProcessing?: boolean;
runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise<EndpointRow | null>;
lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>;
nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean;
@@ -854,5 +856,27 @@ export function createSlackSessionStop(db: Db, options: SlackSessionStopOptions)
for (const action of actions) await processSlackSessionStop(action.id);
return actions.length;
}
- return { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops };
+ async function handleSlackSessionStopPayload(endpoint: EndpointRow, lifecyclePayload: unknown, runtimeContext: RuntimeContext): Promise<void> {
+ const stop = parseSlackSessionStop(
+ lifecyclePayload,
+ endpoint.providerAccountId,
+ );
+ if (stop) {
+ const actionId = await stageSlackSessionStop(
+ endpoint,
+ stop,
+ runtimeContext,
+ );
+ if (actionId) {
+ if (options.deferWebhookProcessing === true) {
+ options.scheduleMessageProcessing(() =>
+ processSlackSessionStop(actionId),
+ );
+ } else {
+ await processSlackSessionStop(actionId);
+ }
+ }
+ }
+ }
+ return { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops, handleSlackSessionStopPayload };
}