Skip to content

The open alternative to Salesforce, designed for AI.

fix(slack): bound Slack Web API retries to the logic function budget (#25483)

Abdul Rahman88fe4360fea3
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.ts
new file mode 100644
index 00000000000..bfbada94ed2
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.ts
@@ -0,0 +1,2 @@
+// a thread page asks Slack for up to 1000 replies, which the 3s default cuts short
+export const SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS = 10 * 1000;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.ts
new file mode 100644
index 00000000000..5296b41d3f6
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.ts
@@ -0,0 +1 @@
+export const SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS = 30 * 1000;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.ts
new file mode 100644
index 00000000000..2790d5efa0f
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.ts
@@ -0,0 +1 @@
+export const SLACK_CLIENT_REQUEST_TIMEOUT_MS = 3 * 1000;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.ts
new file mode 100644
index 00000000000..3cf4eafa86a
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.ts
@@ -0,0 +1,9 @@
+import { type RetryOptions } from '@slack/web-api';
+
+// the library default retries for thirty minutes, past every budget here
+export const SLACK_CLIENT_RETRY_CONFIG: RetryOptions = {
+ retries: 1,
+ factor: 1,
+ minTimeout: 250,
+ maxTimeout: 250,
+};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.ts
new file mode 100644
index 00000000000..53e9213ca43
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.ts
@@ -0,0 +1,2 @@
+// covers the Retry-After wait plus what the first attempt already spent
+export const SLACK_RATE_LIMIT_RETRY_BUDGET_MS = 4 * 1000;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.ts
new file mode 100644
index 00000000000..65881f2c016
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.ts
@@ -0,0 +1,3 @@
+// users.list is 20/min, and the roster match has 120s, so waiting one
+// Retry-After out beats failing the whole run
+export const SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS = 60 * 1000;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts
index fb53031054b..76d922bd40a 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts
@@ -1,4 +1,5 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { ErrorCode } from '@slack/web-api';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { slackPostMessageHandler } from 'src/logic-functions/handlers/slack-post-message-handler';
@@ -22,6 +23,10 @@ describe('slackPostMessageHandler', () => {
});
});
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
it('should return a failure result and skip posting when Slack is not connected', async () => {
getSlackClientMock.mockResolvedValue({
success: false,
@@ -155,6 +160,33 @@ describe('slackPostMessageHandler', () => {
expect(result.success).toBe(true);
});
+ it('should wait out a short Retry-After so a rate limit does not cost the member the answer', async () => {
+ vi.useFakeTimers();
+ postMessageMock
+ .mockRejectedValueOnce(
+ Object.assign(new Error('A rate-limit has been reached'), {
+ code: ErrorCode.RateLimitedError,
+ retryAfter: 1,
+ }),
+ )
+ .mockResolvedValue({ ts: '1700000000.000800', channel: CHANNEL_ID });
+
+ const resultPromise = slackPostMessageHandler({
+ slackChannelId: CHANNEL_ID,
+ messageText: 'hello',
+ });
+
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(await resultPromise).toEqual({
+ success: true,
+ message: 'Message posted to Slack (ts=1700000000.000800).',
+ slackTs: '1700000000.000800',
+ channel: CHANNEL_ID,
+ });
+ expect(postMessageMock).toHaveBeenCalledTimes(2);
+ });
+
it('should return a failure result when the Slack API throws', async () => {
postMessageMock.mockRejectedValue(new Error('channel_not_found'));
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts
index 1f1e5bcd1d5..9e2b498aa55 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts
@@ -1,5 +1,5 @@
-import { type WebClient } from '@slack/web-api';
-import { describe, expect, it, vi } from 'vitest';
+import { ErrorCode, type WebClient } from '@slack/web-api';
+import { afterEach, describe, expect, it, vi } from 'vitest';
import { type SlackRosterMember } from 'src/logic-functions/types/slack-roster-member.type';
import { collectSlackRosterMembers } from 'src/logic-functions/utils/collect-slack-roster-members';
@@ -22,7 +22,49 @@ const buildSlackClient = (
};
};
+const buildRateLimitedError = (retryAfter: number): Error =>
+ Object.assign(new Error(`rate limited, retry in ${retryAfter} seconds`), {
+ code: ErrorCode.RateLimitedError,
+ retryAfter,
+ });
+
describe('collectSlackRosterMembers', () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('should wait out a rate limit when the caller passes a budget for it', async () => {
+ vi.useFakeTimers();
+
+ const usersListMock = vi
+ .fn()
+ .mockRejectedValueOnce(buildRateLimitedError(30))
+ .mockResolvedValue({ members: [{ id: 'U1' }], response_metadata: {} });
+
+ const membersPromise = collectSlackRosterMembers({
+ slackClient: { users: { list: usersListMock } } as unknown as WebClient,
+ rateLimitRetryBudgetMs: 60_000,
+ });
+
+ await vi.advanceTimersByTimeAsync(30_000);
+
+ expect((await membersPromise).members.map((member) => member.id)).toEqual([
+ 'U1',
+ ]);
+ expect(usersListMock).toHaveBeenCalledTimes(2);
+ });
+
+ it('should give up on a rate limit when the caller passes no budget', async () => {
+ const usersListMock = vi.fn().mockRejectedValue(buildRateLimitedError(30));
+
+ await expect(
+ collectSlackRosterMembers({
+ slackClient: { users: { list: usersListMock } } as unknown as WebClient,
+ }),
+ ).rejects.toThrow('rate limited, retry in 30 seconds');
+ expect(usersListMock).toHaveBeenCalledTimes(1);
+ });
+
it('should collect only linkable members', async () => {
const { slackClient } = buildSlackClient([
{
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.ts
new file mode 100644
index 00000000000..50d94cbf4a6
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.ts
@@ -0,0 +1,159 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-request-timeout-ms';
+import { SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-timeout-ms';
+import { fetchSlackAssistantContext } from 'src/logic-functions/utils/fetch-slack-assistant-context';
+
+const {
+ getSlackClientMock,
+ resolveSlackBotUserIdOrThrowMock,
+ fetchSlackThreadMessagesMock,
+ fetchSlackUserIdentityMock,
+ isSlackDirectMessageChannelMock,
+} = vi.hoisted(() => ({
+ getSlackClientMock: vi.fn(),
+ resolveSlackBotUserIdOrThrowMock: vi.fn(),
+ fetchSlackThreadMessagesMock: vi.fn(),
+ fetchSlackUserIdentityMock: vi.fn(),
+ isSlackDirectMessageChannelMock: vi.fn(),
+}));
+
+vi.mock('src/logic-functions/utils/get-slack-client', () => ({
+ getSlackClient: getSlackClientMock,
+}));
+
+vi.mock('src/logic-functions/utils/resolve-slack-bot-user-id-or-throw', () => ({
+ resolveSlackBotUserIdOrThrow: resolveSlackBotUserIdOrThrowMock,
+}));
+
+vi.mock('src/logic-functions/utils/fetch-slack-thread-messages', () => ({
+ fetchSlackThreadMessages: fetchSlackThreadMessagesMock,
+}));
+
+vi.mock('src/logic-functions/utils/fetch-slack-user-identity', () => ({
+ fetchSlackUserIdentity: fetchSlackUserIdentityMock,
+}));
+
+vi.mock('src/logic-functions/utils/is-slack-direct-message-channel', () => ({
+ isSlackDirectMessageChannel: isSlackDirectMessageChannelMock,
+}));
+
+const SLACK_CLIENT = { chat: {} };
+
+const CONTEXT_ARGS = {
+ slackChannelId: 'C123',
+ parentMessageTimestamp: '1700000000.000100',
+ slackMessageTimestamp: '1700000000.000200',
+ slackUserId: 'U123',
+};
+
+describe('fetchSlackAssistantContext', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+ getSlackClientMock.mockResolvedValue({
+ success: true,
+ client: SLACK_CLIENT,
+ });
+ resolveSlackBotUserIdOrThrowMock.mockResolvedValue('UBOT');
+ fetchSlackThreadMessagesMock.mockResolvedValue({
+ tailMessages: [{ ts: '1700000000.000100', text: 'hello', user: 'U123' }],
+ requestMessage: { ts: '1700000000.000200', text: 'hi', user: 'U123' },
+ });
+ fetchSlackUserIdentityMock.mockResolvedValue({
+ slackUserId: 'U123',
+ displayName: 'Ada',
+ });
+ isSlackDirectMessageChannelMock.mockResolvedValue(true);
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ });
+
+ it('should return the thread context when Slack answers in time', async () => {
+ const context = await fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ expect(context.requesterName).toBe('Ada');
+ expect(context.assistantBotUserId).toBe('UBOT');
+ expect(context.isDirectMessage).toBe(true);
+ expect(context.slackClient).toBe(SLACK_CLIENT);
+ expect(context.threadMessages).toHaveLength(1);
+ });
+
+ it('should give thread reads a request timeout that fits a full page of replies', async () => {
+ await fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ expect(getSlackClientMock).toHaveBeenCalledWith({
+ timeout: SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS,
+ });
+ });
+
+ it('should give up on a slow Slack read instead of spending the agent budget', async () => {
+ vi.useFakeTimers();
+ fetchSlackThreadMessagesMock.mockReturnValue(new Promise(() => undefined));
+
+ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);
+
+ const context = await contextPromise;
+
+ expect(context.threadMessages).toEqual([]);
+ expect(context.conversationMessages).toEqual([]);
+ expect(context.requesterName).toBeUndefined();
+ expect(context.isDirectMessage).toBe(false);
+ expect(context.slackClient).toBe(SLACK_CLIENT);
+ });
+
+ it('should give up when acquiring the Slack client hangs', async () => {
+ vi.useFakeTimers();
+ getSlackClientMock.mockReturnValue(new Promise(() => undefined));
+
+ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);
+
+ const context = await contextPromise;
+
+ expect(context.slackClient).toBeUndefined();
+ expect(context.threadMessages).toEqual([]);
+ });
+
+ it('should share one deadline between acquiring the client and reading the thread', async () => {
+ vi.useFakeTimers();
+ getSlackClientMock.mockImplementation(
+ () =>
+ new Promise((resolve) =>
+ setTimeout(
+ () => resolve({ success: true, client: SLACK_CLIENT }),
+ SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS * 0.6,
+ ),
+ ),
+ );
+ fetchSlackThreadMessagesMock.mockReturnValue(new Promise(() => undefined));
+
+ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);
+
+ const context = await contextPromise;
+
+ expect(context.slackClient).toBe(SLACK_CLIENT);
+ expect(context.threadMessages).toEqual([]);
+ });
+
+ it('should return an unreachable context when Slack is not connected', async () => {
+ getSlackClientMock.mockResolvedValue({
+ success: false,
+ error: 'Slack is not connected',
+ });
+
+ const context = await fetchSlackAssistantContext(CONTEXT_ARGS);
+
+ expect(context.slackClient).toBeUndefined();
+ expect(context.threadMessages).toEqual([]);
+ expect(fetchSlackThreadMessagesMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.ts
new file mode 100644
index 00000000000..1513e13651e
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.ts
@@ -0,0 +1,58 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { getSlackClient } from 'src/logic-functions/utils/get-slack-client';
+
+const { webClientMock, getSlackConnectionMock } = vi.hoisted(() => ({
+ webClientMock: vi.fn(),
+ getSlackConnectionMock: vi.fn(),
+}));
+
+vi.mock('@slack/web-api', () => ({
+ WebClient: webClientMock,
+}));
+
+vi.mock('src/logic-functions/utils/get-slack-connection', () => ({
+ getSlackConnection: getSlackConnectionMock,
+}));
+
+describe('getSlackClient', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getSlackConnectionMock.mockResolvedValue({
+ success: true,
+ accessToken: 'xoxb-token',
+ });
+ });
+
+ it('should bound retries and request time so a call cannot outlive its function', async () => {
+ await getSlackClient();
+
+ expect(webClientMock).toHaveBeenCalledWith('xoxb-token', {
+ timeout: 3000,
+ retryConfig: { retries: 1, factor: 1, minTimeout: 250, maxTimeout: 250 },
+ rejectRateLimitedCalls: true,
+ });
+ });
+
+ it('should let a caller with more budget override the defaults', async () => {
+ await getSlackClient({ retryConfig: { retries: 0 }, timeout: 20_000 });
+
+ expect(webClientMock).toHaveBeenCalledWith('xoxb-token', {
+ timeout: 20_000,
+ retryConfig: { retries: 0 },
+ rejectRateLimitedCalls: true,
+ });
+ });
+
+ it('should not build a client when Slack is not connected', async () => {
+ getSlackConnectionMock.mockResolvedValue({
+ success: false,
+ error: 'Slack is not connected',
+ });
+
+ const result = await getSlackClient();
+
+ expect(result).toEqual({ success: false, error: 'Slack is not connected' });
+ expect(webClientMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.ts
new file mode 100644
index 00000000000..668071b6466
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.ts
@@ -0,0 +1,111 @@
+import { ErrorCode } from '@slack/web-api';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';
+
+const buildRateLimitedError = (retryAfter: number): Error => {
+ const error = new Error(`rate limited, retry in ${retryAfter} seconds`);
+
+ Object.assign(error, { code: ErrorCode.RateLimitedError, retryAfter });
+
+ return error;
+};
+
+describe('retrySlackCallWhenRateLimited', () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('should return the result without retrying when the call succeeds', async () => {
+ const call = vi.fn().mockResolvedValue({ ts: '1700000000.000100' });
+
+ const result = await retrySlackCallWhenRateLimited({ call });
+
+ expect(result).toEqual({ ts: '1700000000.000100' });
+ expect(call).toHaveBeenCalledTimes(1);
+ });
+
+ it('should wait out a short Retry-After and retry once', async () => {
+ vi.useFakeTimers();
+
+ const call = vi
+ .fn()
+ .mockRejectedValueOnce(buildRateLimitedError(2))
+ .mockResolvedValue({ ts: '1700000000.000100' });
+
+ const resultPromise = retrySlackCallWhenRateLimited({ call });
+
+ await vi.advanceTimersByTimeAsync(2000);
+
+ expect(await resultPromise).toEqual({ ts: '1700000000.000100' });
+ expect(call).toHaveBeenCalledTimes(2);
+ });
+
+ it('should rethrow when Retry-After is longer than the retry budget', async () => {
+ const call = vi.fn().mockRejectedValue(buildRateLimitedError(60));
+
+ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(
+ 'rate limited, retry in 60 seconds',
+ );
+ expect(call).toHaveBeenCalledTimes(1);
+ });
+
+ it('should rethrow the second rate limit instead of retrying forever', async () => {
+ vi.useFakeTimers();
+
+ const call = vi.fn().mockRejectedValue(buildRateLimitedError(1));
+
+ const resultPromise = retrySlackCallWhenRateLimited({ call });
+ const assertion = expect(resultPromise).rejects.toThrow(
+ 'rate limited, retry in 1 seconds',
+ );
+
+ await vi.advanceTimersByTimeAsync(1000);
+ await assertion;
+
+ expect(call).toHaveBeenCalledTimes(2);
+ });
+
+ it('should not retry when the first attempt already spent the budget', async () => {
+ vi.useFakeTimers();
+
+ const call = vi.fn().mockImplementation(async () => {
+ vi.advanceTimersByTime(3500);
+
+ throw buildRateLimitedError(1);
+ });
+
+ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(
+ 'rate limited, retry in 1 seconds',
+ );
+ expect(call).toHaveBeenCalledTimes(1);
+ });
+
+ it('should wait out a long Retry-After when the caller has the budget for it', async () => {
+ vi.useFakeTimers();
+
+ const call = vi
+ .fn()
+ .mockRejectedValueOnce(buildRateLimitedError(60))
+ .mockResolvedValue({ members: [] });
+
+ const resultPromise = retrySlackCallWhenRateLimited({
+ call,
+ budgetMs: 90_000,
+ });
+
+ await vi.advanceTimersByTimeAsync(60_000);
+
+ expect(await resultPromise).toEqual({ members: [] });
+ expect(call).toHaveBeenCalledTimes(2);
+ });
+
+ it('should not retry an error that is not a rate limit', async () => {
+ const call = vi.fn().mockRejectedValue(new Error('channel_not_found'));
+
+ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(
+ 'channel_not_found',
+ );
+ expect(call).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.ts
new file mode 100644
index 00000000000..d025cae78f6
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.ts
@@ -0,0 +1,57 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';
+
+describe('runWithTimeout', () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('should return the operation value when it settles in time', async () => {
+ const result = await runWithTimeout({
+ operation: Promise.resolve('context'),
+ timeoutMs: 1000,
+ buildTimeoutValue: () => 'fallback',
+ });
+
+ expect(result).toBe('context');
+ });
+
+ it('should return the timeout value when the operation is too slow', async () => {
+ vi.useFakeTimers();
+
+ const pendingOperation = new Promise<string>(() => undefined);
+
+ const resultPromise = runWithTimeout({
+ operation: pendingOperation,
+ timeoutMs: 1000,
+ buildTimeoutValue: () => 'fallback',
+ });
+
+ await vi.advanceTimersByTimeAsync(1000);
+
+ expect(await resultPromise).toBe('fallback');
+ });
+
+ it('should not build the timeout value when the operation wins, so callers do not log a timeout that never happened', async () => {
+ const buildTimeoutValue = vi.fn(() => 'fallback');
+
+ await runWithTimeout({
+ operation: Promise.resolve('context'),
+ timeoutMs: 1000,
+ buildTimeoutValue,
+ });
+
+ expect(buildTimeoutValue).not.toHaveBeenCalled();
+ });
+
+ it('should propagate a rejected operation', async () => {
+ await expect(
+ runWithTimeout({
+ operation: Promise.reject(new Error('read failed')),
+ timeoutMs: 1000,
+ buildTimeoutValue: () => 'fallback',
+ }),
+ ).rejects.toThrow('read failed');
+ });
+});
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts
index b409863b4a2..66a9ec4da70 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts
@@ -2,8 +2,10 @@ import { type WebClient } from '@slack/web-api';
import { isNonEmptyString } from '@sniptt/guards';
import { isDefined } from 'twenty-sdk/utils';
+import { SLACK_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-rate-limit-retry-budget-ms';
import { type LinkableSlackRosterMember } from 'src/logic-functions/types/slack-roster-member.type';
import { isLinkableSlackRosterMember } from 'src/logic-functions/utils/is-linkable-slack-roster-member';
+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';
const MEMBERS_PER_PAGE = 200;
const DEFAULT_MAX_PAGES = 5;
@@ -18,20 +20,28 @@ export const collectSlackRosterMembers = async ({
shouldCollectMember,
maxMembers,
maxPages = DEFAULT_MAX_PAGES,
+ rateLimitRetryBudgetMs = SLACK_RATE_LIMIT_RETRY_BUDGET_MS,
}: {
slackClient: WebClient;
shouldCollectMember?: (member: LinkableSlackRosterMember) => boolean;
maxMembers?: number;
maxPages?: number;
+ rateLimitRetryBudgetMs?: number;
}): Promise<CollectSlackRosterMembersResult> => {
const members: LinkableSlackRosterMember[] = [];
const seenSlackUserIds = new Set<string>();
+ // one window for every page, so five rate limited pages cannot wait five times
+ const rateLimitDeadlineAtMs = Date.now() + rateLimitRetryBudgetMs;
let cursor: string | undefined;
for (let page = 0; page < maxPages; page += 1) {
- const response = await slackClient.users.list({
- limit: MEMBERS_PER_PAGE,
- cursor,
+ const response = await retrySlackCallWhenRateLimited({
+ call: async () =>
+ slackClient.users.list({
+ limit: MEMBERS_PER_PAGE,
+ cursor,
+ }),
+ budgetMs: Math.max(rateLimitDeadlineAtMs - Date.now(), 0),
});
const collectiblePageMembers = (response.members ?? [])
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts
index 2c3bec5a0b0..1809e51e8d2 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts
@@ -1,5 +1,7 @@
import { type WebClient } from '@slack/web-api';
+import { SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-request-timeout-ms';
+import { SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-timeout-ms';
import { type SlackAssistantAgentMessage } from 'src/logic-functions/types/slack-assistant-agent-message.type';
import { type SlackThreadMessage } from 'src/logic-functions/types/slack-thread-message.type';
import { type SlackUserIdentity } from 'src/logic-functions/types/slack-user-identity.type';
@@ -9,6 +11,7 @@ import { fetchSlackUserIdentity } from 'src/logic-functions/utils/fetch-slack-us
import { getSlackClient } from 'src/logic-functions/utils/get-slack-client';
import { isSlackDirectMessageChannel } from 'src/logic-functions/utils/is-slack-direct-message-channel';
import { resolveSlackBotUserIdOrThrow } from 'src/logic-functions/utils/resolve-slack-bot-user-id-or-throw';
+import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';
type SlackAssistantContext = {
conversationMessages: SlackAssistantAgentMessage[];
@@ -32,25 +35,19 @@ const UNREACHABLE_SLACK_CONTEXT: SlackAssistantContext = {
isDirectMessage: false,
};
-export const fetchSlackAssistantContext = async ({
+const readSlackThreadContext = async ({
+ client,
slackChannelId,
parentMessageTimestamp,
slackMessageTimestamp,
slackUserId,
}: {
+ client: WebClient;
slackChannelId: string;
parentMessageTimestamp: string;
slackMessageTimestamp: string;
slackUserId: string | undefined;
}): Promise<SlackAssistantContext> => {
- const slackClientResult = await getSlackClient();
-
- if (!slackClientResult.success) {
- return UNREACHABLE_SLACK_CONTEXT;
- }
-
- const { client } = slackClientResult;
-
const assistantBotUserId = await resolveSlackBotUserIdOrThrow().catch(
(error) => {
console.warn(
@@ -88,3 +85,59 @@ export const fetchSlackAssistantContext = async ({
isDirectMessage,
};
};
+
+export const fetchSlackAssistantContext = async ({
+ slackChannelId,
+ parentMessageTimestamp,
+ slackMessageTimestamp,
+ slackUserId,
+}: {
+ slackChannelId: string;
+ parentMessageTimestamp: string;
+ slackMessageTimestamp: string;
+ slackUserId: string | undefined;
+}): Promise<SlackAssistantContext> => {
+ // the agent budget is already ticking: partial context beats a late answer
+ const contextDeadlineAtMs = Date.now() + SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS;
+
+ const slackClientResult = await runWithTimeout({
+ operation: getSlackClient({
+ timeout: SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS,
+ }),
+ timeoutMs: SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS,
+ buildTimeoutValue: () => {
+ console.warn(
+ `[slack] acquiring the Slack client exceeded ${SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS}ms, answering without thread history`,
+ );
+
+ return {
+ success: false as const,
+ error: 'Timed out acquiring the Slack client',
+ };
+ },
+ });
+
+ if (!slackClientResult.success) {
+ return UNREACHABLE_SLACK_CONTEXT;
+ }
+
+ const { client } = slackClientResult;
+
+ return await runWithTimeout({
+ operation: readSlackThreadContext({
+ client,
+ slackChannelId,
+ parentMessageTimestamp,
+ slackMessageTimestamp,
+ slackUserId,
+ }),
+ timeoutMs: Math.max(contextDeadlineAtMs - Date.now(), 0),
+ buildTimeoutValue: () => {
+ console.warn(
+ `[slack] assistant context read exceeded ${SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS}ms, answering without thread history`,
+ );
+
+ return { ...UNREACHABLE_SLACK_CONTEXT, slackClient: client };
+ },
+ });
+};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts
index bc34ff713d3..d8d5cdbfe42 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts
@@ -1,5 +1,7 @@
import { WebClient, type WebClientOptions } from '@slack/web-api';
+import { SLACK_CLIENT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-client-request-timeout-ms';
+import { SLACK_CLIENT_RETRY_CONFIG } from 'src/logic-functions/constants/slack-client-retry-config';
import { getSlackConnection } from 'src/logic-functions/utils/get-slack-connection';
export const getSlackClient = async (
@@ -15,6 +17,12 @@ export const getSlackClient = async (
return {
success: true,
- client: new WebClient(connectionResult.accessToken, options),
+ client: new WebClient(connectionResult.accessToken, {
+ timeout: SLACK_CLIENT_REQUEST_TIMEOUT_MS,
+ retryConfig: SLACK_CLIENT_RETRY_CONFIG,
+ // otherwise a 429 sleeps the whole Retry-After before any retry budget
+ rejectRateLimitedCalls: true,
+ ...options,
+ }),
};
};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.ts
new file mode 100644
index 00000000000..6ecd86488d6
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.ts
@@ -0,0 +1,9 @@
+import { ErrorCode, type WebAPIRateLimitedError } from '@slack/web-api';
+import { isObject } from '@sniptt/guards';
+
+export const isSlackRateLimitedError = (
+ error: unknown,
+): error is WebAPIRateLimitedError =>
+ isObject(error) &&
+ 'code' in error &&
+ error.code === ErrorCode.RateLimitedError;
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts
index 6423f3d09a4..c1a2f530c07 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts
@@ -2,6 +2,7 @@ import { type WebClient } from '@slack/web-api';
import { isNonEmptyString } from '@sniptt/guards';
import { CoreApiClient } from 'twenty-client-sdk/core';
+import { SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms';
import { listLinkedSlackUserIds } from 'src/logic-functions/data/list-linked-slack-user-ids';
import { findWorkspaceMemberIdsByEmails } from 'src/logic-functions/data/find-workspace-member-ids-by-emails';
import { type SlackRosterMatchSummary } from 'src/logic-functions/types/slack-roster-match.type';
@@ -23,7 +24,10 @@ const runSlackRosterMatch = async ({
const [linkedSlackUserIds, roster] = await Promise.all([
listLinkedSlackUserIds(client, { slackTeamId }),
- collectSlackRosterMembers({ slackClient }),
+ collectSlackRosterMembers({
+ slackClient,
+ rateLimitRetryBudgetMs: SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS,
+ }),
]);
const vouchedRosterEmails = roster.members
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts
index f70d248bc59..3c9ea18b809 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts
@@ -7,6 +7,7 @@ import { buildSlackRecordEntitiesForMessage } from 'src/logic-functions/utils/bu
import { getSlackApiErrorCode } from 'src/logic-functions/utils/get-slack-api-error-code';
import { isSlackMetadataError } from 'src/logic-functions/utils/is-slack-metadata-error';
import { normalizeSlackParentMessageTimestamp } from 'src/logic-functions/utils/normalize-slack-parent-message-timestamp';
+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';
import { sendSlackMessageWithBodyFallbacks } from 'src/logic-functions/utils/send-slack-message-with-body-fallbacks';
export const postSlackMessage = async (
@@ -33,17 +34,20 @@ export const postSlackMessage = async (
failureMessage: 'Failed to post Slack message',
sendMessage: async (bodyFields) => {
try {
- const data = await client.chat.postMessage({
- channel: parameters.slackChannelId,
- thread_ts: parentTimestamp,
- ...(isDefined(parameters.unfurlLinks)
- ? { unfurl_links: parameters.unfurlLinks }
- : {}),
- ...(isDefined(parameters.unfurlMedia)
- ? { unfurl_media: parameters.unfurlMedia }
- : {}),
- ...(entities.length > 0 ? { metadata: { entities } } : {}),
- ...bodyFields,
+ const data = await retrySlackCallWhenRateLimited({
+ call: async () =>
+ client.chat.postMessage({
+ channel: parameters.slackChannelId,
+ thread_ts: parentTimestamp,
+ ...(isDefined(parameters.unfurlLinks)
+ ? { unfurl_links: parameters.unfurlLinks }
+ : {}),
+ ...(isDefined(parameters.unfurlMedia)
+ ? { unfurl_media: parameters.unfurlMedia }
+ : {}),
+ ...(entities.length > 0 ? { metadata: { entities } } : {}),
+ ...bodyFields,
+ }),
});
return {
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts
index 0022a42557c..16d7327415e 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts
@@ -1,6 +1,7 @@
import { type RunAgentResult } from 'twenty-sdk/logic-function';
import { SLACK_ASSISTANT_DEADLINE_ERROR } from 'src/logic-functions/constants/slack-assistant-deadline-error';
+import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';
export const raceSlackAssistantAgentDeadline = async ({
agentRun,
@@ -8,24 +9,13 @@ export const raceSlackAssistantAgentDeadline = async ({
}: {
agentRun: Promise<RunAgentResult>;
deadlineAtMs: number;
-}): Promise<RunAgentResult> => {
- let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
-
- const deadlineResult = new Promise<RunAgentResult>((resolve) => {
- deadlineTimer = setTimeout(
- () =>
- resolve({
- result: null,
- error: SLACK_ASSISTANT_DEADLINE_ERROR,
- success: false,
- }),
- Math.max(deadlineAtMs - Date.now(), 0),
- );
+}): Promise<RunAgentResult> =>
+ await runWithTimeout({
+ operation: agentRun,
+ timeoutMs: Math.max(deadlineAtMs - Date.now(), 0),
+ buildTimeoutValue: () => ({
+ result: null,
+ error: SLACK_ASSISTANT_DEADLINE_ERROR,
+ success: false,
+ }),
});
-
- try {
- return await Promise.race([agentRun, deadlineResult]);
- } finally {
- clearTimeout(deadlineTimer);
- }
-};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.ts
new file mode 100644
index 00000000000..f1c27de5e17
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.ts
@@ -0,0 +1,31 @@
+import { SLACK_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-rate-limit-retry-budget-ms';
+import { isSlackRateLimitedError } from 'src/logic-functions/utils/is-slack-rate-limited-error';
+
+// rejecting a rate limited write outright would cost the member the answer
+export const retrySlackCallWhenRateLimited = async <TResult>({
+ call,
+ budgetMs = SLACK_RATE_LIMIT_RETRY_BUDGET_MS,
+}: {
+ call: () => Promise<TResult>;
+ budgetMs?: number;
+}): Promise<TResult> => {
+ const startedAtMs = Date.now();
+
+ try {
+ return await call();
+ } catch (error) {
+ if (!isSlackRateLimitedError(error)) {
+ throw error;
+ }
+
+ const waitMs = error.retryAfter * 1000;
+
+ if (Date.now() - startedAtMs + waitMs > budgetMs) {
+ throw error;
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, waitMs));
+
+ return await call();
+ }
+};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.ts
new file mode 100644
index 00000000000..9f0685e2c1b
--- /dev/null
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.ts
@@ -0,0 +1,21 @@
+export const runWithTimeout = async <TValue>({
+ operation,
+ timeoutMs,
+ buildTimeoutValue,
+}: {
+ operation: Promise<TValue>;
+ timeoutMs: number;
+ buildTimeoutValue: () => TValue;
+}): Promise<TValue> => {
+ let timeoutTimer: ReturnType<typeof setTimeout> | undefined;
+
+ const timeoutValue = new Promise<TValue>((resolve) => {
+ timeoutTimer = setTimeout(() => resolve(buildTimeoutValue()), timeoutMs);
+ });
+
+ try {
+ return await Promise.race([operation, timeoutValue]);
+ } finally {
+ clearTimeout(timeoutTimer);
+ }
+};
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts
index 7b1da269268..c6944a80d59 100644
--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts
+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts
@@ -2,6 +2,7 @@ import { type WebClient } from '@slack/web-api';
import { isNonEmptyString } from '@sniptt/guards';
import { buildSlackUserLinkConsentBlocks } from 'src/logic-functions/utils/build-slack-user-link-consent-blocks';
+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';
import { toErrorMessage } from 'src/logic-functions/utils/to-error-message.util';
export const sendSlackUserLinkConsentDm = async (
@@ -34,16 +35,19 @@ export const sendSlackUserLinkConsentDm = async (
};
}
- await slackClient.chat.postMessage({
- channel: channelId,
- text: 'A Twenty admin asked to link your Slack account. Approve or decline it here.',
- blocks: buildSlackUserLinkConsentBlocks({
- memberName,
- slackTeamId,
- slackUserId,
- workspaceMemberId,
- slackUserLinkId,
- }),
+ await retrySlackCallWhenRateLimited({
+ call: async () =>
+ slackClient.chat.postMessage({
+ channel: channelId,
+ text: 'A Twenty admin asked to link your Slack account. Approve or decline it here.',
+ blocks: buildSlackUserLinkConsentBlocks({
+ memberName,
+ slackTeamId,
+ slackUserId,
+ workspaceMemberId,
+ slackUserLinkId,
+ }),
+ }),
});
return { success: true };