The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(slack): bound Slack Web API retries to the logic function budget (#25483)
Abdul Rahman88fe4360fea3
diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.tsnew file mode 100644index 00000000000..bfbada94ed2--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-request-timeout-ms.ts@@ -0,0 +1,2 @@+// a thread page asks Slack for up to 1000 replies, which the 3s default cuts short+export const SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS = 10 * 1000;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.tsnew file mode 100644index 00000000000..5296b41d3f6--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-assistant-context-timeout-ms.ts@@ -0,0 +1 @@+export const SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS = 30 * 1000;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.tsnew file mode 100644index 00000000000..2790d5efa0f--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-request-timeout-ms.ts@@ -0,0 +1 @@+export const SLACK_CLIENT_REQUEST_TIMEOUT_MS = 3 * 1000;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.tsnew file mode 100644index 00000000000..3cf4eafa86a--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-client-retry-config.ts@@ -0,0 +1,9 @@+import { type RetryOptions } from '@slack/web-api';++// the library default retries for thirty minutes, past every budget here+export const SLACK_CLIENT_RETRY_CONFIG: RetryOptions = {+ retries: 1,+ factor: 1,+ minTimeout: 250,+ maxTimeout: 250,+};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.tsnew file mode 100644index 00000000000..53e9213ca43--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-rate-limit-retry-budget-ms.ts@@ -0,0 +1,2 @@+// covers the Retry-After wait plus what the first attempt already spent+export const SLACK_RATE_LIMIT_RETRY_BUDGET_MS = 4 * 1000;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.ts b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.tsnew file mode 100644index 00000000000..65881f2c016--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms.ts@@ -0,0 +1,3 @@+// users.list is 20/min, and the roster match has 120s, so waiting one+// Retry-After out beats failing the whole run+export const SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS = 60 * 1000;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.tsindex fb53031054b..76d922bd40a 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/handlers/__tests__/slack-post-message-handler.test.ts@@ -1,4 +1,5 @@-import { beforeEach, describe, expect, it, vi } from 'vitest';+import { ErrorCode } from '@slack/web-api';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import { slackPostMessageHandler } from 'src/logic-functions/handlers/slack-post-message-handler';@@ -22,6 +23,10 @@ describe('slackPostMessageHandler', () => {});});+ afterEach(() => {+ vi.useRealTimers();+ });+it('should return a failure result and skip posting when Slack is not connected', async () => {getSlackClientMock.mockResolvedValue({success: false,@@ -155,6 +160,33 @@ describe('slackPostMessageHandler', () => {expect(result.success).toBe(true);});+ it('should wait out a short Retry-After so a rate limit does not cost the member the answer', async () => {+ vi.useFakeTimers();+ postMessageMock+ .mockRejectedValueOnce(+ Object.assign(new Error('A rate-limit has been reached'), {+ code: ErrorCode.RateLimitedError,+ retryAfter: 1,+ }),+ )+ .mockResolvedValue({ ts: '1700000000.000800', channel: CHANNEL_ID });++ const resultPromise = slackPostMessageHandler({+ slackChannelId: CHANNEL_ID,+ messageText: 'hello',+ });++ await vi.advanceTimersByTimeAsync(1000);++ expect(await resultPromise).toEqual({+ success: true,+ message: 'Message posted to Slack (ts=1700000000.000800).',+ slackTs: '1700000000.000800',+ channel: CHANNEL_ID,+ });+ expect(postMessageMock).toHaveBeenCalledTimes(2);+ });+it('should return a failure result when the Slack API throws', async () => {postMessageMock.mockRejectedValue(new Error('channel_not_found'));diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.tsindex 1f1e5bcd1d5..9e2b498aa55 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/collect-slack-roster-members.test.ts@@ -1,5 +1,5 @@-import { type WebClient } from '@slack/web-api';-import { describe, expect, it, vi } from 'vitest';+import { ErrorCode, type WebClient } from '@slack/web-api';+import { afterEach, describe, expect, it, vi } from 'vitest';import { type SlackRosterMember } from 'src/logic-functions/types/slack-roster-member.type';import { collectSlackRosterMembers } from 'src/logic-functions/utils/collect-slack-roster-members';@@ -22,7 +22,49 @@ const buildSlackClient = (};};+const buildRateLimitedError = (retryAfter: number): Error =>+ Object.assign(new Error(`rate limited, retry in ${retryAfter} seconds`), {+ code: ErrorCode.RateLimitedError,+ retryAfter,+ });+describe('collectSlackRosterMembers', () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ it('should wait out a rate limit when the caller passes a budget for it', async () => {+ vi.useFakeTimers();++ const usersListMock = vi+ .fn()+ .mockRejectedValueOnce(buildRateLimitedError(30))+ .mockResolvedValue({ members: [{ id: 'U1' }], response_metadata: {} });++ const membersPromise = collectSlackRosterMembers({+ slackClient: { users: { list: usersListMock } } as unknown as WebClient,+ rateLimitRetryBudgetMs: 60_000,+ });++ await vi.advanceTimersByTimeAsync(30_000);++ expect((await membersPromise).members.map((member) => member.id)).toEqual([+ 'U1',+ ]);+ expect(usersListMock).toHaveBeenCalledTimes(2);+ });++ it('should give up on a rate limit when the caller passes no budget', async () => {+ const usersListMock = vi.fn().mockRejectedValue(buildRateLimitedError(30));++ await expect(+ collectSlackRosterMembers({+ slackClient: { users: { list: usersListMock } } as unknown as WebClient,+ }),+ ).rejects.toThrow('rate limited, retry in 30 seconds');+ expect(usersListMock).toHaveBeenCalledTimes(1);+ });+it('should collect only linkable members', async () => {const { slackClient } = buildSlackClient([{diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.tsnew file mode 100644index 00000000000..50d94cbf4a6--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/fetch-slack-assistant-context.test.ts@@ -0,0 +1,159 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';++import { SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-request-timeout-ms';+import { SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-timeout-ms';+import { fetchSlackAssistantContext } from 'src/logic-functions/utils/fetch-slack-assistant-context';++const {+ getSlackClientMock,+ resolveSlackBotUserIdOrThrowMock,+ fetchSlackThreadMessagesMock,+ fetchSlackUserIdentityMock,+ isSlackDirectMessageChannelMock,+} = vi.hoisted(() => ({+ getSlackClientMock: vi.fn(),+ resolveSlackBotUserIdOrThrowMock: vi.fn(),+ fetchSlackThreadMessagesMock: vi.fn(),+ fetchSlackUserIdentityMock: vi.fn(),+ isSlackDirectMessageChannelMock: vi.fn(),+}));++vi.mock('src/logic-functions/utils/get-slack-client', () => ({+ getSlackClient: getSlackClientMock,+}));++vi.mock('src/logic-functions/utils/resolve-slack-bot-user-id-or-throw', () => ({+ resolveSlackBotUserIdOrThrow: resolveSlackBotUserIdOrThrowMock,+}));++vi.mock('src/logic-functions/utils/fetch-slack-thread-messages', () => ({+ fetchSlackThreadMessages: fetchSlackThreadMessagesMock,+}));++vi.mock('src/logic-functions/utils/fetch-slack-user-identity', () => ({+ fetchSlackUserIdentity: fetchSlackUserIdentityMock,+}));++vi.mock('src/logic-functions/utils/is-slack-direct-message-channel', () => ({+ isSlackDirectMessageChannel: isSlackDirectMessageChannelMock,+}));++const SLACK_CLIENT = { chat: {} };++const CONTEXT_ARGS = {+ slackChannelId: 'C123',+ parentMessageTimestamp: '1700000000.000100',+ slackMessageTimestamp: '1700000000.000200',+ slackUserId: 'U123',+};++describe('fetchSlackAssistantContext', () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.spyOn(console, 'warn').mockImplementation(() => undefined);+ getSlackClientMock.mockResolvedValue({+ success: true,+ client: SLACK_CLIENT,+ });+ resolveSlackBotUserIdOrThrowMock.mockResolvedValue('UBOT');+ fetchSlackThreadMessagesMock.mockResolvedValue({+ tailMessages: [{ ts: '1700000000.000100', text: 'hello', user: 'U123' }],+ requestMessage: { ts: '1700000000.000200', text: 'hi', user: 'U123' },+ });+ fetchSlackUserIdentityMock.mockResolvedValue({+ slackUserId: 'U123',+ displayName: 'Ada',+ });+ isSlackDirectMessageChannelMock.mockResolvedValue(true);+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ });++ it('should return the thread context when Slack answers in time', async () => {+ const context = await fetchSlackAssistantContext(CONTEXT_ARGS);++ expect(context.requesterName).toBe('Ada');+ expect(context.assistantBotUserId).toBe('UBOT');+ expect(context.isDirectMessage).toBe(true);+ expect(context.slackClient).toBe(SLACK_CLIENT);+ expect(context.threadMessages).toHaveLength(1);+ });++ it('should give thread reads a request timeout that fits a full page of replies', async () => {+ await fetchSlackAssistantContext(CONTEXT_ARGS);++ expect(getSlackClientMock).toHaveBeenCalledWith({+ timeout: SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS,+ });+ });++ it('should give up on a slow Slack read instead of spending the agent budget', async () => {+ vi.useFakeTimers();+ fetchSlackThreadMessagesMock.mockReturnValue(new Promise(() => undefined));++ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);++ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);++ const context = await contextPromise;++ expect(context.threadMessages).toEqual([]);+ expect(context.conversationMessages).toEqual([]);+ expect(context.requesterName).toBeUndefined();+ expect(context.isDirectMessage).toBe(false);+ expect(context.slackClient).toBe(SLACK_CLIENT);+ });++ it('should give up when acquiring the Slack client hangs', async () => {+ vi.useFakeTimers();+ getSlackClientMock.mockReturnValue(new Promise(() => undefined));++ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);++ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);++ const context = await contextPromise;++ expect(context.slackClient).toBeUndefined();+ expect(context.threadMessages).toEqual([]);+ });++ it('should share one deadline between acquiring the client and reading the thread', async () => {+ vi.useFakeTimers();+ getSlackClientMock.mockImplementation(+ () =>+ new Promise((resolve) =>+ setTimeout(+ () => resolve({ success: true, client: SLACK_CLIENT }),+ SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS * 0.6,+ ),+ ),+ );+ fetchSlackThreadMessagesMock.mockReturnValue(new Promise(() => undefined));++ const contextPromise = fetchSlackAssistantContext(CONTEXT_ARGS);++ await vi.advanceTimersByTimeAsync(SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS);++ const context = await contextPromise;++ expect(context.slackClient).toBe(SLACK_CLIENT);+ expect(context.threadMessages).toEqual([]);+ });++ it('should return an unreachable context when Slack is not connected', async () => {+ getSlackClientMock.mockResolvedValue({+ success: false,+ error: 'Slack is not connected',+ });++ const context = await fetchSlackAssistantContext(CONTEXT_ARGS);++ expect(context.slackClient).toBeUndefined();+ expect(context.threadMessages).toEqual([]);+ expect(fetchSlackThreadMessagesMock).not.toHaveBeenCalled();+ });+});diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.tsnew file mode 100644index 00000000000..1513e13651e--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/get-slack-client.test.ts@@ -0,0 +1,58 @@+import { beforeEach, describe, expect, it, vi } from 'vitest';++import { getSlackClient } from 'src/logic-functions/utils/get-slack-client';++const { webClientMock, getSlackConnectionMock } = vi.hoisted(() => ({+ webClientMock: vi.fn(),+ getSlackConnectionMock: vi.fn(),+}));++vi.mock('@slack/web-api', () => ({+ WebClient: webClientMock,+}));++vi.mock('src/logic-functions/utils/get-slack-connection', () => ({+ getSlackConnection: getSlackConnectionMock,+}));++describe('getSlackClient', () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getSlackConnectionMock.mockResolvedValue({+ success: true,+ accessToken: 'xoxb-token',+ });+ });++ it('should bound retries and request time so a call cannot outlive its function', async () => {+ await getSlackClient();++ expect(webClientMock).toHaveBeenCalledWith('xoxb-token', {+ timeout: 3000,+ retryConfig: { retries: 1, factor: 1, minTimeout: 250, maxTimeout: 250 },+ rejectRateLimitedCalls: true,+ });+ });++ it('should let a caller with more budget override the defaults', async () => {+ await getSlackClient({ retryConfig: { retries: 0 }, timeout: 20_000 });++ expect(webClientMock).toHaveBeenCalledWith('xoxb-token', {+ timeout: 20_000,+ retryConfig: { retries: 0 },+ rejectRateLimitedCalls: true,+ });+ });++ it('should not build a client when Slack is not connected', async () => {+ getSlackConnectionMock.mockResolvedValue({+ success: false,+ error: 'Slack is not connected',+ });++ const result = await getSlackClient();++ expect(result).toEqual({ success: false, error: 'Slack is not connected' });+ expect(webClientMock).not.toHaveBeenCalled();+ });+});diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.tsnew file mode 100644index 00000000000..668071b6466--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/retry-slack-call-when-rate-limited.test.ts@@ -0,0 +1,111 @@+import { ErrorCode } from '@slack/web-api';+import { afterEach, describe, expect, it, vi } from 'vitest';++import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';++const buildRateLimitedError = (retryAfter: number): Error => {+ const error = new Error(`rate limited, retry in ${retryAfter} seconds`);++ Object.assign(error, { code: ErrorCode.RateLimitedError, retryAfter });++ return error;+};++describe('retrySlackCallWhenRateLimited', () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ it('should return the result without retrying when the call succeeds', async () => {+ const call = vi.fn().mockResolvedValue({ ts: '1700000000.000100' });++ const result = await retrySlackCallWhenRateLimited({ call });++ expect(result).toEqual({ ts: '1700000000.000100' });+ expect(call).toHaveBeenCalledTimes(1);+ });++ it('should wait out a short Retry-After and retry once', async () => {+ vi.useFakeTimers();++ const call = vi+ .fn()+ .mockRejectedValueOnce(buildRateLimitedError(2))+ .mockResolvedValue({ ts: '1700000000.000100' });++ const resultPromise = retrySlackCallWhenRateLimited({ call });++ await vi.advanceTimersByTimeAsync(2000);++ expect(await resultPromise).toEqual({ ts: '1700000000.000100' });+ expect(call).toHaveBeenCalledTimes(2);+ });++ it('should rethrow when Retry-After is longer than the retry budget', async () => {+ const call = vi.fn().mockRejectedValue(buildRateLimitedError(60));++ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(+ 'rate limited, retry in 60 seconds',+ );+ expect(call).toHaveBeenCalledTimes(1);+ });++ it('should rethrow the second rate limit instead of retrying forever', async () => {+ vi.useFakeTimers();++ const call = vi.fn().mockRejectedValue(buildRateLimitedError(1));++ const resultPromise = retrySlackCallWhenRateLimited({ call });+ const assertion = expect(resultPromise).rejects.toThrow(+ 'rate limited, retry in 1 seconds',+ );++ await vi.advanceTimersByTimeAsync(1000);+ await assertion;++ expect(call).toHaveBeenCalledTimes(2);+ });++ it('should not retry when the first attempt already spent the budget', async () => {+ vi.useFakeTimers();++ const call = vi.fn().mockImplementation(async () => {+ vi.advanceTimersByTime(3500);++ throw buildRateLimitedError(1);+ });++ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(+ 'rate limited, retry in 1 seconds',+ );+ expect(call).toHaveBeenCalledTimes(1);+ });++ it('should wait out a long Retry-After when the caller has the budget for it', async () => {+ vi.useFakeTimers();++ const call = vi+ .fn()+ .mockRejectedValueOnce(buildRateLimitedError(60))+ .mockResolvedValue({ members: [] });++ const resultPromise = retrySlackCallWhenRateLimited({+ call,+ budgetMs: 90_000,+ });++ await vi.advanceTimersByTimeAsync(60_000);++ expect(await resultPromise).toEqual({ members: [] });+ expect(call).toHaveBeenCalledTimes(2);+ });++ it('should not retry an error that is not a rate limit', async () => {+ const call = vi.fn().mockRejectedValue(new Error('channel_not_found'));++ await expect(retrySlackCallWhenRateLimited({ call })).rejects.toThrow(+ 'channel_not_found',+ );+ expect(call).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.tsnew file mode 100644index 00000000000..d025cae78f6--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/__tests__/run-with-timeout.test.ts@@ -0,0 +1,57 @@+import { afterEach, describe, expect, it, vi } from 'vitest';++import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';++describe('runWithTimeout', () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ it('should return the operation value when it settles in time', async () => {+ const result = await runWithTimeout({+ operation: Promise.resolve('context'),+ timeoutMs: 1000,+ buildTimeoutValue: () => 'fallback',+ });++ expect(result).toBe('context');+ });++ it('should return the timeout value when the operation is too slow', async () => {+ vi.useFakeTimers();++ const pendingOperation = new Promise<string>(() => undefined);++ const resultPromise = runWithTimeout({+ operation: pendingOperation,+ timeoutMs: 1000,+ buildTimeoutValue: () => 'fallback',+ });++ await vi.advanceTimersByTimeAsync(1000);++ expect(await resultPromise).toBe('fallback');+ });++ it('should not build the timeout value when the operation wins, so callers do not log a timeout that never happened', async () => {+ const buildTimeoutValue = vi.fn(() => 'fallback');++ await runWithTimeout({+ operation: Promise.resolve('context'),+ timeoutMs: 1000,+ buildTimeoutValue,+ });++ expect(buildTimeoutValue).not.toHaveBeenCalled();+ });++ it('should propagate a rejected operation', async () => {+ await expect(+ runWithTimeout({+ operation: Promise.reject(new Error('read failed')),+ timeoutMs: 1000,+ buildTimeoutValue: () => 'fallback',+ }),+ ).rejects.toThrow('read failed');+ });+});diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.tsindex b409863b4a2..66a9ec4da70 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/collect-slack-roster-members.ts@@ -2,8 +2,10 @@ import { type WebClient } from '@slack/web-api';import { isNonEmptyString } from '@sniptt/guards';import { isDefined } from 'twenty-sdk/utils';+import { SLACK_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-rate-limit-retry-budget-ms';import { type LinkableSlackRosterMember } from 'src/logic-functions/types/slack-roster-member.type';import { isLinkableSlackRosterMember } from 'src/logic-functions/utils/is-linkable-slack-roster-member';+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';const MEMBERS_PER_PAGE = 200;const DEFAULT_MAX_PAGES = 5;@@ -18,20 +20,28 @@ export const collectSlackRosterMembers = async ({shouldCollectMember,maxMembers,maxPages = DEFAULT_MAX_PAGES,+ rateLimitRetryBudgetMs = SLACK_RATE_LIMIT_RETRY_BUDGET_MS,}: {slackClient: WebClient;shouldCollectMember?: (member: LinkableSlackRosterMember) => boolean;maxMembers?: number;maxPages?: number;+ rateLimitRetryBudgetMs?: number;}): Promise<CollectSlackRosterMembersResult> => {const members: LinkableSlackRosterMember[] = [];const seenSlackUserIds = new Set<string>();+ // one window for every page, so five rate limited pages cannot wait five times+ const rateLimitDeadlineAtMs = Date.now() + rateLimitRetryBudgetMs;let cursor: string | undefined;for (let page = 0; page < maxPages; page += 1) {- const response = await slackClient.users.list({- limit: MEMBERS_PER_PAGE,- cursor,+ const response = await retrySlackCallWhenRateLimited({+ call: async () =>+ slackClient.users.list({+ limit: MEMBERS_PER_PAGE,+ cursor,+ }),+ budgetMs: Math.max(rateLimitDeadlineAtMs - Date.now(), 0),});const collectiblePageMembers = (response.members ?? [])diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.tsindex 2c3bec5a0b0..1809e51e8d2 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/fetch-slack-assistant-context.ts@@ -1,5 +1,7 @@import { type WebClient } from '@slack/web-api';+import { SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-request-timeout-ms';+import { SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS } from 'src/logic-functions/constants/slack-assistant-context-timeout-ms';import { type SlackAssistantAgentMessage } from 'src/logic-functions/types/slack-assistant-agent-message.type';import { type SlackThreadMessage } from 'src/logic-functions/types/slack-thread-message.type';import { type SlackUserIdentity } from 'src/logic-functions/types/slack-user-identity.type';@@ -9,6 +11,7 @@ import { fetchSlackUserIdentity } from 'src/logic-functions/utils/fetch-slack-usimport { getSlackClient } from 'src/logic-functions/utils/get-slack-client';import { isSlackDirectMessageChannel } from 'src/logic-functions/utils/is-slack-direct-message-channel';import { resolveSlackBotUserIdOrThrow } from 'src/logic-functions/utils/resolve-slack-bot-user-id-or-throw';+import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';type SlackAssistantContext = {conversationMessages: SlackAssistantAgentMessage[];@@ -32,25 +35,19 @@ const UNREACHABLE_SLACK_CONTEXT: SlackAssistantContext = {isDirectMessage: false,};-export const fetchSlackAssistantContext = async ({+const readSlackThreadContext = async ({+ client,slackChannelId,parentMessageTimestamp,slackMessageTimestamp,slackUserId,}: {+ client: WebClient;slackChannelId: string;parentMessageTimestamp: string;slackMessageTimestamp: string;slackUserId: string | undefined;}): Promise<SlackAssistantContext> => {- const slackClientResult = await getSlackClient();-- if (!slackClientResult.success) {- return UNREACHABLE_SLACK_CONTEXT;- }-- const { client } = slackClientResult;-const assistantBotUserId = await resolveSlackBotUserIdOrThrow().catch((error) => {console.warn(@@ -88,3 +85,59 @@ export const fetchSlackAssistantContext = async ({isDirectMessage,};};++export const fetchSlackAssistantContext = async ({+ slackChannelId,+ parentMessageTimestamp,+ slackMessageTimestamp,+ slackUserId,+}: {+ slackChannelId: string;+ parentMessageTimestamp: string;+ slackMessageTimestamp: string;+ slackUserId: string | undefined;+}): Promise<SlackAssistantContext> => {+ // the agent budget is already ticking: partial context beats a late answer+ const contextDeadlineAtMs = Date.now() + SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS;++ const slackClientResult = await runWithTimeout({+ operation: getSlackClient({+ timeout: SLACK_ASSISTANT_CONTEXT_REQUEST_TIMEOUT_MS,+ }),+ timeoutMs: SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS,+ buildTimeoutValue: () => {+ console.warn(+ `[slack] acquiring the Slack client exceeded ${SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS}ms, answering without thread history`,+ );++ return {+ success: false as const,+ error: 'Timed out acquiring the Slack client',+ };+ },+ });++ if (!slackClientResult.success) {+ return UNREACHABLE_SLACK_CONTEXT;+ }++ const { client } = slackClientResult;++ return await runWithTimeout({+ operation: readSlackThreadContext({+ client,+ slackChannelId,+ parentMessageTimestamp,+ slackMessageTimestamp,+ slackUserId,+ }),+ timeoutMs: Math.max(contextDeadlineAtMs - Date.now(), 0),+ buildTimeoutValue: () => {+ console.warn(+ `[slack] assistant context read exceeded ${SLACK_ASSISTANT_CONTEXT_TIMEOUT_MS}ms, answering without thread history`,+ );++ return { ...UNREACHABLE_SLACK_CONTEXT, slackClient: client };+ },+ });+};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.tsindex bc34ff713d3..d8d5cdbfe42 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/get-slack-client.ts@@ -1,5 +1,7 @@import { WebClient, type WebClientOptions } from '@slack/web-api';+import { SLACK_CLIENT_REQUEST_TIMEOUT_MS } from 'src/logic-functions/constants/slack-client-request-timeout-ms';+import { SLACK_CLIENT_RETRY_CONFIG } from 'src/logic-functions/constants/slack-client-retry-config';import { getSlackConnection } from 'src/logic-functions/utils/get-slack-connection';export const getSlackClient = async (@@ -15,6 +17,12 @@ export const getSlackClient = async (return {success: true,- client: new WebClient(connectionResult.accessToken, options),+ client: new WebClient(connectionResult.accessToken, {+ timeout: SLACK_CLIENT_REQUEST_TIMEOUT_MS,+ retryConfig: SLACK_CLIENT_RETRY_CONFIG,+ // otherwise a 429 sleeps the whole Retry-After before any retry budget+ rejectRateLimitedCalls: true,+ ...options,+ }),};};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.tsnew file mode 100644index 00000000000..6ecd86488d6--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/is-slack-rate-limited-error.ts@@ -0,0 +1,9 @@+import { ErrorCode, type WebAPIRateLimitedError } from '@slack/web-api';+import { isObject } from '@sniptt/guards';++export const isSlackRateLimitedError = (+ error: unknown,+): error is WebAPIRateLimitedError =>+ isObject(error) &&+ 'code' in error &&+ error.code === ErrorCode.RateLimitedError;diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.tsindex 6423f3d09a4..c1a2f530c07 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/match-slack-roster-by-email.ts@@ -2,6 +2,7 @@ import { type WebClient } from '@slack/web-api';import { isNonEmptyString } from '@sniptt/guards';import { CoreApiClient } from 'twenty-client-sdk/core';+import { SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-roster-rate-limit-retry-budget-ms';import { listLinkedSlackUserIds } from 'src/logic-functions/data/list-linked-slack-user-ids';import { findWorkspaceMemberIdsByEmails } from 'src/logic-functions/data/find-workspace-member-ids-by-emails';import { type SlackRosterMatchSummary } from 'src/logic-functions/types/slack-roster-match.type';@@ -23,7 +24,10 @@ const runSlackRosterMatch = async ({const [linkedSlackUserIds, roster] = await Promise.all([listLinkedSlackUserIds(client, { slackTeamId }),- collectSlackRosterMembers({ slackClient }),+ collectSlackRosterMembers({+ slackClient,+ rateLimitRetryBudgetMs: SLACK_ROSTER_RATE_LIMIT_RETRY_BUDGET_MS,+ }),]);const vouchedRosterEmails = roster.membersdiff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.tsindex f70d248bc59..3c9ea18b809 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/post-slack-message.ts@@ -7,6 +7,7 @@ import { buildSlackRecordEntitiesForMessage } from 'src/logic-functions/utils/buimport { getSlackApiErrorCode } from 'src/logic-functions/utils/get-slack-api-error-code';import { isSlackMetadataError } from 'src/logic-functions/utils/is-slack-metadata-error';import { normalizeSlackParentMessageTimestamp } from 'src/logic-functions/utils/normalize-slack-parent-message-timestamp';+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';import { sendSlackMessageWithBodyFallbacks } from 'src/logic-functions/utils/send-slack-message-with-body-fallbacks';export const postSlackMessage = async (@@ -33,17 +34,20 @@ export const postSlackMessage = async (failureMessage: 'Failed to post Slack message',sendMessage: async (bodyFields) => {try {- const data = await client.chat.postMessage({- channel: parameters.slackChannelId,- thread_ts: parentTimestamp,- ...(isDefined(parameters.unfurlLinks)- ? { unfurl_links: parameters.unfurlLinks }- : {}),- ...(isDefined(parameters.unfurlMedia)- ? { unfurl_media: parameters.unfurlMedia }- : {}),- ...(entities.length > 0 ? { metadata: { entities } } : {}),- ...bodyFields,+ const data = await retrySlackCallWhenRateLimited({+ call: async () =>+ client.chat.postMessage({+ channel: parameters.slackChannelId,+ thread_ts: parentTimestamp,+ ...(isDefined(parameters.unfurlLinks)+ ? { unfurl_links: parameters.unfurlLinks }+ : {}),+ ...(isDefined(parameters.unfurlMedia)+ ? { unfurl_media: parameters.unfurlMedia }+ : {}),+ ...(entities.length > 0 ? { metadata: { entities } } : {}),+ ...bodyFields,+ }),});return {diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.tsindex 0022a42557c..16d7327415e 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/race-slack-assistant-agent-deadline.ts@@ -1,6 +1,7 @@import { type RunAgentResult } from 'twenty-sdk/logic-function';import { SLACK_ASSISTANT_DEADLINE_ERROR } from 'src/logic-functions/constants/slack-assistant-deadline-error';+import { runWithTimeout } from 'src/logic-functions/utils/run-with-timeout';export const raceSlackAssistantAgentDeadline = async ({agentRun,@@ -8,24 +9,13 @@ export const raceSlackAssistantAgentDeadline = async ({}: {agentRun: Promise<RunAgentResult>;deadlineAtMs: number;-}): Promise<RunAgentResult> => {- let deadlineTimer: ReturnType<typeof setTimeout> | undefined;-- const deadlineResult = new Promise<RunAgentResult>((resolve) => {- deadlineTimer = setTimeout(- () =>- resolve({- result: null,- error: SLACK_ASSISTANT_DEADLINE_ERROR,- success: false,- }),- Math.max(deadlineAtMs - Date.now(), 0),- );+}): Promise<RunAgentResult> =>+ await runWithTimeout({+ operation: agentRun,+ timeoutMs: Math.max(deadlineAtMs - Date.now(), 0),+ buildTimeoutValue: () => ({+ result: null,+ error: SLACK_ASSISTANT_DEADLINE_ERROR,+ success: false,+ }),});-- try {- return await Promise.race([agentRun, deadlineResult]);- } finally {- clearTimeout(deadlineTimer);- }-};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.tsnew file mode 100644index 00000000000..f1c27de5e17--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/retry-slack-call-when-rate-limited.ts@@ -0,0 +1,31 @@+import { SLACK_RATE_LIMIT_RETRY_BUDGET_MS } from 'src/logic-functions/constants/slack-rate-limit-retry-budget-ms';+import { isSlackRateLimitedError } from 'src/logic-functions/utils/is-slack-rate-limited-error';++// rejecting a rate limited write outright would cost the member the answer+export const retrySlackCallWhenRateLimited = async <TResult>({+ call,+ budgetMs = SLACK_RATE_LIMIT_RETRY_BUDGET_MS,+}: {+ call: () => Promise<TResult>;+ budgetMs?: number;+}): Promise<TResult> => {+ const startedAtMs = Date.now();++ try {+ return await call();+ } catch (error) {+ if (!isSlackRateLimitedError(error)) {+ throw error;+ }++ const waitMs = error.retryAfter * 1000;++ if (Date.now() - startedAtMs + waitMs > budgetMs) {+ throw error;+ }++ await new Promise((resolve) => setTimeout(resolve, waitMs));++ return await call();+ }+};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.tsnew file mode 100644index 00000000000..9f0685e2c1b--- /dev/null+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/run-with-timeout.ts@@ -0,0 +1,21 @@+export const runWithTimeout = async <TValue>({+ operation,+ timeoutMs,+ buildTimeoutValue,+}: {+ operation: Promise<TValue>;+ timeoutMs: number;+ buildTimeoutValue: () => TValue;+}): Promise<TValue> => {+ let timeoutTimer: ReturnType<typeof setTimeout> | undefined;++ const timeoutValue = new Promise<TValue>((resolve) => {+ timeoutTimer = setTimeout(() => resolve(buildTimeoutValue()), timeoutMs);+ });++ try {+ return await Promise.race([operation, timeoutValue]);+ } finally {+ clearTimeout(timeoutTimer);+ }+};diff --git a/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts b/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.tsindex 7b1da269268..c6944a80d59 100644--- a/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts+++ b/packages/twenty-apps/public/slack/src/logic-functions/utils/send-slack-user-link-consent-dm.ts@@ -2,6 +2,7 @@ import { type WebClient } from '@slack/web-api';import { isNonEmptyString } from '@sniptt/guards';import { buildSlackUserLinkConsentBlocks } from 'src/logic-functions/utils/build-slack-user-link-consent-blocks';+import { retrySlackCallWhenRateLimited } from 'src/logic-functions/utils/retry-slack-call-when-rate-limited';import { toErrorMessage } from 'src/logic-functions/utils/to-error-message.util';export const sendSlackUserLinkConsentDm = async (@@ -34,16 +35,19 @@ export const sendSlackUserLinkConsentDm = async (};}- await slackClient.chat.postMessage({- channel: channelId,- text: 'A Twenty admin asked to link your Slack account. Approve or decline it here.',- blocks: buildSlackUserLinkConsentBlocks({- memberName,- slackTeamId,- slackUserId,- workspaceMemberId,- slackUserLinkId,- }),+ await retrySlackCallWhenRateLimited({+ call: async () =>+ slackClient.chat.postMessage({+ channel: channelId,+ text: 'A Twenty admin asked to link your Slack account. Approve or decline it here.',+ blocks: buildSlackUserLinkConsentBlocks({+ memberName,+ slackTeamId,+ slackUserId,+ workspaceMemberId,+ slackUserLinkId,+ }),+ }),});return { success: true };