The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Reconcile stale Fathom media imports (#25434)
nitin1c75c70440bb
diff --git a/packages/twenty-apps/public/fathom/CHANGELOG.md b/packages/twenty-apps/public/fathom/CHANGELOG.mdindex 6c0b4e25999..c039d5ddc96 100644--- a/packages/twenty-apps/public/fathom/CHANGELOG.md+++ b/packages/twenty-apps/public/fathom/CHANGELOG.md@@ -11,6 +11,10 @@ All notable changes to this application are documented in this file.it to try again.- Hold a recording in Processing until its media lands or is settled asunavailable, rather than completing it as soon as the transcript arrives.+- Reconcile unfinished local media imports from the latest seven days daily,+ without scanning Fathom history, and finish cleanup after disconnection.+- Leave rate-limited media imports recoverable after their bounded retries,+ and retry saving a generated download ID before failing the job.## 0.1.0diff --git a/packages/twenty-apps/public/fathom/README.md b/packages/twenty-apps/public/fathom/README.mdindex 591e1508c48..e986f342d1c 100644--- a/packages/twenty-apps/public/fathom/README.md+++ b/packages/twenty-apps/public/fathom/README.md@@ -27,6 +27,11 @@ emails do not need to match.- Fathom generates the downloadable video in the background, so a recordingstays in Processing for a few minutes after its transcript arrives, until itsmedia lands or is settled as unavailable.+- A daily reconciliation pass scans local Call Recordings started in the last+ seven days and resumes media imports inactive for at least 30 minutes.+- Media imports respect Fathom's Retry-After delay for up to three retries.+ Continued throttling leaves the import unfinished for the daily recovery+ pass; recordings outside its seven-day window need Sync Fathom Call.- Media is skipped for recordings above 500 MB, for recordings Fathom has nodownloadable media for, and for limited-access shares that the connectedaccount may view but not download. Fathom records the reason internally so@@ -38,3 +43,10 @@ emails do not need to match.scopes so Fathom receives the new registration.Development setup lives in [SETUP.md](SETUP.md).++## Reliability limits++Import pacing is best-effort: concurrent jobs can reserve overlapping slots.+After Fathom creates a download, Twenty retries saving its ID up to three total+attempts. A crash or failed save can still cause a later retry to request another+download generation; provider creation and local persistence are not atomic.diff --git a/packages/twenty-apps/public/fathom/src/__tests__/disconnected-media-cleanup.integration-test.ts b/packages/twenty-apps/public/fathom/src/__tests__/disconnected-media-cleanup.integration-test.tsnew file mode 100644index 00000000000..e8c0423a022--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/__tests__/disconnected-media-cleanup.integration-test.ts@@ -0,0 +1,100 @@+import { isNonEmptyString } from '@sniptt/guards';+import { randomUUID } from 'node:crypto';+import { CoreApiClient } from 'twenty-client-sdk/core';+import { functionExecute } from 'twenty-sdk/cli';+import { describe, expect, it } from 'vitest';++import { createFathomApplicationCoreApiClient } from 'src/__tests__/utils/create-fathom-application-core-api-client.util';++describe('Fathom disconnected media cleanup', () => {+ it.each([+ { description: 'null media', video: null, audio: null },+ { description: 'empty media arrays', video: [], audio: [] },+ ])(+ 'settles an empty transcript with $description',+ async ({ video, audio }) => {+ const coreApiClient = new CoreApiClient();+ const applicationCoreApiClient =+ await createFathomApplicationCoreApiClient();+ const callRecordingId = randomUUID();+ const connectedAccountId = randomUUID();++ await coreApiClient.mutation({+ createCallRecording: {+ __args: {+ data: {+ id: callRecordingId,+ title: 'Fathom disconnected cleanup integration fixture',+ status: 'PROCESSING',+ transcript: [],+ video,+ audio,+ },+ },+ id: true,+ },+ });++ await applicationCoreApiClient.mutation({+ createFathomRecordingImport: {+ __args: {+ data: {+ id: callRecordingId,+ callRecordingId,+ recordingId: randomUUID(),+ connectedAccountId,+ mediaDownloadId: randomUUID(),+ },+ },+ id: true,+ },+ });++ try {+ const result = await functionExecute({+ appPath: process.cwd(),+ functionName: 'fathom-reconcile-media-imports',+ payload: { disconnectedAccountId: connectedAccountId },+ });++ if (!result.success) {+ throw new Error(result.error.message);+ }++ expect(result.data.status).toBe('SUCCESS');+ expect(result.data.data).toEqual({+ candidateCount: 1,+ updatedRecordingCount: 1,+ shouldContinue: false,+ skipped: false,+ });++ const current = await coreApiClient.query({+ callRecording: {+ __args: { filter: { id: { eq: callRecordingId } } },+ status: true,+ },+ fathomRecordingImport: {+ __args: { filter: { id: { eq: callRecordingId } } },+ mediaFailureReason: true,+ mediaDownloadId: true,+ mediaImportClaimedAt: true,+ },+ });++ expect(current.callRecording?.status).toBe('FAILED');+ expect(current.fathomRecordingImport).toMatchObject({+ mediaFailureReason: 'connected_account_unavailable',+ mediaImportClaimedAt: null,+ });+ expect(+ isNonEmptyString(current.fathomRecordingImport?.mediaDownloadId),+ ).toBe(false);+ } finally {+ await coreApiClient.mutation({+ destroyCallRecording: { __args: { id: callRecordingId }, id: true },+ });+ }+ },+ );+});diff --git a/packages/twenty-apps/public/fathom/src/__tests__/media-import-ownership.integration-test.ts b/packages/twenty-apps/public/fathom/src/__tests__/media-import-ownership.integration-test.tsindex 864e93f423e..b8de16e1bc2 100644--- a/packages/twenty-apps/public/fathom/src/__tests__/media-import-ownership.integration-test.ts+++ b/packages/twenty-apps/public/fathom/src/__tests__/media-import-ownership.integration-test.ts@@ -1,42 +1,10 @@import { randomUUID } from 'node:crypto';-import { CoreApiClient } from 'twenty-client-sdk/core';-import { MetadataApiClient } from 'twenty-client-sdk/metadata';import { describe, expect, it } from 'vitest';-import { APPLICATION_UNIVERSAL_IDENTIFIER } from 'src/constants/universal-identifiers';+import { createFathomApplicationCoreApiClient } from 'src/__tests__/utils/create-fathom-application-core-api-client.util';import { claimFathomMediaImport } from 'src/logic-functions/utils/claim-fathom-media-import.util';import { updateFathomRecordingImport } from 'src/logic-functions/utils/update-fathom-recording-import.util';--const createFathomApplicationCoreApiClient = async (): Promise<CoreApiClient> => {- const metadataApiClient = new MetadataApiClient();- const applicationsResult = await metadataApiClient.query({- findManyApplications: {- id: true,- universalIdentifier: true,- },- });- const fathomApplication = applicationsResult.findManyApplications.find(- (application) =>- application.universalIdentifier === APPLICATION_UNIVERSAL_IDENTIFIER,- );-- if (!fathomApplication) {- throw new Error('Expected the Fathom application to be installed');- }-- const applicationTokenResult = await metadataApiClient.mutation({- generateApplicationToken: {- __args: { applicationId: fathomApplication.id },- applicationAccessToken: { token: true },- },- });-- return new CoreApiClient({- headers: {- Authorization: `Bearer ${applicationTokenResult.generateApplicationToken.applicationAccessToken.token}`,- },- });-};+import { updateFathomMediaDownloadId } from 'src/logic-functions/utils/update-fathom-media-download-id.util';describe('Fathom recording import ownership', () => {it.each([@@ -123,16 +91,30 @@ describe('Fathom recording import ownership', () => {}),).toBe(false);+ const replacementDownloadId = randomUUID();++ expect(+ await updateFathomMediaDownloadId({+ coreApiClient,+ writeContext,+ downloadId: replacementDownloadId,+ }),+ ).toBe(false);+const current = await coreApiClient.query({fathomRecordingImport: {__args: { filter: { id: { eq: fathomRecordingImportId } } },mediaFailureReason: true,+ mediaDownloadId: true,},});expect(current.fathomRecordingImport?.mediaFailureReason).toBe('download_expired',);+ expect(current.fathomRecordingImport?.mediaDownloadId).not.toBe(+ replacementDownloadId,+ );} finally {await coreApiClient.mutation({destroyFathomRecordingImport: {diff --git a/packages/twenty-apps/public/fathom/src/__tests__/utils/create-fathom-application-core-api-client.util.ts b/packages/twenty-apps/public/fathom/src/__tests__/utils/create-fathom-application-core-api-client.util.tsnew file mode 100644index 00000000000..512d3608855--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/__tests__/utils/create-fathom-application-core-api-client.util.ts@@ -0,0 +1,37 @@+import { isDefined } from 'src/utils/is-defined';+import { CoreApiClient } from 'twenty-client-sdk/core';+import { MetadataApiClient } from 'twenty-client-sdk/metadata';++import { APPLICATION_UNIVERSAL_IDENTIFIER } from 'src/constants/universal-identifiers';++export const createFathomApplicationCoreApiClient =+ async (): Promise<CoreApiClient> => {+ const metadataApiClient = new MetadataApiClient();+ const applicationsResult = await metadataApiClient.query({+ findManyApplications: {+ id: true,+ universalIdentifier: true,+ },+ });+ const fathomApplication = applicationsResult.findManyApplications.find(+ (application) =>+ application.universalIdentifier === APPLICATION_UNIVERSAL_IDENTIFIER,+ );++ if (!isDefined(fathomApplication)) {+ throw new Error('Expected the Fathom application to be installed');+ }++ const applicationTokenResult = await metadataApiClient.mutation({+ generateApplicationToken: {+ __args: { applicationId: fathomApplication.id },+ applicationAccessToken: { token: true },+ },+ });++ return new CoreApiClient({+ headers: {+ Authorization: `Bearer ${applicationTokenResult.generateApplicationToken.applicationAccessToken.token}`,+ },+ });+ };diff --git a/packages/twenty-apps/public/fathom/src/constants/fathom-media-reconciliation-selection.constant.ts b/packages/twenty-apps/public/fathom/src/constants/fathom-media-reconciliation-selection.constant.tsnew file mode 100644index 00000000000..5c87a4efe00--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/constants/fathom-media-reconciliation-selection.constant.ts@@ -0,0 +1,28 @@+export const FATHOM_MEDIA_RECONCILIATION_SELECTION = {+ pageInfo: { hasNextPage: true },+ edges: {+ node: {+ id: true,+ video: { fileId: true },+ audio: { fileId: true },+ fathomRecordingImports: {+ __args: { first: 1 },+ edges: {+ node: {+ id: true,+ updatedAt: true,+ recordingId: true,+ connectedAccountId: true,+ mediaDownloadId: true,+ mediaFailureReason: true,+ mediaImportClaimedAt: true,+ mediaUploadCheckpoint: true,+ },+ },+ },+ transcript: true,+ status: true,+ updatedAt: true,+ },+ },+};diff --git a/packages/twenty-apps/public/fathom/src/constants/fathom.constant.ts b/packages/twenty-apps/public/fathom/src/constants/fathom.constant.tsindex c8bf4908592..1bd60b517f9 100644--- a/packages/twenty-apps/public/fathom/src/constants/fathom.constant.ts+++ b/packages/twenty-apps/public/fathom/src/constants/fathom.constant.ts@@ -5,6 +5,7 @@ export const MAX_CALENDAR_EVENT_PAGES = 10;export const FATHOM_BACKFILL_ROUTE_PATH = '/fathom/backfill';export const FATHOM_BACKFILL_MAX_WINDOW_DAYS = 3_650;export const FATHOM_INITIAL_BACKFILL_DAYS = 31;+export const FATHOM_HEALING_WINDOW_DAYS = 7;export const FATHOM_BACKFILL_BATCH_SIZE = 5;export const FATHOM_IMPORT_SLOT_INTERVAL_MILLISECONDS = 20_000;export const FATHOM_BACKFILL_JOB_RETRY_LIMIT = 3;@@ -22,3 +23,8 @@ export const FATHOM_MEDIA_DOWNLOAD_INITIAL_POLL_DELAY_MILLISECONDS = 60_000;export const FATHOM_MEDIA_DOWNLOAD_MAX_POLL_DELAY_MILLISECONDS = 300_000;export const FATHOM_MEDIA_DOWNLOAD_POLL_BACKOFF_EXPONENT = 1.5;export const FATHOM_MEDIA_DOWNLOAD_MAX_POLL_ATTEMPTS = 12;+export const FATHOM_MEDIA_RECONCILIATION_GRACE_PERIOD_MILLISECONDS =+ 30 * 60 * 1_000;+export const FATHOM_MEDIA_RECONCILIATION_PAGE_SIZE = 200;+export const FATHOM_MEDIA_RECONCILIATION_CRON_PATTERN = '30 3 * * *';+export const FATHOM_MEDIA_RECONCILIATION_BATCH_SIZE = 5;diff --git a/packages/twenty-apps/public/fathom/src/constants/universal-identifiers.ts b/packages/twenty-apps/public/fathom/src/constants/universal-identifiers.tsindex 7826d3abc32..a5c82d06dfd 100644--- a/packages/twenty-apps/public/fathom/src/constants/universal-identifiers.ts+++ b/packages/twenty-apps/public/fathom/src/constants/universal-identifiers.ts@@ -53,6 +53,8 @@ export const FATHOM_REQUEST_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER ='87b12e47-2a6c-49e3-aafa-1bb29c0ce01f';export const FATHOM_IMPORT_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER ='4b46124e-ef53-4649-b414-cc5ae47111ff';+export const FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER =+ '3670d009-0c34-454c-9b21-254fd8349f61';export const CALL_RECORDING_VIDEO_FIELD_UNIVERSAL_IDENTIFIER ='bb9523d3-457e-4f4b-8c79-27a77afb87da';export const CALL_RECORDING_AUDIO_FIELD_UNIVERSAL_IDENTIFIER =diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/__tests__/fathom-disconnect.test.ts b/packages/twenty-apps/public/fathom/src/logic-functions/__tests__/fathom-disconnect.test.tsindex 22fb8606dca..ce69138eba0 100644--- a/packages/twenty-apps/public/fathom/src/logic-functions/__tests__/fathom-disconnect.test.ts+++ b/packages/twenty-apps/public/fathom/src/logic-functions/__tests__/fathom-disconnect.test.ts@@ -1,9 +1,11 @@import { beforeEach, describe, expect, it, vi } from 'vitest';import { buildFathomNotFoundError } from 'src/__tests__/utils/build-fathom-not-found-error.util';+import { FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER } from 'src/constants/universal-identifiers';const sdkMocks = vi.hoisted(() => ({deleteWebhook: vi.fn(),+ enqueueJobs: vi.fn(),getConnection: vi.fn(),kvDelete: vi.fn(),kvGet: vi.fn(),@@ -14,8 +16,10 @@ vi.mock('twenty-sdk/define', () => ({defineLogicFunction: (config: unknown) => config,}));-vi.mock('twenty-sdk/logic-function', () => ({+vi.mock('twenty-sdk/logic-function', async (importOriginal) => ({+ ...(await importOriginal<typeof import('twenty-sdk/logic-function')>()),kv: { get: sdkMocks.kvGet, set: sdkMocks.kvSet, delete: sdkMocks.kvDelete },+ enqueueJobs: sdkMocks.enqueueJobs,getConnection: sdkMocks.getConnection,}));@@ -25,9 +29,8 @@ vi.mock('fathom-typescript', () => ({},}));-const { fathomDisconnectHandler } = await import(- 'src/logic-functions/fathom-disconnect'-);+const { fathomDisconnectHandler } =+ await import('src/logic-functions/fathom-disconnect');const payload = {connectionProviderId: 'provider-1',@@ -42,11 +45,19 @@ const registration = {isInitialBackfillEnqueued: true,};+const MEDIA_CLEANUP_JOB = {+ logicFunctionUniversalIdentifier:+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ payloads: [{ disconnectedAccountId: 'connection-1' }],+ retryLimit: 3,+};+describe('fathomDisconnectHandler', () => {beforeEach(() => {vi.resetAllMocks();vi.spyOn(console, 'error').mockImplementation(() => undefined);sdkMocks.kvGet.mockResolvedValue(registration);+ sdkMocks.enqueueJobs.mockResolvedValue({ enqueued: true });sdkMocks.getConnection.mockResolvedValue({id: 'connection-1',accessToken: 'token-1',@@ -62,10 +73,16 @@ describe('fathomDisconnectHandler', () => {]);});+ it('schedules the media cleanup for the disconnected account', async () => {+ expect(await fathomDisconnectHandler(payload)).toEqual({ success: true });+ expect(sdkMocks.enqueueJobs).toHaveBeenCalledWith(MEDIA_CLEANUP_JOB);+ });+it('releases the server claim when no webhook was ever registered', async () => {sdkMocks.kvGet.mockResolvedValue(null);expect(await fathomDisconnectHandler(payload)).toEqual({ success: true });+ expect(sdkMocks.enqueueJobs).toHaveBeenCalledWith(MEDIA_CLEANUP_JOB);expect(sdkMocks.getConnection).not.toHaveBeenCalled();expect(sdkMocks.deleteWebhook).not.toHaveBeenCalled();expect(sdkMocks.kvDelete.mock.calls).toEqual([@@ -87,6 +104,7 @@ describe('fathomDisconnectHandler', () => {await expect(fathomDisconnectHandler(payload)).rejects.toThrow('Fathom unavailable',);+ expect(sdkMocks.enqueueJobs).toHaveBeenCalledWith(MEDIA_CLEANUP_JOB);expect(sdkMocks.kvSet).toHaveBeenCalledWith('fathom-webhook:connection-1', {...registration,isActive: false,@@ -96,4 +114,14 @@ describe('fathomDisconnectHandler', () => {expect.stringContaining('leaked webhook webhook-1'),);});++ it('skips the webhook deletion when the media cleanup cannot be scheduled', async () => {+ sdkMocks.enqueueJobs.mockResolvedValue({ enqueued: false });++ await expect(fathomDisconnectHandler(payload)).rejects.toThrow(+ 'Failed to enqueue Fathom job',+ );+ expect(sdkMocks.deleteWebhook).not.toHaveBeenCalled();+ expect(sdkMocks.kvDelete).not.toHaveBeenCalled();+ });});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/fathom-disconnect.ts b/packages/twenty-apps/public/fathom/src/logic-functions/fathom-disconnect.tsindex 6e0aea0a48b..175eade02c9 100644--- a/packages/twenty-apps/public/fathom/src/logic-functions/fathom-disconnect.ts+++ b/packages/twenty-apps/public/fathom/src/logic-functions/fathom-disconnect.ts@@ -2,11 +2,15 @@ import { defineLogicFunction } from 'twenty-sdk/define';import { getConnection, kv } from 'twenty-sdk/logic-function';import { isDefined } from 'src/utils/is-defined';-import { FATHOM_DISCONNECT_UNIVERSAL_IDENTIFIER } from 'src/constants/universal-identifiers';+import {+ FATHOM_DISCONNECT_UNIVERSAL_IDENTIFIER,+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+} from 'src/constants/universal-identifiers';import { type FathomConnectionHookPayload } from 'src/logic-functions/types/fathom-connection-hook-payload.type';import { type FathomWebhookRegistration } from 'src/logic-functions/types/fathom-webhook-registration.type';import { createFathomClient } from 'src/logic-functions/utils/create-fathom-client.util';import { deleteStaleFathomWebhook } from 'src/logic-functions/utils/delete-stale-fathom-webhook.util';+import { enqueueFathomJobsOrThrow } from 'src/logic-functions/utils/enqueue-fathom-jobs-or-throw.util';import { getFathomConnectionClaimKey } from 'src/logic-functions/utils/get-fathom-connection-claim-key.util';import { getFathomWebhookRegistrationKey } from 'src/logic-functions/utils/get-fathom-webhook-registration-key.util';import { toErrorMessage } from 'src/logic-functions/utils/to-error-message.util';@@ -19,6 +23,14 @@ export const fathomDisconnectHandler = async ();const registration = await kv.get<FathomWebhookRegistration>(registrationKey);+ // Enqueued before the Fathom call because a webhook deletion failure throws+ // to retry the hook, and the media imports still have to be settled.+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ payloads: [{ disconnectedAccountId: payload.connectedAccountId }],+ });+if (!isDefined(registration)) {await kv.delete(getFathomConnectionClaimKey(payload.connectedAccountId), {scope: 'SERVER',@@ -59,7 +71,7 @@ export default defineLogicFunction({universalIdentifier: FATHOM_DISCONNECT_UNIVERSAL_IDENTIFIER,name: 'fathom-disconnect',description:- 'Deletes the registered Fathom webhook after a user removes their connection.',+ 'Deletes the registered Fathom webhook and schedules media cleanup after a user removes their Fathom connection.',timeoutSeconds: 30,handler: fathomDisconnectHandler,});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/fathom-reconcile-media-imports.ts b/packages/twenty-apps/public/fathom/src/logic-functions/fathom-reconcile-media-imports.tsnew file mode 100644index 00000000000..9894efcd148--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/fathom-reconcile-media-imports.ts@@ -0,0 +1,138 @@+import { isNonEmptyArray } from '@sniptt/guards';+import { CoreApiClient } from 'twenty-client-sdk/core';+import { defineLogicFunction } from 'twenty-sdk/define';+import { kv } from 'twenty-sdk/logic-function';+import { z } from 'zod';+import { isDefined } from 'src/utils/is-defined';++import {+ FATHOM_HEALING_WINDOW_DAYS,+ FATHOM_MEDIA_RECONCILIATION_CRON_PATTERN,+ FATHOM_MEDIA_RECONCILIATION_GRACE_PERIOD_MILLISECONDS,+ MILLISECONDS_PER_DAY,+} from 'src/constants/fathom.constant';+import { FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER } from 'src/constants/universal-identifiers';+import { type FathomMediaReconciliationRun } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { buildRetryableFathomError } from 'src/logic-functions/utils/build-retryable-fathom-error.util';+import { cleanupDisconnectedFathomMediaImports } from 'src/logic-functions/utils/cleanup-disconnected-fathom-media-imports.util';+import { enqueueFathomJobsOrThrow } from 'src/logic-functions/utils/enqueue-fathom-jobs-or-throw.util';+import { reconcileFathomMediaImports } from 'src/logic-functions/utils/reconcile-fathom-media-imports.util';++const FATHOM_MEDIA_RECONCILIATION_CHECKPOINT_KEY =+ 'fathom:media-reconciliation:checkpoint';++const fathomMediaReconciliationRunSchema = z.object({+ startedAfter: z.string().datetime(),+ staleBefore: z.string().datetime(),+ afterId: z.string().uuid().optional(),+});++const fathomMediaReconciliationPayloadSchema = z.object({+ disconnectedAccountId: z.string().uuid().optional(),+ recoveryRun: fathomMediaReconciliationRunSchema.optional(),+});++const buildFathomMediaReconciliationRun = (+ now: Date,+): FathomMediaReconciliationRun => ({+ startedAfter: new Date(+ now.getTime() - FATHOM_HEALING_WINDOW_DAYS * MILLISECONDS_PER_DAY,+ ).toISOString(),+ staleBefore: new Date(+ now.getTime() - FATHOM_MEDIA_RECONCILIATION_GRACE_PERIOD_MILLISECONDS,+ ).toISOString(),+});++export const fathomReconcileMediaImportsHandler = async (payload: unknown) => {+ const payloadParseResult = fathomMediaReconciliationPayloadSchema.safeParse(+ payload ?? {},+ );++ if (!payloadParseResult.success) {+ throw new Error('Fathom media reconciliation requires a valid payload');+ }++ if (+ isDefined(payloadParseResult.data.disconnectedAccountId) &&+ isDefined(payloadParseResult.data.recoveryRun)+ ) {+ throw new Error('Fathom media reconciliation payload is ambiguous');+ }++ try {+ const coreApiClient = new CoreApiClient({ runAs: 'application' });+ const { disconnectedAccountId } = payloadParseResult.data;++ if (isDefined(disconnectedAccountId)) {+ const result = await cleanupDisconnectedFathomMediaImports({+ coreApiClient,+ connectedAccountId: disconnectedAccountId,+ });++ if (result.shouldContinue) {+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ payloads: [{ disconnectedAccountId }],+ });+ }++ return result;+ }++ const storedRunParseResult = fathomMediaReconciliationRunSchema.safeParse(+ await kv.get<unknown>(FATHOM_MEDIA_RECONCILIATION_CHECKPOINT_KEY),+ );+ const run =+ payloadParseResult.data.recoveryRun ??+ (storedRunParseResult.success+ ? storedRunParseResult.data+ : buildFathomMediaReconciliationRun(new Date()));+ const result = await reconcileFathomMediaImports({ coreApiClient, run });++ if (isNonEmptyArray(result.disconnectedAccountIds)) {+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ payloads: result.disconnectedAccountIds.map((accountId) => ({+ disconnectedAccountId: accountId,+ })),+ });+ }++ if (result.hasNextPage && isDefined(result.nextAfterId)) {+ const nextRun = {+ ...run,+ afterId: result.nextAfterId,+ };++ await kv.set(FATHOM_MEDIA_RECONCILIATION_CHECKPOINT_KEY, nextRun);+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ payloads: [{ recoveryRun: nextRun }],+ });+ } else {+ await kv.delete(FATHOM_MEDIA_RECONCILIATION_CHECKPOINT_KEY);+ }++ return result;+ } catch (error) {+ throw buildRetryableFathomError({+ operation: 'media import reconciliation',+ error,+ });+ }+};++export default defineLogicFunction({+ universalIdentifier: FATHOM_RECONCILE_MEDIA_IMPORTS_UNIVERSAL_IDENTIFIER,+ name: 'fathom-reconcile-media-imports',+ description:+ 'Recovers unfinished local Fathom media imports and settles imports for disconnected accounts.',+ timeoutSeconds: 300,+ handler: fathomReconcileMediaImportsHandler,+ cronTriggerSettings: {+ pattern: FATHOM_MEDIA_RECONCILIATION_CRON_PATTERN,+ },+});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/schemas/fathom-media-reconciliation-query-result.schema.ts b/packages/twenty-apps/public/fathom/src/logic-functions/schemas/fathom-media-reconciliation-query-result.schema.tsnew file mode 100644index 00000000000..fc0039e89fd--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/schemas/fathom-media-reconciliation-query-result.schema.ts@@ -0,0 +1,22 @@+import { z } from 'zod';++import { callRecordingMediaStateNodeSchema } from 'src/logic-functions/schemas/call-recording-media-state-query-result.schema';++export const fathomMediaReconciliationQueryResultSchema = z.object({+ callRecordings: z+ .object({+ pageInfo: z.object({+ hasNextPage: z.boolean(),+ endCursor: z.string().nullable().optional(),+ }),+ edges: z.array(+ z.object({+ node: callRecordingMediaStateNodeSchema.extend({+ status: z.string(),+ }),+ }),+ ),+ })+ .nullable()+ .optional(),+});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-media-reconciliation-plan.type.ts b/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-media-reconciliation-plan.type.tsnew file mode 100644index 00000000000..ba1c3e0d1a6--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-media-reconciliation-plan.type.ts@@ -0,0 +1,38 @@+import { type CallRecordingMediaState } from 'src/logic-functions/types/call-recording-media-state.type';+import { type FathomRecordingImportReference } from 'src/logic-functions/types/fathom-recording-import-reference.type';++export type FathomMediaReconciliationPlan = {+ callRecordingsToComplete: FathomRecordingImportReference[];+ importGroups: Array<{+ connectedAccountId: string;+ callRecordingIdsToRequest: string[];+ downloadsToPoll: Array<{+ callRecordingId: string;+ downloadId: string;+ }>;+ }>;+ disconnectedAccountIds: string[];+};++export type FathomMediaReconciliationCandidate = CallRecordingMediaState & {+ status: string;+};++export type FathomMediaReconciliationRun = {+ startedAfter: string;+ staleBefore: string;+ afterId?: string;+};++export type FathomMediaReconciliationPage = {+ callRecordings: FathomMediaReconciliationCandidate[];+ hasNextPage: boolean;+};++export type DisconnectedFathomMediaReconciliationPlan = {+ callRecordingsToSettle: FathomRecordingImportReference[];+ callRecordingsToSettleAndComplete: FathomRecordingImportReference[];+ callRecordingsToSettleAndFail: FathomRecordingImportReference[];+ callRecordingsToComplete: FathomRecordingImportReference[];+ callRecordingsToFail: FathomRecordingImportReference[];+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-recording-import-reference.type.ts b/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-recording-import-reference.type.tsnew file mode 100644index 00000000000..076ebeb4b43--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/types/fathom-recording-import-reference.type.ts@@ -0,0 +1,6 @@+export type FathomRecordingImportReference = {+ callRecordingId: string;+ callRecordingUpdatedAt: string;+ fathomRecordingImportId: string;+ fathomRecordingImportUpdatedAt: string;+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-disconnected-fathom-media-reconciliation-plan.test.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-disconnected-fathom-media-reconciliation-plan.test.tsnew file mode 100644index 00000000000..1f11c844b96--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-disconnected-fathom-media-reconciliation-plan.test.ts@@ -0,0 +1,68 @@+import { describe, expect, it } from 'vitest';++import { type FathomMediaReconciliationCandidate } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { buildDisconnectedFathomMediaReconciliationPlan } from 'src/logic-functions/utils/build-disconnected-fathom-media-reconciliation-plan.util';++const buildCallRecording = (+ overrides: Partial<FathomMediaReconciliationCandidate> = {},+): FathomMediaReconciliationCandidate => ({+ id: 'call-recording-id',+ fathomRecordingImportId: 'fathom-recording-import-id',+ fathomRecordingImportUpdatedAt: '2026-09-04T12:01:00.000Z',+ recordingId: '42',+ hasVideo: false,+ hasAudio: false,+ hasTranscript: true,+ hasSummary: true,+ failureReason: undefined,+ connectedAccountId: 'connected-account-id',+ downloadId: undefined,+ uploadCheckpoint: undefined,+ status: 'PROCESSING',+ updatedAt: '2026-09-04T12:00:00.000Z',+ ...overrides,+});++const buildReference = (callRecordingId: string) => ({+ callRecordingId,+ callRecordingUpdatedAt: '2026-09-04T12:00:00.000Z',+ fathomRecordingImportId: 'fathom-recording-import-id',+ fathomRecordingImportUpdatedAt: '2026-09-04T12:01:00.000Z',+});++describe('buildDisconnectedFathomMediaReconciliationPlan', () => {+ it('settles missing media and completes imports only when a transcript exists', () => {+ const plan = buildDisconnectedFathomMediaReconciliationPlan([+ buildCallRecording({ id: 'ready-to-settle-and-complete' }),+ buildCallRecording({+ id: 'ready-to-settle',+ status: 'COMPLETED',+ }),+ buildCallRecording({+ id: 'ready-to-complete',+ hasVideo: true,+ }),+ buildCallRecording({+ id: 'processing-without-transcript',+ hasTranscript: false,+ hasAudio: true,+ }),+ buildCallRecording({+ id: 'missing-media-without-transcript',+ hasTranscript: false,+ }),+ ]);++ expect(plan).toEqual({+ callRecordingsToSettle: [buildReference('ready-to-settle')],+ callRecordingsToSettleAndComplete: [+ buildReference('ready-to-settle-and-complete'),+ ],+ callRecordingsToSettleAndFail: [+ buildReference('missing-media-without-transcript'),+ ],+ callRecordingsToComplete: [buildReference('ready-to-complete')],+ callRecordingsToFail: [buildReference('processing-without-transcript')],+ });+ });+});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-fathom-media-reconciliation-plan.test.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-fathom-media-reconciliation-plan.test.tsnew file mode 100644index 00000000000..5b7a25f13b5--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/build-fathom-media-reconciliation-plan.test.ts@@ -0,0 +1,132 @@+import { describe, expect, it } from 'vitest';++import { buildFathomMediaReconciliationPlan } from 'src/logic-functions/utils/build-fathom-media-reconciliation-plan.util';+import { type FathomMediaReconciliationCandidate } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';++const buildCallRecording = (+ overrides: Partial<FathomMediaReconciliationCandidate> = {},+): FathomMediaReconciliationCandidate => ({+ id: 'call-recording-id',+ fathomRecordingImportId: 'fathom-recording-import-id',+ fathomRecordingImportUpdatedAt: '2026-09-04T12:01:00.000Z',+ recordingId: '42',+ hasVideo: false,+ hasAudio: false,+ hasTranscript: true,+ hasSummary: true,+ failureReason: undefined,+ connectedAccountId: 'connected-account-id',+ downloadId: undefined,+ uploadCheckpoint: undefined,+ status: 'PROCESSING',+ updatedAt: '2026-09-04T12:00:00.000Z',+ ...overrides,+});++const buildReference = (callRecordingId: string) => ({+ callRecordingId,+ callRecordingUpdatedAt: '2026-09-04T12:00:00.000Z',+ fathomRecordingImportId: 'fathom-recording-import-id',+ fathomRecordingImportUpdatedAt: '2026-09-04T12:01:00.000Z',+});++describe('buildFathomMediaReconciliationPlan', () => {+ it('completes settled recordings and groups missing imports by account', () => {+ const plan = buildFathomMediaReconciliationPlan({+ callRecordings: [+ buildCallRecording({ id: 'video-ready', hasVideo: true }),+ buildCallRecording({+ id: 'failure-settled',+ failureReason: 'download_forbidden',+ }),+ buildCallRecording({ id: 'missing-media-1' }),+ buildCallRecording({ id: 'missing-media-2' }),+ buildCallRecording({+ id: 'download-ready-to-poll',+ downloadId: 'download-id',+ }),+ buildCallRecording({+ id: 'another-account',+ connectedAccountId: 'another-connected-account-id',+ }),+ ],+ activeConnectedAccountIds: [+ 'connected-account-id',+ 'another-connected-account-id',+ ],+ });++ expect(plan).toEqual({+ callRecordingsToComplete: [+ buildReference('video-ready'),+ buildReference('failure-settled'),+ ],+ importGroups: [+ {+ connectedAccountId: 'connected-account-id',+ callRecordingIdsToRequest: ['missing-media-1', 'missing-media-2'],+ downloadsToPoll: [+ {+ callRecordingId: 'download-ready-to-poll',+ downloadId: 'download-id',+ },+ ],+ },+ {+ connectedAccountId: 'another-connected-account-id',+ callRecordingIdsToRequest: ['another-account'],+ downloadsToPoll: [],+ },+ ],+ disconnectedAccountIds: [],+ });+ });++ it('does not retry settled media while its transcript is still absent', () => {+ const plan = buildFathomMediaReconciliationPlan({+ callRecordings: [+ buildCallRecording({ hasTranscript: false, hasAudio: true }),+ ],+ activeConnectedAccountIds: ['connected-account-id'],+ });++ expect(plan).toEqual({+ callRecordingsToComplete: [],+ importGroups: [],+ disconnectedAccountIds: [],+ });+ });++ it('repairs missing media on a legacy completed recording', () => {+ const plan = buildFathomMediaReconciliationPlan({+ callRecordings: [+ buildCallRecording({ status: 'COMPLETED', hasTranscript: false }),+ ],+ activeConnectedAccountIds: ['connected-account-id'],+ });++ expect(plan.importGroups).toEqual([+ {+ connectedAccountId: 'connected-account-id',+ callRecordingIdsToRequest: ['call-recording-id'],+ downloadsToPoll: [],+ },+ ]);+ });++ it('routes unfinished imports for disconnected accounts to cleanup', () => {+ const plan = buildFathomMediaReconciliationPlan({+ callRecordings: [+ buildCallRecording(),+ buildCallRecording({ id: 'settled-recording', hasVideo: true }),+ ],+ activeConnectedAccountIds: [],+ });++ expect(plan).toEqual({+ callRecordingsToComplete: [],+ importGroups: [],+ disconnectedAccountIds: ['connected-account-id'],+ });+ });+});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/parse-fathom-media-reconciliation-page.test.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/parse-fathom-media-reconciliation-page.test.tsnew file mode 100644index 00000000000..6934e9b946e--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/__tests__/parse-fathom-media-reconciliation-page.test.ts@@ -0,0 +1,102 @@+import { describe, expect, it } from 'vitest';++import { parseFathomMediaReconciliationPage } from 'src/logic-functions/utils/parse-fathom-media-reconciliation-page.util';++describe('parseFathomMediaReconciliationPage', () => {+ it.each([{}, { callRecordings: null }])(+ 'returns an empty terminal page for an absent connection: %j',+ (queryResult) => {+ expect(parseFathomMediaReconciliationPage(queryResult)).toEqual({+ callRecordings: [],+ hasNextPage: false,+ });+ },+ );++ it.each([false, true])(+ 'preserves the continuation flag %s and normalizes reconciliation state',+ (hasNextPage) => {+ expect(+ parseFathomMediaReconciliationPage({+ callRecordings: {+ pageInfo: { hasNextPage },+ edges: [+ {+ node: {+ id: 'call-recording-id',+ updatedAt: '2026-09-05T00:00:00.000Z',+ status: 'PROCESSING',+ fathomRecordingImports: {+ edges: [+ {+ node: {+ id: 'fathom-recording-import-id',+ updatedAt: '2026-09-05T00:01:00.000Z',+ recordingId: '42',+ connectedAccountId: 'connected-account-id',+ mediaDownloadId: 'download-id',+ mediaFailureReason: '',+ mediaUploadCheckpoint: {+ downloadId: 'download-id',+ fileId: 'audio-file-id',+ kind: 'audio',+ },+ },+ },+ ],+ },+ video: [],+ audio: [{ fileId: 'audio-file-id' }],+ transcript: [],+ },+ },+ ],+ },+ }),+ ).toEqual({+ callRecordings: [+ {+ id: 'call-recording-id',+ updatedAt: '2026-09-05T00:00:00.000Z',+ status: 'PROCESSING',+ fathomRecordingImportId: 'fathom-recording-import-id',+ fathomRecordingImportUpdatedAt: '2026-09-05T00:01:00.000Z',+ recordingId: '42',+ connectedAccountId: 'connected-account-id',+ downloadId: 'download-id',+ failureReason: undefined,+ hasVideo: false,+ hasAudio: true,+ hasTranscript: false,+ hasSummary: false,+ uploadCheckpoint: {+ downloadId: 'download-id',+ fileId: 'audio-file-id',+ kind: 'audio',+ },+ },+ ],+ hasNextPage,+ });+ },+ );++ it('rejects malformed pages instead of treating them as completed runs', () => {+ expect(() =>+ parseFathomMediaReconciliationPage({+ callRecordings: { edges: [], pageInfo: {} },+ }),+ ).toThrow();+ });++ it('rejects a recording without its optimistic concurrency timestamp', () => {+ expect(() =>+ parseFathomMediaReconciliationPage({+ callRecordings: {+ pageInfo: { hasNextPage: false },+ edges: [{ node: { id: 'call-recording-id', status: 'PROCESSING' } }],+ },+ }),+ ).toThrow();+ });+});diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-disconnected-fathom-media-reconciliation-plan.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-disconnected-fathom-media-reconciliation-plan.util.tsnew file mode 100644index 00000000000..693577683f3--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-disconnected-fathom-media-reconciliation-plan.util.ts@@ -0,0 +1,59 @@+import { isDefined } from 'src/utils/is-defined';++import {+ type DisconnectedFathomMediaReconciliationPlan,+ type FathomMediaReconciliationCandidate,+} from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { buildFathomRecordingImportReference } from 'src/logic-functions/utils/build-fathom-recording-import-reference.util';+import { isFathomMediaSettled } from 'src/logic-functions/utils/is-fathom-media-settled.util';++export const buildDisconnectedFathomMediaReconciliationPlan = (+ callRecordings: FathomMediaReconciliationCandidate[],+): DisconnectedFathomMediaReconciliationPlan => {+ const plan: DisconnectedFathomMediaReconciliationPlan = {+ callRecordingsToSettle: [],+ callRecordingsToSettleAndComplete: [],+ callRecordingsToSettleAndFail: [],+ callRecordingsToComplete: [],+ callRecordingsToFail: [],+ };++ for (const callRecording of callRecordings) {+ const reference = buildFathomRecordingImportReference(callRecording);++ if (!isDefined(reference)) {+ continue;+ }++ const isSettled = isFathomMediaSettled(callRecording);+ const isProcessing = callRecording.status === 'PROCESSING';++ if (!isProcessing && !isSettled) {+ plan.callRecordingsToSettle.push(reference);+ continue;+ }++ if (!isProcessing) {+ continue;+ }++ if (!isSettled && callRecording.hasTranscript) {+ plan.callRecordingsToSettleAndComplete.push(reference);+ continue;+ }++ if (!isSettled) {+ plan.callRecordingsToSettleAndFail.push(reference);+ continue;+ }++ if (callRecording.hasTranscript) {+ plan.callRecordingsToComplete.push(reference);+ continue;+ }++ plan.callRecordingsToFail.push(reference);+ }++ return plan;+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-media-reconciliation-plan.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-media-reconciliation-plan.util.tsnew file mode 100644index 00000000000..3a629ec0f98--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-media-reconciliation-plan.util.ts@@ -0,0 +1,88 @@+import { isNonEmptyString } from '@sniptt/guards';+import { isDefined } from 'src/utils/is-defined';++import { type FathomRecordingImportReference } from 'src/logic-functions/types/fathom-recording-import-reference.type';+import {+ type FathomMediaReconciliationCandidate,+ type FathomMediaReconciliationPlan,+} from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { buildFathomRecordingImportReference } from 'src/logic-functions/utils/build-fathom-recording-import-reference.util';+import { isFathomCallRecordingImportComplete } from 'src/logic-functions/utils/is-fathom-call-recording-import-complete.util';+import { isFathomMediaSettled } from 'src/logic-functions/utils/is-fathom-media-settled.util';++export const buildFathomMediaReconciliationPlan = ({+ callRecordings,+ activeConnectedAccountIds,+}: {+ callRecordings: FathomMediaReconciliationCandidate[];+ activeConnectedAccountIds: string[];+}): FathomMediaReconciliationPlan => {+ const callRecordingsToComplete: FathomRecordingImportReference[] = [];+ const activeConnectedAccountIdSet = new Set(activeConnectedAccountIds);+ const disconnectedAccountIdSet = new Set<string>();+ const importGroupsByConnectedAccountId = new Map<+ string,+ Omit<+ FathomMediaReconciliationPlan['importGroups'][number],+ 'connectedAccountId'+ >+ >();++ for (const callRecording of callRecordings) {+ if (!isNonEmptyString(callRecording.connectedAccountId)) {+ continue;+ }++ const reference = buildFathomRecordingImportReference(callRecording);++ if (!isDefined(reference)) {+ continue;+ }++ if (!activeConnectedAccountIdSet.has(callRecording.connectedAccountId)) {+ disconnectedAccountIdSet.add(callRecording.connectedAccountId);+ continue;+ }++ if (+ callRecording.status === 'PROCESSING' &&+ isFathomCallRecordingImportComplete(callRecording)+ ) {+ callRecordingsToComplete.push(reference);+ continue;+ }++ if (isFathomMediaSettled(callRecording)) {+ continue;+ }++ const existingImportGroup = importGroupsByConnectedAccountId.get(+ callRecording.connectedAccountId,+ ) ?? { callRecordingIdsToRequest: [], downloadsToPoll: [] };++ if (isNonEmptyString(callRecording.downloadId)) {+ existingImportGroup.downloadsToPoll.push({+ callRecordingId: callRecording.id,+ downloadId: callRecording.downloadId,+ });+ } else {+ existingImportGroup.callRecordingIdsToRequest.push(callRecording.id);+ }++ importGroupsByConnectedAccountId.set(+ callRecording.connectedAccountId,+ existingImportGroup,+ );+ }++ return {+ callRecordingsToComplete,+ importGroups: [...importGroupsByConnectedAccountId].map(+ ([connectedAccountId, importGroup]) => ({+ connectedAccountId,+ ...importGroup,+ }),+ ),+ disconnectedAccountIds: [...disconnectedAccountIdSet],+ };+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-recording-import-reference.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-recording-import-reference.util.tsnew file mode 100644index 00000000000..c156ea1dae2--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/build-fathom-recording-import-reference.util.ts@@ -0,0 +1,29 @@+import { isNonEmptyString } from '@sniptt/guards';++import { type FathomMediaReconciliationCandidate } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { type FathomRecordingImportReference } from 'src/logic-functions/types/fathom-recording-import-reference.type';++export const buildFathomRecordingImportReference = (+ callRecording: Pick<+ FathomMediaReconciliationCandidate,+ | 'id'+ | 'updatedAt'+ | 'fathomRecordingImportId'+ | 'fathomRecordingImportUpdatedAt'+ >,+): FathomRecordingImportReference | undefined => {+ if (+ !isNonEmptyString(callRecording.fathomRecordingImportId) ||+ !isNonEmptyString(callRecording.fathomRecordingImportUpdatedAt)+ ) {+ return undefined;+ }++ return {+ callRecordingId: callRecording.id,+ callRecordingUpdatedAt: callRecording.updatedAt,+ fathomRecordingImportId: callRecording.fathomRecordingImportId,+ fathomRecordingImportUpdatedAt:+ callRecording.fathomRecordingImportUpdatedAt,+ };+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/cleanup-disconnected-fathom-media-imports.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/cleanup-disconnected-fathom-media-imports.util.tsnew file mode 100644index 00000000000..1491094167d--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/cleanup-disconnected-fathom-media-imports.util.ts@@ -0,0 +1,146 @@+import { isNonEmptyArray } from '@sniptt/guards';+import { type CoreApiClient } from 'twenty-client-sdk/core';+import { isDefined } from 'src/utils/is-defined';++import { FATHOM_MEDIA_FAILURE_REASON } from 'src/constants/fathom-media-failure-reason.constant';+import { buildDisconnectedFathomMediaReconciliationPlan } from 'src/logic-functions/utils/build-disconnected-fathom-media-reconciliation-plan.util';+import { findDisconnectedFathomMediaImports } from 'src/logic-functions/utils/find-disconnected-fathom-media-imports.util';+import { listFathomConnections } from 'src/logic-functions/utils/list-fathom-connections.util';++export const cleanupDisconnectedFathomMediaImports = async ({+ coreApiClient,+ connectedAccountId,+}: {+ coreApiClient: Pick<CoreApiClient, 'query' | 'mutation'>;+ connectedAccountId: string;+}) => {+ const connections = await listFathomConnections();++ if (connections.some(({ id }) => id === connectedAccountId)) {+ return {+ candidateCount: 0,+ updatedRecordingCount: 0,+ shouldContinue: false,+ skipped: true,+ };+ }++ const page = await findDisconnectedFathomMediaImports({+ coreApiClient,+ connectedAccountId,+ });+ const plan = buildDisconnectedFathomMediaReconciliationPlan(+ page.callRecordings,+ );+ let updatedRecordingCount = 0;+ const clearedImportFields = {+ mediaDownloadId: null,+ mediaImportClaimedAt: null,+ mediaUploadCheckpoint: null,+ };+ const settledData = {+ ...clearedImportFields,+ mediaFailureReason:+ FATHOM_MEDIA_FAILURE_REASON.CONNECTED_ACCOUNT_UNAVAILABLE,+ };+ const updates = [+ {+ callRecordings: plan.callRecordingsToSettleAndComplete,+ importData: settledData,+ status: 'COMPLETED',+ },+ {+ callRecordings: plan.callRecordingsToSettleAndFail,+ importData: settledData,+ status: 'FAILED',+ },+ {+ callRecordings: plan.callRecordingsToSettle,+ importData: settledData,+ },+ {+ callRecordings: plan.callRecordingsToComplete,+ importData: clearedImportFields,+ status: 'COMPLETED',+ },+ {+ callRecordings: plan.callRecordingsToFail,+ importData: clearedImportFields,+ status: 'FAILED',+ },+ ];++ for (const { callRecordings, importData, status } of updates) {+ if (!isNonEmptyArray(callRecordings)) {+ continue;+ }++ const importResult = await coreApiClient.mutation({+ updateFathomRecordingImports: {+ __args: {+ filter: {+ connectedAccountId: { eq: connectedAccountId },+ or: callRecordings.map((reference) => ({+ id: { eq: reference.fathomRecordingImportId },+ updatedAt: {+ eq: reference.fathomRecordingImportUpdatedAt,+ },+ })),+ },+ data: importData,+ },+ id: true,+ },+ });+ const updatedImportIds = new Set(+ (importResult.updateFathomRecordingImports ?? []).map(+ ({ id }: { id: string }) => id,+ ),+ );++ if (!isDefined(status)) {+ updatedRecordingCount += updatedImportIds.size;+ continue;+ }++ const callRecordingsWithUpdatedImports = callRecordings.filter(+ ({ fathomRecordingImportId }) =>+ updatedImportIds.has(fathomRecordingImportId),+ );++ if (!isNonEmptyArray(callRecordingsWithUpdatedImports)) {+ continue;+ }++ const callRecordingResult = await coreApiClient.mutation({+ updateCallRecordings: {+ __args: {+ filter: {+ status: { eq: 'PROCESSING' },+ or: callRecordingsWithUpdatedImports.map((reference) => ({+ id: { eq: reference.callRecordingId },+ updatedAt: { eq: reference.callRecordingUpdatedAt },+ fathomRecordingImports: {+ id: { eq: reference.fathomRecordingImportId },+ connectedAccountId: { eq: connectedAccountId },+ },+ })),+ },+ data: { status },+ },+ id: true,+ },+ });++ updatedRecordingCount +=+ callRecordingResult.updateCallRecordings?.length ?? 0;+ }++ return {+ candidateCount: page.callRecordings.length,+ updatedRecordingCount,+ shouldContinue:+ page.hasNextPage || updatedRecordingCount < page.callRecordings.length,+ skipped: false,+ };+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/complete-fathom-call-recording-imports.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/complete-fathom-call-recording-imports.util.tsnew file mode 100644index 00000000000..089dfc36530--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/complete-fathom-call-recording-imports.util.ts@@ -0,0 +1,48 @@+import { type CoreApiClient } from 'twenty-client-sdk/core';++import { type FathomRecordingImportReference } from 'src/logic-functions/types/fathom-recording-import-reference.type';+import { chunkIntoBatches } from 'src/utils/chunk-into-batches.util';++const CALL_RECORDING_COMPLETION_BATCH_SIZE = 200;++export const completeFathomCallRecordingImports = async ({+ coreApiClient,+ callRecordings,+}: {+ coreApiClient: Pick<CoreApiClient, 'mutation'>;+ callRecordings: FathomRecordingImportReference[];+}): Promise<number> => {+ let completedCallRecordingCount = 0;++ for (const callRecordingBatch of chunkIntoBatches(+ callRecordings,+ CALL_RECORDING_COMPLETION_BATCH_SIZE,+ )) {+ const mutationResult = await coreApiClient.mutation({+ updateCallRecordings: {+ __args: {+ filter: {+ status: { eq: 'PROCESSING' },+ or: callRecordingBatch.map((reference) => ({+ id: { eq: reference.callRecordingId },+ updatedAt: { eq: reference.callRecordingUpdatedAt },+ fathomRecordingImports: {+ id: { eq: reference.fathomRecordingImportId },+ updatedAt: {+ eq: reference.fathomRecordingImportUpdatedAt,+ },+ },+ })),+ },+ data: { status: 'COMPLETED' },+ },+ id: true,+ },+ });++ completedCallRecordingCount +=+ mutationResult.updateCallRecordings?.length ?? 0;+ }++ return completedCallRecordingCount;+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-disconnected-fathom-media-imports.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-disconnected-fathom-media-imports.util.tsnew file mode 100644index 00000000000..109fd798790--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-disconnected-fathom-media-imports.util.ts@@ -0,0 +1,40 @@+import { type CoreApiClient } from 'twenty-client-sdk/core';++import { FATHOM_MEDIA_RECONCILIATION_SELECTION } from 'src/constants/fathom-media-reconciliation-selection.constant';+import { FATHOM_MEDIA_RECONCILIATION_PAGE_SIZE } from 'src/constants/fathom.constant';+import { type FathomMediaReconciliationPage } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { parseFathomMediaReconciliationPage } from 'src/logic-functions/utils/parse-fathom-media-reconciliation-page.util';++export const findDisconnectedFathomMediaImports = async ({+ coreApiClient,+ connectedAccountId,+}: {+ coreApiClient: Pick<CoreApiClient, 'query'>;+ connectedAccountId: string;+}): Promise<FathomMediaReconciliationPage> => {+ const queryResult = await coreApiClient.query({+ callRecordings: {+ __args: {+ filter: {+ fathomRecordingImports: {+ connectedAccountId: { eq: connectedAccountId },+ },+ or: [+ {+ fathomRecordingImports: {+ mediaFailureReason: { is: 'NULL' },+ },+ video: { is: 'NULL' },+ audio: { is: 'NULL' },+ },+ { status: { eq: 'PROCESSING' } },+ ],+ },+ first: FATHOM_MEDIA_RECONCILIATION_PAGE_SIZE,+ orderBy: [{ id: 'AscNullsLast' }],+ },+ ...FATHOM_MEDIA_RECONCILIATION_SELECTION,+ },+ });+ return parseFathomMediaReconciliationPage(queryResult);+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-stale-fathom-media-imports.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-stale-fathom-media-imports.util.tsnew file mode 100644index 00000000000..f01c79ddcef--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/find-stale-fathom-media-imports.util.ts@@ -0,0 +1,88 @@+import { isNonEmptyString } from '@sniptt/guards';+import { type CoreApiClient } from 'twenty-client-sdk/core';++import { FATHOM_MEDIA_RECONCILIATION_SELECTION } from 'src/constants/fathom-media-reconciliation-selection.constant';+import { FATHOM_MEDIA_RECONCILIATION_PAGE_SIZE } from 'src/constants/fathom.constant';+import {+ type FathomMediaReconciliationPage,+ type FathomMediaReconciliationRun,+} from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { parseFathomMediaReconciliationPage } from 'src/logic-functions/utils/parse-fathom-media-reconciliation-page.util';++export const findStaleFathomMediaImports = async ({+ coreApiClient,+ run,+ activeConnectedAccountIds,+}: {+ coreApiClient: Pick<CoreApiClient, 'query'>;+ run: FathomMediaReconciliationRun;+ activeConnectedAccountIds: string[];+}): Promise<FathomMediaReconciliationPage> => {+ const queryResult = await coreApiClient.query({+ callRecordings: {+ __args: {+ filter: {+ and: [+ {+ fathomRecordingImports: {+ connectedAccountId: { is: 'NOT_NULL' },+ updatedAt: { lte: run.staleBefore },+ },+ },+ {+ or: [+ { startedAt: { gte: run.startedAfter } },+ {+ fathomRecordingImports: {+ connectedAccountId: {+ notIn: activeConnectedAccountIds,+ },+ },+ },+ ],+ },+ ...(isNonEmptyString(run.afterId)+ ? [{ id: { gt: run.afterId } }]+ : []),+ {+ or: [+ {+ fathomRecordingImports: {+ mediaFailureReason: { is: 'NULL' },+ },+ video: { is: 'NULL' },+ audio: { is: 'NULL' },+ },+ {+ status: { eq: 'PROCESSING' },+ transcript: { is: 'NOT_NULL' },+ or: [+ { video: { is: 'NOT_NULL' } },+ { audio: { is: 'NOT_NULL' } },+ {+ fathomRecordingImports: {+ mediaFailureReason: { is: 'NOT_NULL' },+ },+ },+ ],+ },+ {+ status: { eq: 'PROCESSING' },+ fathomRecordingImports: {+ connectedAccountId: {+ notIn: activeConnectedAccountIds,+ },+ },+ },+ ],+ },+ ],+ },+ first: FATHOM_MEDIA_RECONCILIATION_PAGE_SIZE,+ orderBy: [{ id: 'AscNullsLast' }],+ },+ ...FATHOM_MEDIA_RECONCILIATION_SELECTION,+ },+ });+ return parseFathomMediaReconciliationPage(queryResult);+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/parse-fathom-media-reconciliation-page.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/parse-fathom-media-reconciliation-page.util.tsnew file mode 100644index 00000000000..b5963c2e22f--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/parse-fathom-media-reconciliation-page.util.ts@@ -0,0 +1,25 @@+import { fathomMediaReconciliationQueryResultSchema } from 'src/logic-functions/schemas/fathom-media-reconciliation-query-result.schema';+import { type FathomMediaReconciliationPage } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { mapCallRecordingMediaState } from 'src/logic-functions/utils/map-call-recording-media-state.util';+import { isDefined } from 'src/utils/is-defined';++export const parseFathomMediaReconciliationPage = (+ queryResult: unknown,+): FathomMediaReconciliationPage => {+ const connection =+ fathomMediaReconciliationQueryResultSchema.parse(+ queryResult,+ ).callRecordings;++ if (!isDefined(connection)) {+ return { callRecordings: [], hasNextPage: false };+ }++ return {+ callRecordings: connection.edges.map(({ node }) => ({+ ...mapCallRecordingMediaState(node),+ status: node.status,+ })),+ hasNextPage: connection.pageInfo.hasNextPage,+ };+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/reconcile-fathom-media-imports.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/reconcile-fathom-media-imports.util.tsnew file mode 100644index 00000000000..adb3e681ec3--- /dev/null+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/reconcile-fathom-media-imports.util.ts@@ -0,0 +1,101 @@+import { isNonEmptyString } from '@sniptt/guards';+import { type CoreApiClient } from 'twenty-client-sdk/core';++import { FATHOM_MEDIA_RECONCILIATION_BATCH_SIZE } from 'src/constants/fathom.constant';+import {+ FATHOM_IMPORT_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER,+ FATHOM_REQUEST_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER,+} from 'src/constants/universal-identifiers';+import { type FathomMediaReconciliationRun } from 'src/logic-functions/types/fathom-media-reconciliation-plan.type';+import { buildFathomMediaReconciliationPlan } from 'src/logic-functions/utils/build-fathom-media-reconciliation-plan.util';+import { completeFathomCallRecordingImports } from 'src/logic-functions/utils/complete-fathom-call-recording-imports.util';+import { enqueueFathomJobsOrThrow } from 'src/logic-functions/utils/enqueue-fathom-jobs-or-throw.util';+import { findStaleFathomMediaImports } from 'src/logic-functions/utils/find-stale-fathom-media-imports.util';+import { listFathomConnections } from 'src/logic-functions/utils/list-fathom-connections.util';+import { reserveFathomImportSlots } from 'src/logic-functions/utils/reserve-fathom-import-slots.util';+import { chunkIntoBatches } from 'src/utils/chunk-into-batches.util';++export const reconcileFathomMediaImports = async ({+ coreApiClient,+ run,+}: {+ coreApiClient: Pick<CoreApiClient, 'query' | 'mutation'>;+ run: FathomMediaReconciliationRun;+}) => {+ const connections = await listFathomConnections();+ const activeConnectedAccountIds = connections.map(({ id }) => id);++ const reconciliationPage = await findStaleFathomMediaImports({+ coreApiClient,+ run,+ activeConnectedAccountIds,+ });+ const reconciliationPlan = buildFathomMediaReconciliationPlan({+ callRecordings: reconciliationPage.callRecordings,+ activeConnectedAccountIds,+ });+ const completedCallRecordingCount = await completeFathomCallRecordingImports({+ coreApiClient,+ callRecordings: reconciliationPlan.callRecordingsToComplete,+ });+ let enqueuedCallRecordingCount = 0;++ for (const importGroup of reconciliationPlan.importGroups) {+ const requestBatches = chunkIntoBatches(+ importGroup.callRecordingIdsToRequest,+ FATHOM_MEDIA_RECONCILIATION_BATCH_SIZE,+ );+ const pollBatches = chunkIntoBatches(+ importGroup.downloadsToPoll,+ FATHOM_MEDIA_RECONCILIATION_BATCH_SIZE,+ );+ const { slotDelays } = await reserveFathomImportSlots({+ connectedAccountId: importGroup.connectedAccountId,+ slotCount: requestBatches.length + pollBatches.length,+ });+ let slotIndex = 0;++ for (const callRecordingIds of requestBatches) {+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_REQUEST_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER,+ payloads: callRecordingIds.map((callRecordingId) => ({+ callRecordingId,+ })),+ delayMs: slotDelays[slotIndex],+ });+ enqueuedCallRecordingCount += callRecordingIds.length;+ slotIndex += 1;+ }++ for (const downloads of pollBatches) {+ await enqueueFathomJobsOrThrow({+ logicFunctionUniversalIdentifier:+ FATHOM_IMPORT_MEDIA_DOWNLOAD_UNIVERSAL_IDENTIFIER,+ payloads: downloads.map(({ callRecordingId, downloadId }) => ({+ callRecordingId,+ downloadId,+ attempt: 0,+ })),+ delayMs: slotDelays[slotIndex],+ });+ enqueuedCallRecordingCount += downloads.length;+ slotIndex += 1;+ }+ }++ const nextAfterId =+ reconciliationPage.callRecordings[+ reconciliationPage.callRecordings.length - 1+ ]?.id;++ return {+ candidateCount: reconciliationPage.callRecordings.length,+ completedCallRecordingCount,+ enqueuedCallRecordingCount,+ disconnectedAccountIds: reconciliationPlan.disconnectedAccountIds,+ hasNextPage:+ reconciliationPage.hasNextPage && isNonEmptyString(nextAfterId),+ nextAfterId,+ };+};diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/run-fathom-media-import.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/run-fathom-media-import.util.tsindex fbaa3dfcf42..1a9a3da40a3 100644--- a/packages/twenty-apps/public/fathom/src/logic-functions/utils/run-fathom-media-import.util.ts+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/run-fathom-media-import.util.ts@@ -178,22 +178,23 @@ export const runFathomMediaImport = async (rawPayload: unknown) => {} catch (error) {const retryDelay = getFathomRetryAfterDelay({ error, now: new Date() });- if (- isDefined(retryDelay) &&- (payload.rateLimitAttempt ?? 0) < MAX_RATE_LIMIT_RETRIES- ) {+ const rateLimitAttempt = payload.rateLimitAttempt ?? 0;++ if (isDefined(retryDelay)) {+ if (rateLimitAttempt >= MAX_RATE_LIMIT_RETRIES) {+ return { success: true, outcome: 'deferred' };+ }+await enqueueNextAttempt({downloadId: writeContext.downloadId ?? undefined,- rateLimitAttempt: (payload.rateLimitAttempt ?? 0) + 1,+ rateLimitAttempt: rateLimitAttempt + 1,delay: retryDelay,});return { success: true, outcome: 'rate-limited' };}- const failureReason = isDefined(retryDelay)- ? 'rate_limit_exhausted'- : getFathomMediaFailureReasonForError(error);+ const failureReason = getFathomMediaFailureReasonForError(error);if (!isDefined(failureReason)) {throw error;diff --git a/packages/twenty-apps/public/fathom/src/logic-functions/utils/update-fathom-media-download-id.util.ts b/packages/twenty-apps/public/fathom/src/logic-functions/utils/update-fathom-media-download-id.util.tsindex 7e454633a35..e3653061a65 100644--- a/packages/twenty-apps/public/fathom/src/logic-functions/utils/update-fathom-media-download-id.util.ts+++ b/packages/twenty-apps/public/fathom/src/logic-functions/utils/update-fathom-media-download-id.util.ts@@ -1,7 +1,12 @@+import { setTimeout } from 'node:timers/promises';+import { type CoreApiClient } from 'twenty-client-sdk/core';import { type FathomMediaWriteContext } from 'src/logic-functions/types/fathom-media-write-context.type';import { updateFathomRecordingImport } from 'src/logic-functions/utils/update-fathom-recording-import.util';+const MAX_DOWNLOAD_ID_SAVE_ATTEMPTS = 3;+const DOWNLOAD_ID_SAVE_RETRY_DELAY_MILLISECONDS = 500;+export const updateFathomMediaDownloadId = async ({coreApiClient,downloadId,@@ -10,12 +15,24 @@ export const updateFathomMediaDownloadId = async ({coreApiClient: Pick<CoreApiClient, 'mutation'>;downloadId: string | null;writeContext: FathomMediaWriteContext;-}): Promise<boolean> =>- updateFathomRecordingImport({- coreApiClient,- writeContext,- fields: {- mediaDownloadId: downloadId,- mediaUploadCheckpoint: null,- },- });+}): Promise<boolean> => {+ const saveDownloadId = () =>+ updateFathomRecordingImport({+ coreApiClient,+ writeContext,+ fields: {+ mediaDownloadId: downloadId,+ mediaUploadCheckpoint: null,+ },+ });++ for (let attempt = 1; attempt < MAX_DOWNLOAD_ID_SAVE_ATTEMPTS; attempt++) {+ try {+ return await saveDownloadId();+ } catch {+ await setTimeout(DOWNLOAD_ID_SAVE_RETRY_DELAY_MILLISECONDS * attempt);+ }+ }++ return saveDownloadId();+};