Skip to content

The open alternative to Salesforce, designed for AI.

Refactor client auto heal (#19032)

Paul Rastoindb9da3194faf
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts
deleted file mode 100644
index e88572e5cc5..00000000000
--- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-import { InjectRepository } from '@nestjs/typeorm';
-
-import { Command } from 'nest-commander';
-import { Repository } from 'typeorm';
-
-import { ActiveOrSuspendedWorkspacesMigrationCommandRunner } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
-import { RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspaces-migration.command-runner';
-import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
-import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service';
-import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
-import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
-import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
-
-@Command({
- name: 'upgrade:1-20:generate-application-sdk-clients',
- description:
- 'Generate SDK client archives for all existing applications so drivers do not crash with ARCHIVE_NOT_FOUND',
-})
-export class GenerateApplicationSdkClientsCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
- constructor(
- @InjectRepository(WorkspaceEntity)
- protected readonly workspaceRepository: Repository<WorkspaceEntity>,
- protected readonly twentyORMGlobalManager: GlobalWorkspaceOrmManager,
- protected readonly dataSourceService: DataSourceService,
- @InjectRepository(ApplicationEntity)
- private readonly applicationRepository: Repository<ApplicationEntity>,
- private readonly sdkClientGenerationService: SdkClientGenerationService,
- ) {
- super(workspaceRepository, twentyORMGlobalManager, dataSourceService);
- }
-
- override async runOnWorkspace({
- workspaceId,
- options,
- }: RunOnWorkspaceArgs): Promise<void> {
- const dryRun = options.dryRun ?? false;
-
- const applications = await this.applicationRepository.find({
- where: { workspaceId },
- });
-
- this.logger.log(
- `Found ${applications.length} application(s) in workspace ${workspaceId}`,
- );
-
- for (const application of applications) {
- if (dryRun) {
- this.logger.log(
- `[DRY RUN] Would generate SDK client for application ${application.universalIdentifier} (${application.id})`,
- );
- continue;
- }
-
- try {
- await this.sdkClientGenerationService.generateSdkClientForApplication({
- workspaceId,
- applicationId: application.id,
- applicationUniversalIdentifier: application.universalIdentifier,
- });
- } catch (error) {
- this.logger.error(
- `Failed to generate SDK client for application ${application.universalIdentifier} (${application.id}): ${error instanceof Error ? error.message : String(error)}`,
- );
- }
- }
- }
-}
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts
index 7e6da522501..c1d249e6a16 100644
--- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-20/1-20-upgrade-version-command.module.ts
@@ -6,7 +6,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg
import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command';
import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command';
import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command';
-import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command';
import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command';
import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -17,10 +16,8 @@ import { MigrateRichTextToTextCommand } from 'src/database/commands/upgrade-vers
import { SeedCliApplicationRegistrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-seed-cli-application-registration.command';
import { UpdateStandardIndexViewNamesCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-update-standard-index-view-names.command';
import { ApplicationRegistrationModule } from 'src/engine/core-modules/application/application-registration/application-registration.module';
-import { ApplicationEntity } from 'src/engine/core-modules/application/application.entity';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
-import { SdkClientModule } from 'src/engine/core-modules/sdk-client/sdk-client.module';
import { UserWorkspaceEntity } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { CalendarChannelEntity } from 'src/engine/metadata-modules/calendar-channel/entities/calendar-channel.entity';
@@ -40,7 +37,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
imports: [
TypeOrmModule.forFeature([
WorkspaceEntity,
- ApplicationEntity,
ConnectedAccountEntity,
MessageChannelEntity,
CalendarChannelEntity,
@@ -57,7 +53,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
ApplicationRegistrationModule,
WorkspaceMigrationModule,
FeatureFlagModule,
- SdkClientModule,
WorkflowCommonModule,
],
providers: [
@@ -70,7 +65,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
BackfillSelectFieldOptionIdsCommand,
DeleteOrphanNavigationMenuItemsCommand,
- GenerateApplicationSdkClientsCommand,
SeedCliApplicationRegistrationCommand,
MigrateRichTextToTextCommand,
MigrateMessagingInfrastructureToMetadataCommand,
@@ -87,7 +81,6 @@ import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-commo
BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
BackfillSelectFieldOptionIdsCommand,
DeleteOrphanNavigationMenuItemsCommand,
- GenerateApplicationSdkClientsCommand,
SeedCliApplicationRegistrationCommand,
MigrateRichTextToTextCommand,
MigrateMessagingInfrastructureToMetadataCommand,
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts
index 017724acac6..e63e62031bc 100644
--- a/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/upgrade.command.ts
@@ -38,7 +38,6 @@ import { BackfillNavigationMenuItemTypeCommand } from 'src/database/commands/upg
import { BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-page-layouts-and-fields-widget-view-fields.command';
import { BackfillSelectFieldOptionIdsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-backfill-select-field-option-ids.command';
import { DeleteOrphanNavigationMenuItemsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-delete-orphan-navigation-menu-items.command';
-import { GenerateApplicationSdkClientsCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-generate-application-sdk-clients.command';
import { IdentifyObjectPermissionMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-object-permission-metadata.command';
import { IdentifyPermissionFlagMetadataCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-identify-permission-flag-metadata.command';
import { MakeObjectPermissionUniversalIdentifierAndApplicationIdNotNullableMigrationCommand } from 'src/database/commands/upgrade-version-command/1-20/1-20-make-object-permission-universal-identifier-and-application-id-not-nullable-migration.command';
@@ -106,7 +105,6 @@ export class UpgradeCommand extends UpgradeCommandRunner {
protected readonly backfillCommandMenuItemsCommand: BackfillCommandMenuItemsCommand,
protected readonly deleteOrphanNavigationMenuItemsCommand: DeleteOrphanNavigationMenuItemsCommand,
protected readonly backfillPageLayoutsAndFieldsWidgetViewFieldsCommand: BackfillPageLayoutsAndFieldsWidgetViewFieldsCommand,
- protected readonly generateApplicationSdkClientsCommand: GenerateApplicationSdkClientsCommand,
protected readonly seedCliApplicationRegistrationCommand: SeedCliApplicationRegistrationCommand,
protected readonly migrateRichTextToTextCommand: MigrateRichTextToTextCommand,
protected readonly migrateMessagingInfrastructureToMetadataCommand: MigrateMessagingInfrastructureToMetadataCommand,
@@ -176,7 +174,6 @@ export class UpgradeCommand extends UpgradeCommandRunner {
this.backfillSelectFieldOptionIdsCommand,
this.updateStandardIndexViewNamesCommand,
this.makeWorkflowSearchableCommand,
- this.generateApplicationSdkClientsCommand,
];
this.allCommands = {
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
index 5b75757c8ae..126c1d042b9 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver.ts
@@ -721,6 +721,7 @@ export class LambdaDriver implements LogicFunctionDriver {
const sdkArchiveBuffer =
await this.sdkClientArchiveService.downloadArchiveBuffer({
workspaceId: flatApplication.workspaceId,
+ applicationId: flatApplication.id,
applicationUniversalIdentifier,
});
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts
index 72aacd97cd9..9216f296433 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver.ts
@@ -113,6 +113,7 @@ export class LocalDriver implements LogicFunctionDriver {
await this.sdkClientArchiveService.downloadAndExtractToPackage({
workspaceId: flatApplication.workspaceId,
+ applicationId: flatApplication.id,
applicationUniversalIdentifier,
targetPackagePath: sdkPackagePath,
});
diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts
index 026735eb733..b8752fcab3b 100644
--- a/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts
+++ b/packages/twenty-server/src/engine/core-modules/sdk-client/controllers/sdk-client.controller.ts
@@ -59,6 +59,7 @@ export class SdkClientController {
const fileBuffer =
await this.sdkClientArchiveService.getClientModuleFromArchive({
workspaceId: workspace.id,
+ applicationId,
applicationUniversalIdentifier: application.universalIdentifier,
moduleName,
});
diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts
index 657c255732d..27402eb90db 100644
--- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-archive.service.ts
@@ -1,11 +1,7 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { createWriteStream } from 'fs';
import * as fs from 'fs/promises';
-import { join } from 'path';
-import { type Readable } from 'stream';
-import { pipeline } from 'stream/promises';
import { FileFolder } from 'twenty-shared/types';
import { Repository } from 'typeorm';
@@ -16,12 +12,12 @@ import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
-import { TemporaryDirManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/temporary-dir-manager';
import { type SdkModuleName } from 'src/engine/core-modules/sdk-client/constants/allowed-sdk-modules';
import {
SdkClientException,
SdkClientExceptionCode,
} from 'src/engine/core-modules/sdk-client/exceptions/sdk-client.exception';
+import { SdkClientGenerationService } from 'src/engine/core-modules/sdk-client/sdk-client-generation.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
@@ -29,75 +25,74 @@ const SDK_CLIENT_ARCHIVE_NAME = 'twenty-client-sdk.zip';
@Injectable()
export class SdkClientArchiveService {
+ private readonly logger = new Logger(SdkClientArchiveService.name);
+
constructor(
private readonly fileStorageService: FileStorageService,
@InjectRepository(ApplicationEntity)
private readonly applicationRepository: Repository<ApplicationEntity>,
private readonly workspaceCacheService: WorkspaceCacheService,
+ private readonly sdkClientGenerationService: SdkClientGenerationService,
) {}
async downloadAndExtractToPackage({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
targetPackagePath,
}: {
workspaceId: string;
+ applicationId: string;
applicationUniversalIdentifier: string;
targetPackagePath: string;
}): Promise<void> {
- const temporaryDirManager = new TemporaryDirManager();
-
- try {
- const { sourceTemporaryDir } = await temporaryDirManager.init();
- const archivePath = join(sourceTemporaryDir, SDK_CLIENT_ARCHIVE_NAME);
-
- const archiveStream = await this.readArchiveStream({
- workspaceId,
- applicationUniversalIdentifier,
- });
-
- await pipeline(archiveStream, createWriteStream(archivePath));
+ const archiveBuffer = await this.downloadArchiveBufferOrGenerate({
+ workspaceId,
+ applicationId,
+ applicationUniversalIdentifier,
+ });
- await fs.rm(targetPackagePath, { recursive: true, force: true });
- await fs.mkdir(targetPackagePath, { recursive: true });
+ await fs.rm(targetPackagePath, { recursive: true, force: true });
+ await fs.mkdir(targetPackagePath, { recursive: true });
- const { default: unzipper } = await import('unzipper');
- const directory = await unzipper.Open.file(archivePath);
+ const { default: unzipper } = await import('unzipper');
+ const directory = await unzipper.Open.buffer(archiveBuffer);
- await directory.extract({ path: targetPackagePath });
- } finally {
- await temporaryDirManager.clean();
- }
+ await directory.extract({ path: targetPackagePath });
}
async downloadArchiveBuffer({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
+ applicationId: string;
applicationUniversalIdentifier: string;
}): Promise<Buffer> {
- const archiveStream = await this.readArchiveStream({
+ return this.downloadArchiveBufferOrGenerate({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
});
-
- return streamToBuffer(archiveStream);
}
async getClientModuleFromArchive({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
moduleName,
}: {
workspaceId: string;
+ applicationId: string;
applicationUniversalIdentifier: string;
moduleName: SdkModuleName;
}): Promise<Buffer> {
const filePath = `dist/${moduleName}.mjs`;
- const archiveBuffer = await this.downloadArchiveBuffer({
+ const archiveBuffer = await this.downloadArchiveBufferOrGenerate({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
});
@@ -135,32 +130,41 @@ export class SdkClientArchiveService {
]);
}
- private async readArchiveStream({
+ private async downloadArchiveBufferOrGenerate({
workspaceId,
+ applicationId,
applicationUniversalIdentifier,
}: {
workspaceId: string;
+ applicationId: string;
applicationUniversalIdentifier: string;
- }): Promise<Readable> {
+ }): Promise<Buffer> {
try {
- return await this.fileStorageService.readFile({
+ const stream = await this.fileStorageService.readFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.GeneratedSdkClient,
resourcePath: SDK_CLIENT_ARCHIVE_NAME,
});
+
+ return await streamToBuffer(stream);
} catch (error) {
if (
- error instanceof FileStorageException &&
- error.code === FileStorageExceptionCode.FILE_NOT_FOUND
+ !(error instanceof FileStorageException) ||
+ error.code !== FileStorageExceptionCode.FILE_NOT_FOUND
) {
- throw new SdkClientException(
- `SDK client archive "${SDK_CLIENT_ARCHIVE_NAME}" not found for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}".`,
- SdkClientExceptionCode.ARCHIVE_NOT_FOUND,
- );
+ throw error;
}
-
- throw error;
}
+
+ this.logger.warn(
+ `SDK client archive missing for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}", generating on-the-fly`,
+ );
+
+ return this.sdkClientGenerationService.generateSdkClientForApplication({
+ workspaceId,
+ applicationId,
+ applicationUniversalIdentifier,
+ });
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts
index 0392df0e774..0e074835a26 100644
--- a/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/sdk-client/sdk-client-generation.service.ts
@@ -44,13 +44,13 @@ export class SdkClientGenerationService {
workspaceId: string;
applicationId: string;
applicationUniversalIdentifier: string;
- }): Promise<void> {
+ }): Promise<Buffer> {
const graphqlSchema = await this.workspaceSchemaFactory.createGraphQLSchema(
{ id: workspaceId } as WorkspaceEntity,
applicationId,
);
- await this.generateAndStore({
+ const archiveBuffer = await this.generateAndStore({
workspaceId,
applicationId,
applicationUniversalIdentifier,
@@ -60,6 +60,8 @@ export class SdkClientGenerationService {
this.logger.log(
`Generated SDK client for application ${applicationUniversalIdentifier}`,
);
+
+ return archiveBuffer;
}
private async generateAndStore({
@@ -72,7 +74,7 @@ export class SdkClientGenerationService {
applicationId: string;
applicationUniversalIdentifier: string;
schema: string;
- }): Promise<void> {
+ }): Promise<Buffer> {
const temporaryDirManager = new TemporaryDirManager();
try {
@@ -101,12 +103,14 @@ export class SdkClientGenerationService {
await createZipFile(tempPackageRoot, archivePath);
+ const archiveBuffer = await fs.readFile(archivePath);
+
await this.fileStorageService.writeFile({
workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.GeneratedSdkClient,
resourcePath: SDK_CLIENT_ARCHIVE_NAME,
- sourceFile: await fs.readFile(archivePath),
+ sourceFile: archiveBuffer,
mimeType: 'application/zip',
settings: { isTemporaryFile: false, toDelete: false },
});
@@ -119,6 +123,8 @@ export class SdkClientGenerationService {
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'flatApplicationMaps',
]);
+
+ return archiveBuffer;
} catch (error) {
throw new SdkClientException(
`Failed to generate SDK client for application "${applicationUniversalIdentifier}" in workspace "${workspaceId}": ${error instanceof Error ? error.message : String(error)}`,