Skip to content

The open alternative to Salesforce, designed for AI.

fix(workflow): exclude soft-deleted workspaces from the core-consistency cron (#24740)

Charles Bochetebfbe8f24d31
diff --git a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
index 6fe038f01d5..32a47f2a8a0 100644
--- a/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
+++ b/packages/twenty-server/src/database/commands/command-runners/workspace-iterator.service.ts
@@ -8,7 +8,7 @@ import {
WorkspaceActivationStatus,
} from 'twenty-shared/workspace';
import { isDefined } from 'twenty-shared/utils';
-import { DataSource, MoreThanOrEqual, Repository } from 'typeorm';
+import { DataSource, MoreThanOrEqual, Raw, Repository } from 'typeorm';
import { CommandShutdownService } from 'src/database/commands/command-runners/command-shutdown.service';
import { activationStatusIn } from 'src/database/commands/command-runners/utils/activation-status-in.util';
@@ -17,17 +17,24 @@ import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspac
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { WorkspaceMigrationRunnerException } from 'src/engine/workspace-manager/workspace-migration/workspace-migration-runner/exceptions/workspace-migration-runner.exception';
+export type WorkspaceIteratorShard = {
+ index: number;
+ total: number;
+};
+
export type WorkspaceIteratorArgs = {
workspaceIds?: string[];
activationStatuses?: WorkspaceActivationStatus[];
startFromWorkspaceId?: string;
workspaceCountLimit?: number;
+ shard?: WorkspaceIteratorShard;
dryRun?: boolean;
callback: (context: WorkspaceIteratorContext) => Promise<void>;
};
export type WorkspaceIteratorContext = {
workspaceId: string;
+ databaseSchema?: string;
dataSource?: DataSource;
index: number;
total: number;
@@ -122,6 +129,7 @@ export class WorkspaceIteratorService {
await callback({
workspaceId,
+ databaseSchema: workspace?.databaseSchema ?? undefined,
dataSource,
index,
total: workspaceIdsToProcess.length,
@@ -166,11 +174,29 @@ export class WorkspaceIteratorService {
private async fetchWorkspaceIds(
options: Pick<
WorkspaceIteratorArgs,
- 'activationStatuses' | 'startFromWorkspaceId' | 'workspaceCountLimit'
+ | 'activationStatuses'
+ | 'startFromWorkspaceId'
+ | 'workspaceCountLimit'
+ | 'shard'
>,
): Promise<string[]> {
const activationStatuses =
options.activationStatuses ?? DEFAULT_ACTIVATION_STATUSES;
+ const shard = options.shard;
+
+ if (isDefined(shard)) {
+ if (isDefined(options.startFromWorkspaceId)) {
+ throw new Error(
+ 'Cannot combine shard with startFromWorkspaceId in workspace iterator',
+ );
+ }
+
+ if (shard.index < 0 || shard.index >= shard.total || shard.total > 256) {
+ throw new Error(
+ `Invalid workspace iterator shard ${shard.index}/${shard.total}`,
+ );
+ }
+ }
const workspaces = await this.workspaceRepository.find({
select: ['id'],
@@ -179,6 +205,15 @@ export class WorkspaceIteratorService {
...(options.startFromWorkspaceId
? { id: MoreThanOrEqual(options.startFromWorkspaceId) }
: {}),
+ ...(isDefined(shard)
+ ? {
+ id: Raw(
+ (alias) =>
+ `mod(get_byte(uuid_send(${alias}), 0), :shardTotal) = :shardIndex`,
+ { shardTotal: shard.total, shardIndex: shard.index },
+ ),
+ }
+ : {}),
},
order: { id: 'ASC' },
take: options.workspaceCountLimit,
diff --git a/packages/twenty-server/src/modules/workflow/workflow-core-consistency/services/workflow-core-consistency.service.ts b/packages/twenty-server/src/modules/workflow/workflow-core-consistency/services/workflow-core-consistency.service.ts
index 71b01a8745f..6a6c7aab5d6 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-core-consistency/services/workflow-core-consistency.service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-core-consistency/services/workflow-core-consistency.service.ts
@@ -4,6 +4,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
import { isDefined } from 'twenty-shared/utils';
import { DataSource } from 'typeorm';
+import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
@@ -16,6 +17,9 @@ import {
type DriftCounts = Record<string, number>;
+const CRON_INTERVAL_HOURS = 3;
+const SHARD_TOTAL = 24 / CRON_INTERVAL_HOURS;
+
// Detect drift between the workspace source-of-truth records and their core
// mirror. The dual-write is best-effort (async, not transactional), so core can
// silently fall out of sync; this quantifies that per workspace as metrics.
@@ -26,27 +30,35 @@ export class WorkflowCoreConsistencyService {
constructor(
@InjectDataSource()
private readonly coreDataSource: DataSource,
+ private readonly workspaceIteratorService: WorkspaceIteratorService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly metricsService: MetricsService,
private readonly exceptionHandlerService: ExceptionHandlerService,
) {}
async runConsistencyCheck(): Promise<void> {
- const workspaces: Array<{ workspaceId: string; databaseSchema: string }> =
- await this.coreDataSource.query(
- `SELECT id AS "workspaceId", "databaseSchema"
- FROM core."workspace"
- WHERE "activationStatus" IN ('ACTIVE', 'SUSPENDED') AND "databaseSchema" IS NOT NULL`,
- );
+ const shard = {
+ index:
+ Math.floor(new Date().getUTCHours() / CRON_INTERVAL_HOURS) %
+ SHARD_TOTAL,
+ total: SHARD_TOTAL,
+ };
+
+ const report = await this.workspaceIteratorService.iterate({
+ shard,
+ callback: async ({ workspaceId, databaseSchema }) => {
+ if (!isDefined(databaseSchema)) {
+ return;
+ }
- for (const { workspaceId, databaseSchema } of workspaces) {
- try {
await this.checkWorkspace(workspaceId, databaseSchema);
- } catch (error) {
- this.exceptionHandlerService.captureExceptions([error], {
- workspace: { id: workspaceId },
- });
- }
+ },
+ });
+
+ for (const { workspaceId, error } of report.fail) {
+ this.exceptionHandlerService.captureExceptions([error], {
+ workspace: { id: workspaceId },
+ });
}
}
diff --git a/packages/twenty-server/src/modules/workflow/workflow-core-consistency/workflow-core-consistency.module.ts b/packages/twenty-server/src/modules/workflow/workflow-core-consistency/workflow-core-consistency.module.ts
index 9430c040185..33e47e6a8dc 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-core-consistency/workflow-core-consistency.module.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-core-consistency/workflow-core-consistency.module.ts
@@ -1,5 +1,6 @@
import { Module } from '@nestjs/common';
+import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
import { WorkflowCoreConsistencyCronCommand } from 'src/modules/workflow/workflow-core-consistency/crons/commands/workflow-core-consistency-cron.command';
@@ -7,7 +8,7 @@ import { WorkflowCoreConsistencyCronJob } from 'src/modules/workflow/workflow-co
import { WorkflowCoreConsistencyService } from 'src/modules/workflow/workflow-core-consistency/services/workflow-core-consistency.service';
@Module({
- imports: [MetricsModule, WorkspaceCacheModule],
+ imports: [WorkspaceIteratorModule, MetricsModule, WorkspaceCacheModule],
providers: [
WorkflowCoreConsistencyService,
WorkflowCoreConsistencyCronJob,