Skip to content

The open alternative to Salesforce, designed for AI.

Batch global admin chat report transactions (#27746)

Félix Malfait632b1374bf87
diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.ts
new file mode 100644
index 00000000000..6554179056c
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.ts
@@ -0,0 +1,162 @@
+import { DataSource } from 'typeorm';
+
+import { AdminChatThreadScope } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-scope.enum';
+import { AdminChatThreadSortDirection } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-sort-direction.enum';
+import { AdminChatThreadSortField } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-sort-field.enum';
+import { AdminPanelGlobalChatThreadsService } from 'src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service';
+import { AgentHistoryWorkspaceStorageService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-history-workspace-storage.service';
+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';
+
+jest.mock('src/engine/core-modules/workspace/workspace.entity', () => ({
+ WorkspaceEntity: class WorkspaceEntity {},
+}));
+
+const DATABASE_URL = process.env.ADMIN_CHAT_REPORT_TEST_DATABASE_URL;
+const WORKSPACE_IDS = Array.from(
+ { length: 26 },
+ (_, index) =>
+ `20202020-1111-4111-8111-${String(index + 1).padStart(12, '0')}`,
+);
+const OPTIONS = {
+ scope: AdminChatThreadScope.ALL,
+ hasErrorOnly: false,
+ userNeverEngagedOnly: false,
+ sortBy: AdminChatThreadSortField.CREATED_AT,
+ sortDirection: AdminChatThreadSortDirection.ASC,
+ limit: 1,
+ offset: 1,
+};
+
+(DATABASE_URL ? describe : describe.skip)(
+ 'admin chat report on PostgreSQL',
+ () => {
+ const dataSource = new DataSource({
+ type: 'postgres',
+ url: DATABASE_URL,
+ entities: [],
+ synchronize: false,
+ });
+ const storage = new AgentHistoryWorkspaceStorageService(dataSource);
+ const service = new AdminPanelGlobalChatThreadsService(
+ {
+ find: jest.fn().mockResolvedValue(WORKSPACE_IDS.map((id) => ({ id }))),
+ },
+ storage,
+ );
+
+ beforeAll(async () => {
+ jest.useRealTimers();
+ if (new URL(DATABASE_URL!).pathname !== '/admin_chat_report_test') {
+ throw new Error('Use the isolated admin_chat_report_test database');
+ }
+ await dataSource.initialize();
+ await dataSource.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');
+ });
+
+ beforeEach(async () => {
+ await dataSource.query(`CREATE SCHEMA core;
+ CREATE TABLE core.workspace (id uuid PRIMARY KEY, "displayName" text, "allowImpersonation" boolean, "deletedAt" timestamptz);
+ CREATE TABLE core."userWorkspace" (id uuid, "userId" uuid, "workspaceId" uuid, "deletedAt" timestamptz);
+ CREATE TABLE core."user" (id uuid, email text, "firstName" text, "lastName" text)`);
+ for (const [index, workspaceId] of WORKSPACE_IDS.entries()) {
+ const schema = getWorkspaceSchemaName(workspaceId);
+ await dataSource.query(`CREATE SCHEMA "${schema}";
+ CREATE TABLE "${schema}"."agentChatThread" (id uuid, title text, "workspaceMemberId" uuid, "deletedAt" timestamptz, "createdAt" timestamptz, "updatedAt" timestamptz);
+ CREATE TABLE "${schema}"."agentMessage" (id uuid, "threadId" uuid, role text, "isHidden" boolean);
+ CREATE TABLE "${schema}"."agentMessagePart" ("messageId" uuid, "toolName" text, "toolOutput" jsonb);
+ CREATE TABLE "${schema}"."workspaceMember" (id uuid, "userId" uuid)`);
+ await dataSource.query(
+ 'INSERT INTO core.workspace VALUES ($1, $2, true, null)',
+ [workspaceId, `Workspace ${index}`],
+ );
+ if (index === 0 || index === 25) {
+ await dataSource.query(
+ `INSERT INTO "${schema}"."agentChatThread" (id, title, "createdAt", "updatedAt") VALUES ($1, $2, $3, $3)`,
+ [
+ workspaceId,
+ `Chat ${index}`,
+ new Date(Date.UTC(2026, 0, index + 1)),
+ ],
+ );
+ }
+ }
+ });
+
+ afterEach(async () => {
+ jest.restoreAllMocks();
+ if (!dataSource.isInitialized) {
+ return;
+ }
+ for (const workspaceId of WORKSPACE_IDS) {
+ await dataSource.query(
+ `DROP SCHEMA IF EXISTS "${getWorkspaceSchemaName(workspaceId)}" CASCADE`,
+ );
+ }
+ await dataSource.query('DROP SCHEMA IF EXISTS core CASCADE');
+ });
+
+ afterAll(async () => {
+ if (dataSource.isInitialized) {
+ await dataSource.destroy();
+ }
+ });
+
+ it('releases earlier workspace locks while later report queries run and preserves global pagination', async () => {
+ const createQueryRunner = dataSource.createQueryRunner.bind(dataSource);
+ const migrationRunner = createQueryRunner();
+ await migrationRunner.connect();
+ await migrationRunner.query("SET lock_timeout = '100ms'");
+ let isMigrationCompleted = false;
+ jest.spyOn(dataSource, 'createQueryRunner').mockImplementation((mode) => {
+ const runner = createQueryRunner(mode);
+ const query = runner.query.bind(runner);
+ jest
+ .spyOn(runner, 'query')
+ .mockImplementation(async (...parameters) => {
+ const [sql, values] = parameters;
+ if (
+ typeof sql === 'string' &&
+ sql.includes('WITH candidates') &&
+ Array.isArray(values) &&
+ Array.isArray(values[0]) &&
+ values[0][0] === WORKSPACE_IDS[25]
+ ) {
+ await migrationRunner.query(
+ `ALTER TABLE "${getWorkspaceSchemaName(WORKSPACE_IDS[0])}"."agentChatThread" ADD COLUMN "reportLockRegression" text`,
+ );
+ isMigrationCompleted = true;
+ }
+ return query(...parameters);
+ });
+ return runner;
+ });
+ try {
+ await expect(
+ service.getGlobalChatThreads(OPTIONS),
+ ).resolves.toMatchObject({
+ threads: [{ id: WORKSPACE_IDS[25] }],
+ totalCount: 2,
+ hasMore: false,
+ });
+ expect(isMigrationCompleted).toBe(true);
+ } finally {
+ await migrationRunner.release();
+ }
+ });
+
+ it('continues past a batch with no provisioned history tables', async () => {
+ for (const workspaceId of WORKSPACE_IDS.slice(0, 25)) {
+ await dataSource.query(
+ `DROP SCHEMA "${getWorkspaceSchemaName(workspaceId)}" CASCADE`,
+ );
+ }
+ await expect(
+ service.getGlobalChatThreads({ ...OPTIONS, offset: 0 }),
+ ).resolves.toMatchObject({
+ threads: [{ id: WORKSPACE_IDS[25] }],
+ totalCount: 1,
+ hasMore: false,
+ });
+ });
+ },
+);
diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts
index 3ab83cc1cb8..85a56b649de 100644
--- a/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts
@@ -2,7 +2,9 @@ import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/ge
import { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
+import chunk from 'lodash.chunk';
import { Repository } from 'typeorm';
+import { isNonEmptyArray } from 'twenty-shared/utils';
import {
ASK_QUESTION_TOOL_NAME,
ASK_QUESTIONS_TOOL_NAME,
@@ -46,6 +48,8 @@ type GlobalChatThreadRawRow = {
updatedAt: Date;
};
+const ADMIN_CHAT_REPORT_WORKSPACE_BATCH_SIZE = 25;
+
const COLUMN_BY_SORT_FIELD: Record<
AdminChatThreadSortField,
'messageCount' | 'userReplyCount' | 'createdAt' | 'updatedAt'
@@ -60,7 +64,10 @@ const COLUMN_BY_SORT_FIELD: Record<
export class AdminPanelGlobalChatThreadsService {
constructor(
@InjectRepository(WorkspaceEntity)
- private readonly workspaceRepository: Repository<WorkspaceEntity>,
+ private readonly workspaceRepository: Pick<
+ Repository<WorkspaceEntity>,
+ 'find'
+ >,
@Inject(AgentHistoryWorkspaceStorageService)
private readonly historyStorage: Pick<
AgentHistoryWorkspaceStorageService,
@@ -97,35 +104,38 @@ export class AdminPanelGlobalChatThreadsService {
);
};
- await this.historyStorage.runReadOnlyReport(
- workspaces.map((workspace) => workspace.id),
- async ({ manager, partitions }) => {
- // Fence for the 2.46 cross-upgrade window: turns only have a status
- // once the 2.46 commands reached their workspace. Remove once 2.46
- // leaves the window.
- const schemasWithTurnStatus = new Set(
- (
- await manager.query<{ schemaName: string }[]>(
- `SELECT table_schema AS "schemaName" FROM information_schema.columns
- WHERE table_name = 'agentTurn' AND column_name = 'status' AND table_schema = ANY($1::text[])`,
- [
- partitions.map(({ workspaceIds }) =>
- getWorkspaceSchemaName(workspaceIds[0]),
- ),
- ],
- )
- ).map(({ schemaName }) => schemaName),
- );
+ // Release earlier workspace locks before scanning the next batch so upgrades can proceed.
+ for (const workspaceBatch of chunk(
+ workspaces,
+ ADMIN_CHAT_REPORT_WORKSPACE_BATCH_SIZE,
+ )) {
+ await this.historyStorage.runReadOnlyReport(
+ workspaceBatch.map((workspace) => workspace.id),
+ async ({ manager, partitions }) => {
+ if (!isNonEmptyArray(partitions)) {
+ return;
+ }
+
+ // Fence for the 2.46 cross-upgrade window: turns only have a status
+ // once the 2.46 commands reached their workspace. Remove once 2.46
+ // leaves the window.
+ const schemasWithTurnStatus = new Set(
+ (
+ await manager.query<{ schemaName: string }[]>(
+ `SELECT table_schema AS "schemaName" FROM information_schema.columns
+ WHERE table_name = 'agentTurn' AND column_name = 'status' AND table_schema = ANY($1::text[])`,
+ [
+ partitions.map(({ workspaceIds }) =>
+ getWorkspaceSchemaName(workspaceIds[0]),
+ ),
+ ],
+ )
+ ).map(({ schemaName }) => schemaName),
+ );
- for (
- let offsetIndex = 0;
- offsetIndex < partitions.length;
- offsetIndex += 25
- ) {
const parameters: unknown[] = [];
- const queries = partitions
- .slice(offsetIndex, offsetIndex + 25)
- .map(({ workspaceIds, table }, partitionIndex) => {
+ const queries = partitions.map(
+ ({ workspaceIds, table }, partitionIndex) => {
const search = args.searchTerm?.trim().replace(/[\\%_]/g, '\\$&');
const hasErrorSql = schemasWithTurnStatus.has(
getWorkspaceSchemaName(workspaceIds[0]),
@@ -167,7 +177,8 @@ export class AdminPanelGlobalChatThreadsService {
partitionIndex,
);
return `(${query.replace(/\$(\d+)/g, (_, position: string) => `$${Number(position) + parameterOffset}`)})`;
- });
+ },
+ );
const rows = await manager.query<
(GlobalChatThreadRawRow & {
totalCount: string;
@@ -182,9 +193,9 @@ export class AdminPanelGlobalChatThreadsService {
candidates = [...candidates, ...rows]
.sort(compare)
.slice(0, offset + limit);
- }
- },
- );
+ },
+ );
+ }
const threads = candidates.slice(offset, offset + limit);
return {
threads,