The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Batch global admin chat report transactions (#27746)
Félix Malfait632b1374bf87
diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.tsnew file mode 100644index 00000000000..6554179056c--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/services/__tests__/admin-panel-global-chat-threads.postgres.spec.ts@@ -0,0 +1,162 @@+import { DataSource } from 'typeorm';++import { AdminChatThreadScope } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-scope.enum';+import { AdminChatThreadSortDirection } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-sort-direction.enum';+import { AdminChatThreadSortField } from 'src/engine/core-modules/admin-panel/enums/admin-chat-thread-sort-field.enum';+import { AdminPanelGlobalChatThreadsService } from 'src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service';+import { AgentHistoryWorkspaceStorageService } from 'src/engine/metadata-modules/ai/ai-history/services/agent-history-workspace-storage.service';+import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/get-workspace-schema-name.util';++jest.mock('src/engine/core-modules/workspace/workspace.entity', () => ({+ WorkspaceEntity: class WorkspaceEntity {},+}));++const DATABASE_URL = process.env.ADMIN_CHAT_REPORT_TEST_DATABASE_URL;+const WORKSPACE_IDS = Array.from(+ { length: 26 },+ (_, index) =>+ `20202020-1111-4111-8111-${String(index + 1).padStart(12, '0')}`,+);+const OPTIONS = {+ scope: AdminChatThreadScope.ALL,+ hasErrorOnly: false,+ userNeverEngagedOnly: false,+ sortBy: AdminChatThreadSortField.CREATED_AT,+ sortDirection: AdminChatThreadSortDirection.ASC,+ limit: 1,+ offset: 1,+};++(DATABASE_URL ? describe : describe.skip)(+ 'admin chat report on PostgreSQL',+ () => {+ const dataSource = new DataSource({+ type: 'postgres',+ url: DATABASE_URL,+ entities: [],+ synchronize: false,+ });+ const storage = new AgentHistoryWorkspaceStorageService(dataSource);+ const service = new AdminPanelGlobalChatThreadsService(+ {+ find: jest.fn().mockResolvedValue(WORKSPACE_IDS.map((id) => ({ id }))),+ },+ storage,+ );++ beforeAll(async () => {+ jest.useRealTimers();+ if (new URL(DATABASE_URL!).pathname !== '/admin_chat_report_test') {+ throw new Error('Use the isolated admin_chat_report_test database');+ }+ await dataSource.initialize();+ await dataSource.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');+ });++ beforeEach(async () => {+ await dataSource.query(`CREATE SCHEMA core;+ CREATE TABLE core.workspace (id uuid PRIMARY KEY, "displayName" text, "allowImpersonation" boolean, "deletedAt" timestamptz);+ CREATE TABLE core."userWorkspace" (id uuid, "userId" uuid, "workspaceId" uuid, "deletedAt" timestamptz);+ CREATE TABLE core."user" (id uuid, email text, "firstName" text, "lastName" text)`);+ for (const [index, workspaceId] of WORKSPACE_IDS.entries()) {+ const schema = getWorkspaceSchemaName(workspaceId);+ await dataSource.query(`CREATE SCHEMA "${schema}";+ CREATE TABLE "${schema}"."agentChatThread" (id uuid, title text, "workspaceMemberId" uuid, "deletedAt" timestamptz, "createdAt" timestamptz, "updatedAt" timestamptz);+ CREATE TABLE "${schema}"."agentMessage" (id uuid, "threadId" uuid, role text, "isHidden" boolean);+ CREATE TABLE "${schema}"."agentMessagePart" ("messageId" uuid, "toolName" text, "toolOutput" jsonb);+ CREATE TABLE "${schema}"."workspaceMember" (id uuid, "userId" uuid)`);+ await dataSource.query(+ 'INSERT INTO core.workspace VALUES ($1, $2, true, null)',+ [workspaceId, `Workspace ${index}`],+ );+ if (index === 0 || index === 25) {+ await dataSource.query(+ `INSERT INTO "${schema}"."agentChatThread" (id, title, "createdAt", "updatedAt") VALUES ($1, $2, $3, $3)`,+ [+ workspaceId,+ `Chat ${index}`,+ new Date(Date.UTC(2026, 0, index + 1)),+ ],+ );+ }+ }+ });++ afterEach(async () => {+ jest.restoreAllMocks();+ if (!dataSource.isInitialized) {+ return;+ }+ for (const workspaceId of WORKSPACE_IDS) {+ await dataSource.query(+ `DROP SCHEMA IF EXISTS "${getWorkspaceSchemaName(workspaceId)}" CASCADE`,+ );+ }+ await dataSource.query('DROP SCHEMA IF EXISTS core CASCADE');+ });++ afterAll(async () => {+ if (dataSource.isInitialized) {+ await dataSource.destroy();+ }+ });++ it('releases earlier workspace locks while later report queries run and preserves global pagination', async () => {+ const createQueryRunner = dataSource.createQueryRunner.bind(dataSource);+ const migrationRunner = createQueryRunner();+ await migrationRunner.connect();+ await migrationRunner.query("SET lock_timeout = '100ms'");+ let isMigrationCompleted = false;+ jest.spyOn(dataSource, 'createQueryRunner').mockImplementation((mode) => {+ const runner = createQueryRunner(mode);+ const query = runner.query.bind(runner);+ jest+ .spyOn(runner, 'query')+ .mockImplementation(async (...parameters) => {+ const [sql, values] = parameters;+ if (+ typeof sql === 'string' &&+ sql.includes('WITH candidates') &&+ Array.isArray(values) &&+ Array.isArray(values[0]) &&+ values[0][0] === WORKSPACE_IDS[25]+ ) {+ await migrationRunner.query(+ `ALTER TABLE "${getWorkspaceSchemaName(WORKSPACE_IDS[0])}"."agentChatThread" ADD COLUMN "reportLockRegression" text`,+ );+ isMigrationCompleted = true;+ }+ return query(...parameters);+ });+ return runner;+ });+ try {+ await expect(+ service.getGlobalChatThreads(OPTIONS),+ ).resolves.toMatchObject({+ threads: [{ id: WORKSPACE_IDS[25] }],+ totalCount: 2,+ hasMore: false,+ });+ expect(isMigrationCompleted).toBe(true);+ } finally {+ await migrationRunner.release();+ }+ });++ it('continues past a batch with no provisioned history tables', async () => {+ for (const workspaceId of WORKSPACE_IDS.slice(0, 25)) {+ await dataSource.query(+ `DROP SCHEMA "${getWorkspaceSchemaName(workspaceId)}" CASCADE`,+ );+ }+ await expect(+ service.getGlobalChatThreads({ ...OPTIONS, offset: 0 }),+ ).resolves.toMatchObject({+ threads: [{ id: WORKSPACE_IDS[25] }],+ totalCount: 1,+ hasMore: false,+ });+ });+ },+);diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.tsindex 3ab83cc1cb8..85a56b649de 100644--- a/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/services/admin-panel-global-chat-threads.service.ts@@ -2,7 +2,9 @@ import { getWorkspaceSchemaName } from 'src/engine/workspace-datasource/utils/geimport { escapeIdentifier } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';import { Inject, Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';+import chunk from 'lodash.chunk';import { Repository } from 'typeorm';+import { isNonEmptyArray } from 'twenty-shared/utils';import {ASK_QUESTION_TOOL_NAME,ASK_QUESTIONS_TOOL_NAME,@@ -46,6 +48,8 @@ type GlobalChatThreadRawRow = {updatedAt: Date;};+const ADMIN_CHAT_REPORT_WORKSPACE_BATCH_SIZE = 25;+const COLUMN_BY_SORT_FIELD: Record<AdminChatThreadSortField,'messageCount' | 'userReplyCount' | 'createdAt' | 'updatedAt'@@ -60,7 +64,10 @@ const COLUMN_BY_SORT_FIELD: Record<export class AdminPanelGlobalChatThreadsService {constructor(@InjectRepository(WorkspaceEntity)- private readonly workspaceRepository: Repository<WorkspaceEntity>,+ private readonly workspaceRepository: Pick<+ Repository<WorkspaceEntity>,+ 'find'+ >,@Inject(AgentHistoryWorkspaceStorageService)private readonly historyStorage: Pick<AgentHistoryWorkspaceStorageService,@@ -97,35 +104,38 @@ export class AdminPanelGlobalChatThreadsService {);};- await this.historyStorage.runReadOnlyReport(- workspaces.map((workspace) => workspace.id),- async ({ manager, partitions }) => {- // Fence for the 2.46 cross-upgrade window: turns only have a status- // once the 2.46 commands reached their workspace. Remove once 2.46- // leaves the window.- const schemasWithTurnStatus = new Set(- (- await manager.query<{ schemaName: string }[]>(- `SELECT table_schema AS "schemaName" FROM information_schema.columns- WHERE table_name = 'agentTurn' AND column_name = 'status' AND table_schema = ANY($1::text[])`,- [- partitions.map(({ workspaceIds }) =>- getWorkspaceSchemaName(workspaceIds[0]),- ),- ],- )- ).map(({ schemaName }) => schemaName),- );+ // Release earlier workspace locks before scanning the next batch so upgrades can proceed.+ for (const workspaceBatch of chunk(+ workspaces,+ ADMIN_CHAT_REPORT_WORKSPACE_BATCH_SIZE,+ )) {+ await this.historyStorage.runReadOnlyReport(+ workspaceBatch.map((workspace) => workspace.id),+ async ({ manager, partitions }) => {+ if (!isNonEmptyArray(partitions)) {+ return;+ }++ // Fence for the 2.46 cross-upgrade window: turns only have a status+ // once the 2.46 commands reached their workspace. Remove once 2.46+ // leaves the window.+ const schemasWithTurnStatus = new Set(+ (+ await manager.query<{ schemaName: string }[]>(+ `SELECT table_schema AS "schemaName" FROM information_schema.columns+ WHERE table_name = 'agentTurn' AND column_name = 'status' AND table_schema = ANY($1::text[])`,+ [+ partitions.map(({ workspaceIds }) =>+ getWorkspaceSchemaName(workspaceIds[0]),+ ),+ ],+ )+ ).map(({ schemaName }) => schemaName),+ );- for (- let offsetIndex = 0;- offsetIndex < partitions.length;- offsetIndex += 25- ) {const parameters: unknown[] = [];- const queries = partitions- .slice(offsetIndex, offsetIndex + 25)- .map(({ workspaceIds, table }, partitionIndex) => {+ const queries = partitions.map(+ ({ workspaceIds, table }, partitionIndex) => {const search = args.searchTerm?.trim().replace(/[\\%_]/g, '\\$&');const hasErrorSql = schemasWithTurnStatus.has(getWorkspaceSchemaName(workspaceIds[0]),@@ -167,7 +177,8 @@ export class AdminPanelGlobalChatThreadsService {partitionIndex,);return `(${query.replace(/\$(\d+)/g, (_, position: string) => `$${Number(position) + parameterOffset}`)})`;- });+ },+ );const rows = await manager.query<(GlobalChatThreadRawRow & {totalCount: string;@@ -182,9 +193,9 @@ export class AdminPanelGlobalChatThreadsService {candidates = [...candidates, ...rows].sort(compare).slice(0, offset + limit);- }- },- );+ },+ );+ }const threads = candidates.slice(offset, offset + limit);return {threads,