The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
optimize workspace export command (#20000)
neo77320f7ba82d748
diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.tsnew file mode 100644index 00000000000..b483a7c346b--- /dev/null+++ b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts@@ -0,0 +1,72 @@+import { formatPgCopyField } from 'src/database/commands/workspace-export/utils/format-pg-copy-value.util';++describe('formatPgCopyField', () => {+ it('should return \\N for null and undefined', () => {+ expect(formatPgCopyField(null)).toBe('\\N');+ expect(formatPgCopyField(undefined)).toBe('\\N');+ });++ it('should return t/f for booleans (not true/false)', () => {+ expect(formatPgCopyField(true)).toBe('t');+ expect(formatPgCopyField(false)).toBe('f');+ });++ it('should return \\N for non-finite numbers', () => {+ expect(formatPgCopyField(NaN)).toBe('\\N');+ expect(formatPgCopyField(Infinity)).toBe('\\N');+ });++ it('should escape tabs, newlines, and backslashes in strings', () => {+ expect(formatPgCopyField('col1\tcol2')).toBe('col1\\tcol2');+ expect(formatPgCopyField('line1\nline2')).toBe('line1\\nline2');+ expect(formatPgCopyField('path\\to\\file')).toBe('path\\\\to\\\\file');+ expect(formatPgCopyField('a\r\nb')).toBe('a\\r\\nb');+ });++ it('should not quote strings (COPY format is unquoted)', () => {+ const result = formatPgCopyField('hello world');++ expect(result).toBe('hello world');+ expect(result).not.toContain("'");+ });++ it('should format dates as ISO strings without quotes', () => {+ const date = new Date('2024-01-15T10:30:00.000Z');++ expect(formatPgCopyField(date)).toBe('2024-01-15T10:30:00.000Z');+ });++ it('should escape special chars in JSON column values', () => {+ const value = { key: 'value\twith\ttabs' };++ expect(formatPgCopyField(value, true)).toBe(+ '{"key":"value\\\\twith\\\\ttabs"}',+ );+ });++ it('should format PostgreSQL array literals', () => {+ expect(formatPgCopyField([])).toBe('{}');+ expect(formatPgCopyField(['a', 'b'])).toBe('{"a","b"}');+ expect(formatPgCopyField(['val\twith\ttab'])).toBe('{"val\\twith\\ttab"}');+ });++ it('should JSON-serialize arrays of objects with escaping', () => {+ const value = [{ id: 1, name: 'test\ttab' }];++ expect(formatPgCopyField(value)).toBe('[{"id":1,"name":"test\\\\ttab"}]');+ });++ it('should handle a row with mixed types matching COPY tab-delimited format', () => {+ const fields = [+ formatPgCopyField('abc-123'),+ formatPgCopyField(null),+ formatPgCopyField(true),+ formatPgCopyField(42),+ formatPgCopyField(new Date('2025-03-28T00:00:00.000Z')),+ ];++ expect(fields.join('\t')).toBe(+ 'abc-123\t\\N\tt\t42\t2025-03-28T00:00:00.000Z',+ );+ });+});diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.tsnew file mode 100644index 00000000000..261237b42cd--- /dev/null+++ b/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts@@ -0,0 +1,57 @@+import { isDefined } from 'twenty-shared/utils';++const escapeCopyText = (text: string): string => {+ return text+ .replace(/\\/g, '\\\\')+ .replace(/\t/g, '\\t')+ .replace(/\n/g, '\\n')+ .replace(/\r/g, '\\r');+};++export const formatPgCopyField = (+ value: unknown,+ isJsonColumn = false,+): string => {+ if (!isDefined(value)) return '\\N';++ if (isJsonColumn) return escapeCopyText(JSON.stringify(value));++ if (typeof value === 'boolean') return value ? 't' : 'f';++ if (typeof value === 'number') {+ if (!Number.isFinite(value)) return '\\N';++ return String(value);+ }++ if (typeof value === 'bigint') return String(value);++ if (value instanceof Date) return value.toISOString();++ if (Array.isArray(value)) {+ if (value.length === 0) return '{}';++ if (isDefined(value[0]) && typeof value[0] === 'object') {+ return escapeCopyText(JSON.stringify(value));+ }++ const formattedElements = value.map((element) => {+ if (!isDefined(element)) return 'NULL';++ const escapedElement = String(element)+ .replace(/\\/g, '\\\\')+ .replace(/\t/g, '\\t')+ .replace(/\n/g, '\\n')+ .replace(/\r/g, '\\r')+ .replace(/"/g, '\\"');++ return `"${escapedElement}"`;+ });++ return `{${formattedElements.join(',')}}`;+ }++ if (typeof value === 'object') return escapeCopyText(JSON.stringify(value));++ return escapeCopyText(String(value));+};diff --git a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.tsindex 92249419f7c..bbaa92b41fe 100644--- a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts+++ b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts@@ -22,9 +22,10 @@ import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-expoimport { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util';import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util';import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';-import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util';+import { formatPgCopyField } from './utils/format-pg-copy-value.util';+import { isNonEmptyArray } from 'twenty-shared/utils';-const BATCH_SIZE = 5000;+const BATCH_SIZE = 10_000;type WorkspaceExportParams = {workspaceId: string;@@ -219,47 +220,103 @@ export class WorkspaceExportService {excludedColumns,}: WriteRowsOptions): Promise<void> {const whereFragment = whereClause ? ` WHERE ${whereClause}` : '';-- const [{ count: totalCount }] = await queryRunner.query(- `SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereFragment}`,- queryParameters,- );-- if (totalCount === 0) return;-- this.logger.log(` ${displayName}: ${totalCount} rows`);-+ let columnNames: string[] | undefined;let insertPrefix: string | undefined;+ let totalRows = 0;- for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) {+ for (let offset = 0; ; offset += BATCH_SIZE) {const rows: Record<string, unknown>[] = await queryRunner.query(`SELECT * FROM "${schemaName}"."${tableName}"${whereFragment} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,queryParameters,);- const batchStatements: string[] = [];+ if (!isNonEmptyArray(rows)) break;- for (const row of rows) {- const columnNames = Object.keys(row).filter(+ if (!columnNames) {+ columnNames = Object.keys(rows[0]).filter((columnName) => !excludedColumns?.has(columnName),);+ insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);+ }- if (!insertPrefix) {- insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);- }+ totalRows += rows.length;++ const valueTuples: string[] = [];+ for (const row of rows) {const formattedValues = columnNames.map((columnName) =>formatSqlValue(row[columnName], jsonColumns?.has(columnName)),);- batchStatements.push(- generateInsertStatement(insertPrefix, formattedValues),- );+ valueTuples.push(`(${formattedValues.join(', ')})`);}- if (!stream.write(batchStatements.join(''))) {+ const statement = `${insertPrefix}${valueTuples.join(', ')};\n`;++ if (!stream.write(statement)) {await once(stream, 'drain');}++ if (rows.length < BATCH_SIZE) break;+ }++ if (totalRows > 0) {+ this.logger.log(` ${displayName}: ${totalRows} rows`);+ }+ }++ private async writeCopyRows({+ schemaName,+ tableName,+ displayName,+ queryRunner,+ stream,+ jsonColumns,+ excludedColumns,+ }: Omit<WriteRowsOptions, 'whereClause' | 'queryParameters'>): Promise<void> {+ let columnNames: string[] | undefined;+ let totalRows = 0;++ for (let offset = 0; ; offset += BATCH_SIZE) {+ const rows: Record<string, unknown>[] = await queryRunner.query(+ `SELECT * FROM "${schemaName}"."${tableName}" ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,+ );++ if (!isNonEmptyArray(rows)) break;++ if (!columnNames) {+ columnNames = Object.keys(rows[0]).filter(+ (columnName) => !excludedColumns?.has(columnName),+ );++ const escapedColumns = columnNames.map(escapeIdentifier).join(', ');++ stream.write(+ `COPY ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumns}) FROM stdin;\n`,+ );+ }++ totalRows += rows.length;++ for (const row of rows) {+ const values = columnNames.map((columnName) =>+ formatPgCopyField(row[columnName], jsonColumns?.has(columnName)),+ );++ if (!stream.write(values.join('\t') + '\n')) {+ await once(stream, 'drain');+ }+ }++ if (rows.length < BATCH_SIZE) break;+ }++ if (isNonEmptyArray(columnNames)) {+ stream.write('\\.\n\n');+ }++ if (totalRows > 0) {+ this.logger.log(` ${displayName}: ${totalRows} rows`);}}@@ -319,7 +376,7 @@ export class WorkspaceExportService {);try {- await this.writeRows({+ await this.writeCopyRows({schemaName,tableName,displayName: objectMetadata.nameSingular,