Skip to content

The open alternative to Salesforce, designed for AI.

optimize workspace export command (#20000)

neo77320f7ba82d748
diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts
new file mode 100644
index 00000000000..b483a7c346b
--- /dev/null
+++ b/packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts
@@ -0,0 +1,72 @@
+import { formatPgCopyField } from 'src/database/commands/workspace-export/utils/format-pg-copy-value.util';
+
+describe('formatPgCopyField', () => {
+ it('should return \\N for null and undefined', () => {
+ expect(formatPgCopyField(null)).toBe('\\N');
+ expect(formatPgCopyField(undefined)).toBe('\\N');
+ });
+
+ it('should return t/f for booleans (not true/false)', () => {
+ expect(formatPgCopyField(true)).toBe('t');
+ expect(formatPgCopyField(false)).toBe('f');
+ });
+
+ it('should return \\N for non-finite numbers', () => {
+ expect(formatPgCopyField(NaN)).toBe('\\N');
+ expect(formatPgCopyField(Infinity)).toBe('\\N');
+ });
+
+ it('should escape tabs, newlines, and backslashes in strings', () => {
+ expect(formatPgCopyField('col1\tcol2')).toBe('col1\\tcol2');
+ expect(formatPgCopyField('line1\nline2')).toBe('line1\\nline2');
+ expect(formatPgCopyField('path\\to\\file')).toBe('path\\\\to\\\\file');
+ expect(formatPgCopyField('a\r\nb')).toBe('a\\r\\nb');
+ });
+
+ it('should not quote strings (COPY format is unquoted)', () => {
+ const result = formatPgCopyField('hello world');
+
+ expect(result).toBe('hello world');
+ expect(result).not.toContain("'");
+ });
+
+ it('should format dates as ISO strings without quotes', () => {
+ const date = new Date('2024-01-15T10:30:00.000Z');
+
+ expect(formatPgCopyField(date)).toBe('2024-01-15T10:30:00.000Z');
+ });
+
+ it('should escape special chars in JSON column values', () => {
+ const value = { key: 'value\twith\ttabs' };
+
+ expect(formatPgCopyField(value, true)).toBe(
+ '{"key":"value\\\\twith\\\\ttabs"}',
+ );
+ });
+
+ it('should format PostgreSQL array literals', () => {
+ expect(formatPgCopyField([])).toBe('{}');
+ expect(formatPgCopyField(['a', 'b'])).toBe('{"a","b"}');
+ expect(formatPgCopyField(['val\twith\ttab'])).toBe('{"val\\twith\\ttab"}');
+ });
+
+ it('should JSON-serialize arrays of objects with escaping', () => {
+ const value = [{ id: 1, name: 'test\ttab' }];
+
+ expect(formatPgCopyField(value)).toBe('[{"id":1,"name":"test\\\\ttab"}]');
+ });
+
+ it('should handle a row with mixed types matching COPY tab-delimited format', () => {
+ const fields = [
+ formatPgCopyField('abc-123'),
+ formatPgCopyField(null),
+ formatPgCopyField(true),
+ formatPgCopyField(42),
+ formatPgCopyField(new Date('2025-03-28T00:00:00.000Z')),
+ ];
+
+ expect(fields.join('\t')).toBe(
+ 'abc-123\t\\N\tt\t42\t2025-03-28T00:00:00.000Z',
+ );
+ });
+});
diff --git a/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts b/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts
new file mode 100644
index 00000000000..261237b42cd
--- /dev/null
+++ b/packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts
@@ -0,0 +1,57 @@
+import { isDefined } from 'twenty-shared/utils';
+
+const escapeCopyText = (text: string): string => {
+ return text
+ .replace(/\\/g, '\\\\')
+ .replace(/\t/g, '\\t')
+ .replace(/\n/g, '\\n')
+ .replace(/\r/g, '\\r');
+};
+
+export const formatPgCopyField = (
+ value: unknown,
+ isJsonColumn = false,
+): string => {
+ if (!isDefined(value)) return '\\N';
+
+ if (isJsonColumn) return escapeCopyText(JSON.stringify(value));
+
+ if (typeof value === 'boolean') return value ? 't' : 'f';
+
+ if (typeof value === 'number') {
+ if (!Number.isFinite(value)) return '\\N';
+
+ return String(value);
+ }
+
+ if (typeof value === 'bigint') return String(value);
+
+ if (value instanceof Date) return value.toISOString();
+
+ if (Array.isArray(value)) {
+ if (value.length === 0) return '{}';
+
+ if (isDefined(value[0]) && typeof value[0] === 'object') {
+ return escapeCopyText(JSON.stringify(value));
+ }
+
+ const formattedElements = value.map((element) => {
+ if (!isDefined(element)) return 'NULL';
+
+ const escapedElement = String(element)
+ .replace(/\\/g, '\\\\')
+ .replace(/\t/g, '\\t')
+ .replace(/\n/g, '\\n')
+ .replace(/\r/g, '\\r')
+ .replace(/"/g, '\\"');
+
+ return `"${escapedElement}"`;
+ });
+
+ return `{${formattedElements.join(',')}}`;
+ }
+
+ if (typeof value === 'object') return escapeCopyText(JSON.stringify(value));
+
+ return escapeCopyText(String(value));
+};
diff --git a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts
index 92249419f7c..bbaa92b41fe 100644
--- a/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts
+++ b/packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts
@@ -22,9 +22,10 @@ import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-expo
import { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util';
import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util';
import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
-import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util';
+import { formatPgCopyField } from './utils/format-pg-copy-value.util';
+import { isNonEmptyArray } from 'twenty-shared/utils';
-const BATCH_SIZE = 5000;
+const BATCH_SIZE = 10_000;
type WorkspaceExportParams = {
workspaceId: string;
@@ -219,47 +220,103 @@ export class WorkspaceExportService {
excludedColumns,
}: WriteRowsOptions): Promise<void> {
const whereFragment = whereClause ? ` WHERE ${whereClause}` : '';
-
- const [{ count: totalCount }] = await queryRunner.query(
- `SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereFragment}`,
- queryParameters,
- );
-
- if (totalCount === 0) return;
-
- this.logger.log(` ${displayName}: ${totalCount} rows`);
-
+ let columnNames: string[] | undefined;
let insertPrefix: string | undefined;
+ let totalRows = 0;
- for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) {
+ for (let offset = 0; ; offset += BATCH_SIZE) {
const rows: Record<string, unknown>[] = await queryRunner.query(
`SELECT * FROM "${schemaName}"."${tableName}"${whereFragment} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
queryParameters,
);
- const batchStatements: string[] = [];
+ if (!isNonEmptyArray(rows)) break;
- for (const row of rows) {
- const columnNames = Object.keys(row).filter(
+ if (!columnNames) {
+ columnNames = Object.keys(rows[0]).filter(
(columnName) => !excludedColumns?.has(columnName),
);
+ insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
+ }
- if (!insertPrefix) {
- insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
- }
+ totalRows += rows.length;
+
+ const valueTuples: string[] = [];
+ for (const row of rows) {
const formattedValues = columnNames.map((columnName) =>
formatSqlValue(row[columnName], jsonColumns?.has(columnName)),
);
- batchStatements.push(
- generateInsertStatement(insertPrefix, formattedValues),
- );
+ valueTuples.push(`(${formattedValues.join(', ')})`);
}
- if (!stream.write(batchStatements.join(''))) {
+ const statement = `${insertPrefix}${valueTuples.join(', ')};\n`;
+
+ if (!stream.write(statement)) {
await once(stream, 'drain');
}
+
+ if (rows.length < BATCH_SIZE) break;
+ }
+
+ if (totalRows > 0) {
+ this.logger.log(` ${displayName}: ${totalRows} rows`);
+ }
+ }
+
+ private async writeCopyRows({
+ schemaName,
+ tableName,
+ displayName,
+ queryRunner,
+ stream,
+ jsonColumns,
+ excludedColumns,
+ }: Omit<WriteRowsOptions, 'whereClause' | 'queryParameters'>): Promise<void> {
+ let columnNames: string[] | undefined;
+ let totalRows = 0;
+
+ for (let offset = 0; ; offset += BATCH_SIZE) {
+ const rows: Record<string, unknown>[] = await queryRunner.query(
+ `SELECT * FROM "${schemaName}"."${tableName}" ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
+ );
+
+ if (!isNonEmptyArray(rows)) break;
+
+ if (!columnNames) {
+ columnNames = Object.keys(rows[0]).filter(
+ (columnName) => !excludedColumns?.has(columnName),
+ );
+
+ const escapedColumns = columnNames.map(escapeIdentifier).join(', ');
+
+ stream.write(
+ `COPY ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumns}) FROM stdin;\n`,
+ );
+ }
+
+ totalRows += rows.length;
+
+ for (const row of rows) {
+ const values = columnNames.map((columnName) =>
+ formatPgCopyField(row[columnName], jsonColumns?.has(columnName)),
+ );
+
+ if (!stream.write(values.join('\t') + '\n')) {
+ await once(stream, 'drain');
+ }
+ }
+
+ if (rows.length < BATCH_SIZE) break;
+ }
+
+ if (isNonEmptyArray(columnNames)) {
+ stream.write('\\.\n\n');
+ }
+
+ if (totalRows > 0) {
+ this.logger.log(` ${displayName}: ${totalRows} rows`);
}
}
@@ -319,7 +376,7 @@ export class WorkspaceExportService {
);
try {
- await this.writeRows({
+ await this.writeCopyRows({
schemaName,
tableName,
displayName: objectMetadata.nameSingular,