Skip to content

The open alternative to Salesforce, designed for AI.

feat(metadata): adopt SYSTEM writability, starting with search vectors (#24335)

Abdul Rahman8a163323c2c2
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts
index a8c49946959..c73e610c6be 100644
--- a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts
@@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';
import { BackfillActivityTargetsJunctionTargetCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787123540000-backfill-activity-targets-junction-target.command';
import { MigrateCommandMenuItemLabelsToPlaceholdersCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787127900000-migrate-command-menu-item-labels-to-placeholders.command';
+import { MarkSearchVectorFieldsSystemCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command';
import { ApplicationModule } from 'src/engine/core-modules/application/application.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@@ -21,6 +22,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace
],
providers: [
BackfillActivityTargetsJunctionTargetCommand,
+ MarkSearchVectorFieldsSystemCommand,
MigrateCommandMenuItemLabelsToPlaceholdersCommand,
],
})
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.ts
new file mode 100644
index 00000000000..89af86b6ea7
--- /dev/null
+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.ts
@@ -0,0 +1,101 @@
+import { Command } from 'nest-commander';
+
+import groupBy from 'lodash.groupby';
+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
+
+import { ProvisionedWorkspaceCommandRunner } from 'src/database/commands/command-runners/provisioned-workspace.command-runner';
+import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';
+import { type RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspace.command-runner';
+import { RegisteredWorkspaceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-workspace-command.decorator';
+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
+import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service';
+
+@RegisteredWorkspaceCommand('2.33.0', 1787138325228)
+@Command({
+ name: 'upgrade:2-33:mark-search-vector-fields-system',
+ description:
+ 'Mark searchVector fields as SYSTEM writability on existing workspaces: they are Postgres generated columns nothing may write',
+})
+export class MarkSearchVectorFieldsSystemCommand extends ProvisionedWorkspaceCommandRunner {
+ constructor(
+ protected readonly workspaceIteratorService: WorkspaceIteratorService,
+ private readonly workspaceCacheService: WorkspaceCacheService,
+ private readonly workspaceMigrationValidateBuildAndRunService: WorkspaceMigrationValidateBuildAndRunService,
+ ) {
+ super(workspaceIteratorService);
+ }
+
+ override async runOnWorkspace({
+ workspaceId,
+ options,
+ }: RunOnWorkspaceArgs): Promise<void> {
+ const isDryRun = options.dryRun ?? false;
+
+ const { flatFieldMetadataMaps } =
+ await this.workspaceCacheService.getOrRecompute(workspaceId, [
+ 'flatFieldMetadataMaps',
+ ]);
+
+ const flatFieldMetadatasToUpdate = Object.values(
+ flatFieldMetadataMaps.byUniversalIdentifier,
+ )
+ .filter(isDefined)
+ .filter(
+ (flatFieldMetadata) =>
+ flatFieldMetadata.type === FieldMetadataType.TS_VECTOR &&
+ flatFieldMetadata.writability !== MetadataWritability.SYSTEM,
+ )
+ .map((flatFieldMetadata) => ({
+ ...flatFieldMetadata,
+ writability: MetadataWritability.SYSTEM,
+ }));
+
+ if (flatFieldMetadatasToUpdate.length === 0) {
+ return;
+ }
+
+ this.logger.log(
+ `${isDryRun ? '[DRY RUN] Would mark' : 'Marking'} ${flatFieldMetadatasToUpdate.length} search vector field(s) as SYSTEM for workspace ${workspaceId}`,
+ );
+
+ const flatFieldMetadatasToUpdateByApplicationUniversalIdentifier = groupBy(
+ flatFieldMetadatasToUpdate,
+ (flatFieldMetadata) => flatFieldMetadata.applicationUniversalIdentifier,
+ );
+
+ for (const [
+ applicationUniversalIdentifier,
+ flatEntityToUpdate,
+ ] of Object.entries(
+ flatFieldMetadatasToUpdateByApplicationUniversalIdentifier,
+ )) {
+ const validateAndBuildResult =
+ await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(
+ {
+ isSystemBuild: true,
+ workspaceId,
+ applicationUniversalIdentifier,
+ dryRun: isDryRun,
+ allFlatEntityOperationByMetadataName: {
+ fieldMetadata: {
+ flatEntityToCreate: [],
+ flatEntityToDelete: [],
+ flatEntityToUpdate,
+ },
+ },
+ },
+ );
+
+ if (validateAndBuildResult.status === 'fail') {
+ this.logger.error(
+ `Failed to mark search vector fields as SYSTEM:\n${JSON.stringify(validateAndBuildResult, null, 2)}`,
+ );
+
+ throw new Error(
+ `Failed to mark search vector fields as SYSTEM for workspace ${workspaceId}`,
+ );
+ }
+ }
+ }
+}
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.ts
new file mode 100644
index 00000000000..0852e45f55c
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.ts
@@ -0,0 +1,67 @@
+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';
+
+import { validateTsVectorFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util';
+import { type UniversalFlatEntityUpdate } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-entity-update.type';
+import { type UniversalFlatFieldMetadata } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-field-metadata.type';
+
+const createFlatEntityToValidate = (
+ overrides: Partial<
+ UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>
+ > = {},
+): UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR> =>
+ ({
+ type: FieldMetadataType.TS_VECTOR,
+ name: 'searchVector',
+ isSystem: true,
+ writability: MetadataWritability.SYSTEM,
+ ...overrides,
+ }) as UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>;
+
+const callValidator = (
+ flatEntityToValidate: UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>,
+ update?: UniversalFlatEntityUpdate<'fieldMetadata'>,
+) =>
+ validateTsVectorFlatFieldMetadata({
+ flatEntityToValidate,
+ update,
+ } as Parameters<typeof validateTsVectorFlatFieldMetadata>[0]);
+
+describe('validateTsVectorFlatFieldMetadata', () => {
+ it('should return no errors for a SYSTEM writability search vector', () => {
+ const errors = callValidator(createFlatEntityToValidate());
+
+ expect(errors).toEqual([]);
+ });
+
+ it('should reject a create with non-SYSTEM writability', () => {
+ const errors = callValidator(
+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),
+ );
+
+ expect(errors).toHaveLength(1);
+ expect(errors[0].message).toBe(
+ 'Field type TS_VECTOR must have SYSTEM writability',
+ );
+ });
+
+ it('should reject an update that sets non-SYSTEM writability', () => {
+ const errors = callValidator(
+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),
+ { writability: MetadataWritability.OPEN },
+ );
+
+ expect(errors).toHaveLength(1);
+ expect(errors[0].message).toBe(
+ 'Field type TS_VECTOR must have SYSTEM writability',
+ );
+ });
+
+ it('should accept an update leaving writability untouched on a not-yet-migrated OPEN search vector', () => {
+ const errors = callValidator(
+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),
+ { isActive: false },
+ );
+
+ expect(errors).toEqual([]);
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts
index cd8e14a6235..7072699ecc4 100644
--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts
@@ -1,5 +1,9 @@
import { msg } from '@lingui/core/macro';
-import { type FieldMetadataType } from 'twenty-shared/types';
+import {
+ MetadataWritability,
+ type FieldMetadataType,
+} from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
import { FieldMetadataExceptionCode } from 'src/engine/metadata-modules/field-metadata/field-metadata.exception';
import { type FlatFieldMetadataTypeValidationArgs } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata-type-validator.type';
@@ -7,6 +11,7 @@ import { type FlatFieldMetadataValidationError } from 'src/engine/metadata-modul
export const validateTsVectorFlatFieldMetadata = ({
flatEntityToValidate,
+ update,
}: FlatFieldMetadataTypeValidationArgs<FieldMetadataType.TS_VECTOR>): FlatFieldMetadataValidationError[] => {
const errors: FlatFieldMetadataValidationError[] = [];
@@ -28,5 +33,20 @@ export const validateTsVectorFlatFieldMetadata = ({
});
}
+ const isUpdateLeavingWritabilityUntouched =
+ isDefined(update) && !('writability' in update);
+
+ if (
+ !isUpdateLeavingWritabilityUntouched &&
+ flatEntityToValidate.writability !== MetadataWritability.SYSTEM
+ ) {
+ errors.push({
+ code: FieldMetadataExceptionCode.INVALID_FIELD_INPUT,
+ message: 'Field type TS_VECTOR must have SYSTEM writability',
+ value: flatEntityToValidate.writability,
+ userFriendlyMessage: msg`Field type TS_VECTOR must have SYSTEM writability`,
+ });
+ }
+
return errors;
};
diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts
index 220754e5464..a8ba222ec9c 100644
--- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts
@@ -249,7 +249,7 @@ const PARTIAL_SEARCH_VECTOR_FIELD = {
isSystem: true,
isSystemSideEffect: true,
isUIEditable: false,
- writability: MetadataWritability.OPEN,
+ writability: MetadataWritability.SYSTEM,
isLabelSyncedWithName: false,
defaultValue: null,
options: null,
diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.ts
new file mode 100644
index 00000000000..a1447dd01d9
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.ts
@@ -0,0 +1,16 @@
+import { MetadataWritability } from 'twenty-shared/types';
+
+import { buildSearchVectorFlatFieldMetadataForCustomObject } from 'src/engine/metadata-modules/object-metadata/utils/build-search-vector-flat-field-metadata-for-custom-object.util';
+
+describe('buildSearchVectorFlatFieldMetadataForCustomObject', () => {
+ const searchVector = buildSearchVectorFlatFieldMetadataForCustomObject({
+ flatObjectMetadata: {
+ universalIdentifier: '20202020-1111-4111-8111-111111111111',
+ applicationUniversalIdentifier: '20202020-2222-4222-8222-222222222222',
+ },
+ });
+
+ it('marks the search vector field as SYSTEM, since Postgres generates the column', () => {
+ expect(searchVector.writability).toBe(MetadataWritability.SYSTEM);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.ts b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.ts
new file mode 100644
index 00000000000..a423a51a792
--- /dev/null
+++ b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.ts
@@ -0,0 +1,51 @@
+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';
+import { isDefined } from 'twenty-shared/utils';
+
+import { computeTwentyStandardApplicationAllFlatEntityMaps } from 'src/engine/workspace-manager/twenty-standard-application/utils/twenty-standard-application-all-flat-entity-maps.constant';
+
+const WORKSPACE_ID = '20202020-1111-4111-8111-111111111111';
+const TWENTY_STANDARD_APPLICATION_ID = '20202020-2222-4222-8222-222222222222';
+const NOW = '2024-01-01T00:00:00.000Z';
+
+describe('Standard field writability', () => {
+ const { allFlatEntityMaps } =
+ computeTwentyStandardApplicationAllFlatEntityMaps({
+ now: NOW,
+ workspaceId: WORKSPACE_ID,
+ twentyStandardApplicationId: TWENTY_STANDARD_APPLICATION_ID,
+ });
+
+ const standardFlatFieldMetadatas = Object.values(
+ allFlatEntityMaps.flatFieldMetadataMaps.byUniversalIdentifier,
+ ).filter(isDefined);
+
+ const searchVectorFlatFieldMetadatas = standardFlatFieldMetadatas.filter(
+ (flatFieldMetadata) =>
+ flatFieldMetadata.type === FieldMetadataType.TS_VECTOR,
+ );
+
+ it('marks every search vector field as SYSTEM, since Postgres generates the column', () => {
+ expect(searchVectorFlatFieldMetadatas.length).toBeGreaterThan(0);
+ expect(
+ searchVectorFlatFieldMetadatas.every(
+ (flatFieldMetadata) =>
+ flatFieldMetadata.writability === MetadataWritability.SYSTEM,
+ ),
+ ).toBe(true);
+ });
+
+ it('leaves every other standard field OPEN', () => {
+ const nonSearchVectorWritabilities = new Set(
+ standardFlatFieldMetadatas
+ .filter(
+ (flatFieldMetadata) =>
+ flatFieldMetadata.type !== FieldMetadataType.TS_VECTOR,
+ )
+ .map((flatFieldMetadata) => flatFieldMetadata.writability),
+ );
+
+ expect([...nonSearchVectorWritabilities]).toEqual([
+ MetadataWritability.OPEN,
+ ]);
+ });
+});
diff --git a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts
index 450642e42aa..7ce62c446f7 100644
--- a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts
@@ -88,7 +88,12 @@ export const createStandardFieldFlatMetadata = <
isNullable,
isUnique,
isUIEditable,
- writability: MetadataWritability.OPEN,
+ writability:
+ name in PARTIAL_SYSTEM_FLAT_FIELD_METADATAS
+ ? PARTIAL_SYSTEM_FLAT_FIELD_METADATAS[
+ name as keyof typeof PARTIAL_SYSTEM_FLAT_FIELD_METADATAS
+ ].writability
+ : MetadataWritability.OPEN,
isLabelSyncedWithName: false,
overrides: null,
defaultValue: defaultValue ?? null,
diff --git a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap
index 81a13367cb8..e305ba99e8b 100644
--- a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap
+++ b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap
@@ -10,9 +10,15 @@ exports[`Sync application should reject hijacking the searchVector system field
"errors": [
{
"code": "FIELD_MUTATION_NOT_ALLOWED",
- "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, isUIEditable",
+ "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, isUIEditable, writability",
"userFriendlyMessage": "System fields cannot be updated",
},
+ {
+ "code": "INVALID_FIELD_INPUT",
+ "message": "Field type TS_VECTOR must have SYSTEM writability",
+ "userFriendlyMessage": "Field type TS_VECTOR must have SYSTEM writability",
+ "value": "OPEN",
+ },
],
"flatEntityMinimalInformation": {
"name": "searchVector",
@@ -122,7 +128,7 @@ exports[`Sync application should reject hijacking the searchVector system field
"errors": [
{
"code": "FIELD_MUTATION_NOT_ALLOWED",
- "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, label, name, isUIEditable",
+ "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, label, name, isUIEditable, writability",
"userFriendlyMessage": "System fields cannot be updated",
},
{
@@ -131,6 +137,12 @@ exports[`Sync application should reject hijacking the searchVector system field
"userFriendlyMessage": "Field type TS_VECTOR must be named "searchVector"",
"value": "stolenSearchVector",
},
+ {
+ "code": "INVALID_FIELD_INPUT",
+ "message": "Field type TS_VECTOR must have SYSTEM writability",
+ "userFriendlyMessage": "Field type TS_VECTOR must have SYSTEM writability",
+ "value": "OPEN",
+ },
],
"flatEntityMinimalInformation": {
"name": "stolenSearchVector",
diff --git a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap
index b47fe15fca6..2a5ba24e3ad 100644
--- a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap
+++ b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap
@@ -393,7 +393,7 @@ exports[`syncApplication should delete old field and create equivalent one when
"universalIdentifier": Any<String>,
"universalSettings": null,
"updatedAt": Any<String>,
- "writability": "OPEN",
+ "writability": "SYSTEM",
},
],
},
@@ -1863,7 +1863,7 @@ exports[`syncApplication should return workspace migration actions on initial sy
"universalIdentifier": Any<String>,
"universalSettings": null,
"updatedAt": Any<String>,
- "writability": "OPEN",
+ "writability": "SYSTEM",
},
],
},