The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(metadata): adopt SYSTEM writability, starting with search vectors (#24335)
Abdul Rahman8a163323c2c2
diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.tsindex a8c49946959..c73e610c6be 100644--- a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-upgrade-version-command.module.ts@@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';import { WorkspaceIteratorModule } from 'src/database/commands/command-runners/workspace-iterator.module';import { BackfillActivityTargetsJunctionTargetCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787123540000-backfill-activity-targets-junction-target.command';import { MigrateCommandMenuItemLabelsToPlaceholdersCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787127900000-migrate-command-menu-item-labels-to-placeholders.command';+import { MarkSearchVectorFieldsSystemCommand } from 'src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command';import { ApplicationModule } from 'src/engine/core-modules/application/application.module';import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';@@ -21,6 +22,7 @@ import { WorkspaceMigrationModule } from 'src/engine/workspace-manager/workspace],providers: [BackfillActivityTargetsJunctionTargetCommand,+ MarkSearchVectorFieldsSystemCommand,MigrateCommandMenuItemLabelsToPlaceholdersCommand,],})diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.tsnew file mode 100644index 00000000000..89af86b6ea7--- /dev/null+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/2-33/2-33-workspace-command-1787138325228-mark-search-vector-fields-system.command.ts@@ -0,0 +1,101 @@+import { Command } from 'nest-commander';++import groupBy from 'lodash.groupby';+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';++import { ProvisionedWorkspaceCommandRunner } from 'src/database/commands/command-runners/provisioned-workspace.command-runner';+import { WorkspaceIteratorService } from 'src/database/commands/command-runners/workspace-iterator.service';+import { type RunOnWorkspaceArgs } from 'src/database/commands/command-runners/workspace.command-runner';+import { RegisteredWorkspaceCommand } from 'src/engine/core-modules/upgrade/decorators/registered-workspace-command.decorator';+import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';+import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration/services/workspace-migration-validate-build-and-run-service';++@RegisteredWorkspaceCommand('2.33.0', 1787138325228)+@Command({+ name: 'upgrade:2-33:mark-search-vector-fields-system',+ description:+ 'Mark searchVector fields as SYSTEM writability on existing workspaces: they are Postgres generated columns nothing may write',+})+export class MarkSearchVectorFieldsSystemCommand extends ProvisionedWorkspaceCommandRunner {+ constructor(+ protected readonly workspaceIteratorService: WorkspaceIteratorService,+ private readonly workspaceCacheService: WorkspaceCacheService,+ private readonly workspaceMigrationValidateBuildAndRunService: WorkspaceMigrationValidateBuildAndRunService,+ ) {+ super(workspaceIteratorService);+ }++ override async runOnWorkspace({+ workspaceId,+ options,+ }: RunOnWorkspaceArgs): Promise<void> {+ const isDryRun = options.dryRun ?? false;++ const { flatFieldMetadataMaps } =+ await this.workspaceCacheService.getOrRecompute(workspaceId, [+ 'flatFieldMetadataMaps',+ ]);++ const flatFieldMetadatasToUpdate = Object.values(+ flatFieldMetadataMaps.byUniversalIdentifier,+ )+ .filter(isDefined)+ .filter(+ (flatFieldMetadata) =>+ flatFieldMetadata.type === FieldMetadataType.TS_VECTOR &&+ flatFieldMetadata.writability !== MetadataWritability.SYSTEM,+ )+ .map((flatFieldMetadata) => ({+ ...flatFieldMetadata,+ writability: MetadataWritability.SYSTEM,+ }));++ if (flatFieldMetadatasToUpdate.length === 0) {+ return;+ }++ this.logger.log(+ `${isDryRun ? '[DRY RUN] Would mark' : 'Marking'} ${flatFieldMetadatasToUpdate.length} search vector field(s) as SYSTEM for workspace ${workspaceId}`,+ );++ const flatFieldMetadatasToUpdateByApplicationUniversalIdentifier = groupBy(+ flatFieldMetadatasToUpdate,+ (flatFieldMetadata) => flatFieldMetadata.applicationUniversalIdentifier,+ );++ for (const [+ applicationUniversalIdentifier,+ flatEntityToUpdate,+ ] of Object.entries(+ flatFieldMetadatasToUpdateByApplicationUniversalIdentifier,+ )) {+ const validateAndBuildResult =+ await this.workspaceMigrationValidateBuildAndRunService.validateBuildAndRunWorkspaceMigration(+ {+ isSystemBuild: true,+ workspaceId,+ applicationUniversalIdentifier,+ dryRun: isDryRun,+ allFlatEntityOperationByMetadataName: {+ fieldMetadata: {+ flatEntityToCreate: [],+ flatEntityToDelete: [],+ flatEntityToUpdate,+ },+ },+ },+ );++ if (validateAndBuildResult.status === 'fail') {+ this.logger.error(+ `Failed to mark search vector fields as SYSTEM:\n${JSON.stringify(validateAndBuildResult, null, 2)}`,+ );++ throw new Error(+ `Failed to mark search vector fields as SYSTEM for workspace ${workspaceId}`,+ );+ }+ }+ }+}diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.tsnew file mode 100644index 00000000000..0852e45f55c--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/__tests__/validate-ts-vector-flat-field-metadata.util.spec.ts@@ -0,0 +1,67 @@+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';++import { validateTsVectorFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util';+import { type UniversalFlatEntityUpdate } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-entity-update.type';+import { type UniversalFlatFieldMetadata } from 'src/engine/workspace-manager/workspace-migration/universal-flat-entity/types/universal-flat-field-metadata.type';++const createFlatEntityToValidate = (+ overrides: Partial<+ UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>+ > = {},+): UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR> =>+ ({+ type: FieldMetadataType.TS_VECTOR,+ name: 'searchVector',+ isSystem: true,+ writability: MetadataWritability.SYSTEM,+ ...overrides,+ }) as UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>;++const callValidator = (+ flatEntityToValidate: UniversalFlatFieldMetadata<FieldMetadataType.TS_VECTOR>,+ update?: UniversalFlatEntityUpdate<'fieldMetadata'>,+) =>+ validateTsVectorFlatFieldMetadata({+ flatEntityToValidate,+ update,+ } as Parameters<typeof validateTsVectorFlatFieldMetadata>[0]);++describe('validateTsVectorFlatFieldMetadata', () => {+ it('should return no errors for a SYSTEM writability search vector', () => {+ const errors = callValidator(createFlatEntityToValidate());++ expect(errors).toEqual([]);+ });++ it('should reject a create with non-SYSTEM writability', () => {+ const errors = callValidator(+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),+ );++ expect(errors).toHaveLength(1);+ expect(errors[0].message).toBe(+ 'Field type TS_VECTOR must have SYSTEM writability',+ );+ });++ it('should reject an update that sets non-SYSTEM writability', () => {+ const errors = callValidator(+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),+ { writability: MetadataWritability.OPEN },+ );++ expect(errors).toHaveLength(1);+ expect(errors[0].message).toBe(+ 'Field type TS_VECTOR must have SYSTEM writability',+ );+ });++ it('should accept an update leaving writability untouched on a not-yet-migrated OPEN search vector', () => {+ const errors = callValidator(+ createFlatEntityToValidate({ writability: MetadataWritability.OPEN }),+ { isActive: false },+ );++ expect(errors).toEqual([]);+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.tsindex cd8e14a6235..7072699ecc4 100644--- a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/validators/utils/validate-ts-vector-flat-field-metadata.util.ts@@ -1,5 +1,9 @@import { msg } from '@lingui/core/macro';-import { type FieldMetadataType } from 'twenty-shared/types';+import {+ MetadataWritability,+ type FieldMetadataType,+} from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';import { FieldMetadataExceptionCode } from 'src/engine/metadata-modules/field-metadata/field-metadata.exception';import { type FlatFieldMetadataTypeValidationArgs } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata-type-validator.type';@@ -7,6 +11,7 @@ import { type FlatFieldMetadataValidationError } from 'src/engine/metadata-modulexport const validateTsVectorFlatFieldMetadata = ({flatEntityToValidate,+ update,}: FlatFieldMetadataTypeValidationArgs<FieldMetadataType.TS_VECTOR>): FlatFieldMetadataValidationError[] => {const errors: FlatFieldMetadataValidationError[] = [];@@ -28,5 +33,20 @@ export const validateTsVectorFlatFieldMetadata = ({});}+ const isUpdateLeavingWritabilityUntouched =+ isDefined(update) && !('writability' in update);++ if (+ !isUpdateLeavingWritabilityUntouched &&+ flatEntityToValidate.writability !== MetadataWritability.SYSTEM+ ) {+ errors.push({+ code: FieldMetadataExceptionCode.INVALID_FIELD_INPUT,+ message: 'Field type TS_VECTOR must have SYSTEM writability',+ value: flatEntityToValidate.writability,+ userFriendlyMessage: msg`Field type TS_VECTOR must have SYSTEM writability`,+ });+ }+return errors;};diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.tsindex 220754e5464..a8ba222ec9c 100644--- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts+++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/constants/partial-system-flat-field-metadatas.constant.ts@@ -249,7 +249,7 @@ const PARTIAL_SEARCH_VECTOR_FIELD = {isSystem: true,isSystemSideEffect: true,isUIEditable: false,- writability: MetadataWritability.OPEN,+ writability: MetadataWritability.SYSTEM,isLabelSyncedWithName: false,defaultValue: null,options: null,diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.tsnew file mode 100644index 00000000000..a1447dd01d9--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/utils/__tests__/build-search-vector-flat-field-metadata-for-custom-object.util.spec.ts@@ -0,0 +1,16 @@+import { MetadataWritability } from 'twenty-shared/types';++import { buildSearchVectorFlatFieldMetadataForCustomObject } from 'src/engine/metadata-modules/object-metadata/utils/build-search-vector-flat-field-metadata-for-custom-object.util';++describe('buildSearchVectorFlatFieldMetadataForCustomObject', () => {+ const searchVector = buildSearchVectorFlatFieldMetadataForCustomObject({+ flatObjectMetadata: {+ universalIdentifier: '20202020-1111-4111-8111-111111111111',+ applicationUniversalIdentifier: '20202020-2222-4222-8222-222222222222',+ },+ });++ it('marks the search vector field as SYSTEM, since Postgres generates the column', () => {+ expect(searchVector.writability).toBe(MetadataWritability.SYSTEM);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.ts b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.tsnew file mode 100644index 00000000000..a423a51a792--- /dev/null+++ b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/__tests__/standard-field-writability.spec.ts@@ -0,0 +1,51 @@+import { FieldMetadataType, MetadataWritability } from 'twenty-shared/types';+import { isDefined } from 'twenty-shared/utils';++import { computeTwentyStandardApplicationAllFlatEntityMaps } from 'src/engine/workspace-manager/twenty-standard-application/utils/twenty-standard-application-all-flat-entity-maps.constant';++const WORKSPACE_ID = '20202020-1111-4111-8111-111111111111';+const TWENTY_STANDARD_APPLICATION_ID = '20202020-2222-4222-8222-222222222222';+const NOW = '2024-01-01T00:00:00.000Z';++describe('Standard field writability', () => {+ const { allFlatEntityMaps } =+ computeTwentyStandardApplicationAllFlatEntityMaps({+ now: NOW,+ workspaceId: WORKSPACE_ID,+ twentyStandardApplicationId: TWENTY_STANDARD_APPLICATION_ID,+ });++ const standardFlatFieldMetadatas = Object.values(+ allFlatEntityMaps.flatFieldMetadataMaps.byUniversalIdentifier,+ ).filter(isDefined);++ const searchVectorFlatFieldMetadatas = standardFlatFieldMetadatas.filter(+ (flatFieldMetadata) =>+ flatFieldMetadata.type === FieldMetadataType.TS_VECTOR,+ );++ it('marks every search vector field as SYSTEM, since Postgres generates the column', () => {+ expect(searchVectorFlatFieldMetadatas.length).toBeGreaterThan(0);+ expect(+ searchVectorFlatFieldMetadatas.every(+ (flatFieldMetadata) =>+ flatFieldMetadata.writability === MetadataWritability.SYSTEM,+ ),+ ).toBe(true);+ });++ it('leaves every other standard field OPEN', () => {+ const nonSearchVectorWritabilities = new Set(+ standardFlatFieldMetadatas+ .filter(+ (flatFieldMetadata) =>+ flatFieldMetadata.type !== FieldMetadataType.TS_VECTOR,+ )+ .map((flatFieldMetadata) => flatFieldMetadata.writability),+ );++ expect([...nonSearchVectorWritabilities]).toEqual([+ MetadataWritability.OPEN,+ ]);+ });+});diff --git a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.tsindex 450642e42aa..7ce62c446f7 100644--- a/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts+++ b/packages/twenty-server/src/engine/workspace-manager/twenty-standard-application/utils/field-metadata/create-standard-field-flat-metadata.util.ts@@ -88,7 +88,12 @@ export const createStandardFieldFlatMetadata = <isNullable,isUnique,isUIEditable,- writability: MetadataWritability.OPEN,+ writability:+ name in PARTIAL_SYSTEM_FLAT_FIELD_METADATAS+ ? PARTIAL_SYSTEM_FLAT_FIELD_METADATAS[+ name as keyof typeof PARTIAL_SYSTEM_FLAT_FIELD_METADATAS+ ].writability+ : MetadataWritability.OPEN,isLabelSyncedWithName: false,overrides: null,defaultValue: defaultValue ?? null,diff --git a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snapindex 81a13367cb8..e305ba99e8b 100644--- a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap+++ b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/failing-sync-application-search-vector-identifier-hijack.integration-spec.ts.snap@@ -10,9 +10,15 @@ exports[`Sync application should reject hijacking the searchVector system field"errors": [{"code": "FIELD_MUTATION_NOT_ALLOWED",- "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, isUIEditable",+ "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, isUIEditable, writability","userFriendlyMessage": "System fields cannot be updated",},+ {+ "code": "INVALID_FIELD_INPUT",+ "message": "Field type TS_VECTOR must have SYSTEM writability",+ "userFriendlyMessage": "Field type TS_VECTOR must have SYSTEM writability",+ "value": "OPEN",+ },],"flatEntityMinimalInformation": {"name": "searchVector",@@ -122,7 +128,7 @@ exports[`Sync application should reject hijacking the searchVector system field"errors": [{"code": "FIELD_MUTATION_NOT_ALLOWED",- "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, label, name, isUIEditable",+ "message": "System fields only allow updating: universalSettings, isActive. Forbidden properties: description, icon, label, name, isUIEditable, writability","userFriendlyMessage": "System fields cannot be updated",},{@@ -131,6 +137,12 @@ exports[`Sync application should reject hijacking the searchVector system field"userFriendlyMessage": "Field type TS_VECTOR must be named "searchVector"","value": "stolenSearchVector",},+ {+ "code": "INVALID_FIELD_INPUT",+ "message": "Field type TS_VECTOR must have SYSTEM writability",+ "userFriendlyMessage": "Field type TS_VECTOR must have SYSTEM writability",+ "value": "OPEN",+ },],"flatEntityMinimalInformation": {"name": "stolenSearchVector",diff --git a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snapindex b47fe15fca6..2a5ba24e3ad 100644--- a/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap+++ b/packages/twenty-server/test/integration/metadata/suites/application/__snapshots__/successful-sync-application-workspace-migration.integration-spec.ts.snap@@ -393,7 +393,7 @@ exports[`syncApplication should delete old field and create equivalent one when"universalIdentifier": Any<String>,"universalSettings": null,"updatedAt": Any<String>,- "writability": "OPEN",+ "writability": "SYSTEM",},],},@@ -1863,7 +1863,7 @@ exports[`syncApplication should return workspace migration actions on initial sy"universalIdentifier": Any<String>,"universalSettings": null,"updatedAt": Any<String>,- "writability": "OPEN",+ "writability": "SYSTEM",},],},