The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(search): restrict ILIKE fallback to CJK input (#26926)
Weikoba5df5da3a1f
diff --git a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts b/packages/twenty-server/src/engine/core-modules/search/services/search.service.tsindex 97b5e37891e..268d7fe3e83 100644--- a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts+++ b/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts@@ -40,6 +40,7 @@ import {} from 'src/engine/core-modules/search/exceptions/search.exception';import { type RecordsWithObjectMetadataItem } from 'src/engine/core-modules/search/types/records-with-object-metadata-item.type';import { formatSearchTerms } from 'src/engine/core-modules/search/utils/format-search-terms';+import { hasCjkCharacters } from 'src/engine/core-modules/search/utils/has-cjk-characters';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';import { type IDataloaders } from 'src/engine/dataloaders/dataloader.interface';import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';@@ -203,11 +204,8 @@ export class SearchService {});}- // Runs a fast tsvector query first (uses GIN index). If tsvector returns zero- // results for an object type on the first page, falls back to ILIKE on the- // searchVector text to catch cases where tokenization fails (e.g. CJK text).- // Skipped when tsvector finds any results (partial results mean the data just- // has fewer matches, not a tokenization issue) and on paginated requests.+ // The simple text search configuration does not segment continuous CJK text,+ // so substring matching is needed when the full-text search misses.async buildSearchQueryAndGetRecordsWithFallback<Entity extends ObjectLiteral,>({@@ -246,7 +244,7 @@ export class SearchService {if (tsvectorResults.length > 0 ||- !isNonEmptyString(searchInput.trim()) ||+ !hasCjkCharacters(searchInput) ||isDefined(after)) {return tsvectorResults;diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.ts b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.tsnew file mode 100644index 00000000000..4ef704d01a3--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.ts@@ -0,0 +1,31 @@+import { hasCjkCharacters } from 'src/engine/core-modules/search/utils/has-cjk-characters';++describe('hasCjkCharacters', () => {+ it.each([+ '商业',+ 'ひらがな',+ 'カタカナ',+ '한국어',+ 'Acme 商业',+ 'Acmeカタカナ',+ '𠀀',+ 'カタカナ',+ '한',+ ])('returns true for %s', (text) => {+ expect(hasCjkCharacters(text)).toBe(true);+ });++ it.each([+ 'bignardi',+ 'nardi',+ 'café',+ '12345',+ '🙂',+ 'Привет',+ 'مرحبا',+ '',+ ' ',+ ])('returns false for %s', (text) => {+ expect(hasCjkCharacters(text)).toBe(false);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.ts b/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.tsnew file mode 100644index 00000000000..0d1ec23d671--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.ts@@ -0,0 +1,5 @@+const CJK_CHARACTER_PATTERN =+ /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]/u;++export const hasCjkCharacters = (text: string) =>+ CJK_CHARACTER_PATTERN.test(text);diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex f698d3abce4..547006cfbeb 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -2513,7 +2513,7 @@ export class ConfigVariables {@ConfigVariablesMetadata({group: ConfigVariablesGroup.ADVANCED_SETTINGS,description:- 'Timeout in milliseconds for the search ILIKE fallback query per searchable object. Triggered only when the tsvector query returns 0 results on the first page (e.g. CJK input). When the timeout fires the fallback is skipped for that object.',+ 'Timeout in milliseconds for the search ILIKE fallback query per searchable object. Triggered only for input containing CJK characters when the tsvector query returns 0 results on the first page. When the timeout fires the fallback is skipped for that object.',type: ConfigVariableType.NUMBER,isEnvOnly: true,})