Skip to content

The open alternative to Salesforce, designed for AI.

fix(search): restrict ILIKE fallback to CJK input (#26926)

Weikoba5df5da3a1f
diff --git a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts b/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts
index 97b5e37891e..268d7fe3e83 100644
--- a/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/search/services/search.service.ts
@@ -40,6 +40,7 @@ import {
} from 'src/engine/core-modules/search/exceptions/search.exception';
import { type RecordsWithObjectMetadataItem } from 'src/engine/core-modules/search/types/records-with-object-metadata-item.type';
import { formatSearchTerms } from 'src/engine/core-modules/search/utils/format-search-terms';
+import { hasCjkCharacters } from 'src/engine/core-modules/search/utils/has-cjk-characters';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { type IDataloaders } from 'src/engine/dataloaders/dataloader.interface';
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
@@ -203,11 +204,8 @@ export class SearchService {
});
}
- // Runs a fast tsvector query first (uses GIN index). If tsvector returns zero
- // results for an object type on the first page, falls back to ILIKE on the
- // searchVector text to catch cases where tokenization fails (e.g. CJK text).
- // Skipped when tsvector finds any results (partial results mean the data just
- // has fewer matches, not a tokenization issue) and on paginated requests.
+ // The simple text search configuration does not segment continuous CJK text,
+ // so substring matching is needed when the full-text search misses.
async buildSearchQueryAndGetRecordsWithFallback<
Entity extends ObjectLiteral,
>({
@@ -246,7 +244,7 @@ export class SearchService {
if (
tsvectorResults.length > 0 ||
- !isNonEmptyString(searchInput.trim()) ||
+ !hasCjkCharacters(searchInput) ||
isDefined(after)
) {
return tsvectorResults;
diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.ts b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.ts
new file mode 100644
index 00000000000..4ef704d01a3
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/search/utils/__tests__/has-cjk-characters.spec.ts
@@ -0,0 +1,31 @@
+import { hasCjkCharacters } from 'src/engine/core-modules/search/utils/has-cjk-characters';
+
+describe('hasCjkCharacters', () => {
+ it.each([
+ '商业',
+ 'ひらがな',
+ 'カタカナ',
+ '한국어',
+ 'Acme 商业',
+ 'Acmeカタカナ',
+ '𠀀',
+ 'カタカナ',
+ '한',
+ ])('returns true for %s', (text) => {
+ expect(hasCjkCharacters(text)).toBe(true);
+ });
+
+ it.each([
+ 'bignardi',
+ 'nardi',
+ 'café',
+ '12345',
+ '🙂',
+ 'Привет',
+ 'مرحبا',
+ '',
+ ' ',
+ ])('returns false for %s', (text) => {
+ expect(hasCjkCharacters(text)).toBe(false);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.ts b/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.ts
new file mode 100644
index 00000000000..0d1ec23d671
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/search/utils/has-cjk-characters.ts
@@ -0,0 +1,5 @@
+const CJK_CHARACTER_PATTERN =
+ /[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]/u;
+
+export const hasCjkCharacters = (text: string) =>
+ CJK_CHARACTER_PATTERN.test(text);
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
index f698d3abce4..547006cfbeb 100644
--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
@@ -2513,7 +2513,7 @@ export class ConfigVariables {
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.ADVANCED_SETTINGS,
description:
- 'Timeout in milliseconds for the search ILIKE fallback query per searchable object. Triggered only when the tsvector query returns 0 results on the first page (e.g. CJK input). When the timeout fires the fallback is skipped for that object.',
+ 'Timeout in milliseconds for the search ILIKE fallback query per searchable object. Triggered only for input containing CJK characters when the tsvector query returns 0 results on the first page. When the timeout fires the fallback is skipped for that object.',
type: ConfigVariableType.NUMBER,
isEnvOnly: true,
})