Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): support multilingual full-text search

johaven9a462c501f32
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
index 41408154..13df0ff4 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
@@ -8,6 +8,14 @@ describe(FilesContentStoreMySQL.name, () => {
let filesIndexerMySQL: FilesContentStoreMySQL
let db: { execute: Mock }
+ const sqlText = (query: any): string => {
+ if (typeof query === 'string') return query
+ if (Array.isArray(query)) return query.map(sqlText).join('')
+ if (Array.isArray(query?.value)) return query.value.join('')
+ if (Array.isArray(query?.queryChunks)) return query.queryChunks.map(sqlText).join('')
+ return ''
+ }
+
beforeAll(async () => {
db = { execute: vi.fn() }
@@ -220,9 +228,106 @@ describe(FilesContentStoreMySQL.name, () => {
const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'foo', 10)
expect(db.execute).toHaveBeenCalledTimes(1)
+ const query = sqlText(db.execute.mock.calls[0][0])
+ expect(query).toContain('SELECT')
+ expect(query).not.toContain('path')
+ expect(query).not.toContain('content,')
+ expect(query).toContain('MATCH (content)')
+ expect(query).not.toContain('content LIKE')
expect(res).toEqual([])
})
+ it('should use only LIKE for a CJK-only search', async () => {
+ db.execute.mockResolvedValueOnce([[]])
+
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文', 10)
+
+ const query = sqlText(db.execute.mock.calls[0][0])
+ expect(query).toContain('content LIKE')
+ expect(query).not.toContain('MATCH (content)')
+ })
+
+ it('should use LIKE for non-segmented scripts and MATCH for Devanagari', async () => {
+ db.execute.mockResolvedValue([[]])
+
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'ภาษาไทย', 10)
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते मिलान', 10)
+
+ const thaiQuery = sqlText(db.execute.mock.calls[0][0])
+ const devanagariQuery = sqlText(db.execute.mock.calls[1][0])
+ expect(thaiQuery).toContain('content LIKE')
+ expect(thaiQuery).not.toContain('MATCH (content)')
+ expect(devanagariQuery).toContain('MATCH (content)')
+ expect(devanagariQuery).not.toContain('content LIKE')
+ })
+
+ it('should use only LIKE when a mixed search contains CJK', async () => {
+ db.execute.mockResolvedValueOnce([[]])
+
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'report 中文', 10)
+
+ const query = sqlText(db.execute.mock.calls[0][0])
+ expect(query).toContain('WHERE (content LIKE')
+ expect(query).not.toContain('MATCH (content)')
+ })
+
+ it('should translate required, optional, excluded and exact CJK terms', async () => {
+ db.execute.mockResolvedValue([[]])
+
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+中文 +文档 -秘密', 10)
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文 文档', 10)
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '"中文 文档"', 10)
+
+ const requiredQuery = sqlText(db.execute.mock.calls[0][0])
+ const optionalQuery = sqlText(db.execute.mock.calls[1][0])
+ const exactPhraseQuery = sqlText(db.execute.mock.calls[2][0])
+ expect(requiredQuery).toContain('WHERE (content LIKE')
+ expect(requiredQuery).toContain("ESCAPE '=' AND content LIKE")
+ expect(requiredQuery).toContain('content NOT LIKE')
+ expect(optionalQuery).toContain("ESCAPE '=' OR content LIKE")
+ // One LIKE in the score and one in WHERE means the quoted phrase was kept as one term.
+ expect(exactPhraseQuery.match(/content LIKE/g)).toHaveLength(2)
+ })
+
+ it('should keep identical ids from different indexes with their score order', async () => {
+ db.execute.mockResolvedValueOnce([
+ [
+ { id: 3, sourceIndex: 'files_content_u_1', score: 10 },
+ { id: 3, sourceIndex: 'files_content_s_2', score: 5 }
+ ]
+ ])
+ db.execute.mockResolvedValueOnce([
+ [
+ {
+ id: 3,
+ sourceIndex: 'files_content_s_2',
+ path: '/shared',
+ name: 'shared.txt',
+ mime: 'text/plain',
+ mtime: 1730000000001,
+ content: 'foo shared'
+ },
+ {
+ id: 3,
+ sourceIndex: 'files_content_u_1',
+ path: '/personal',
+ name: 'personal.txt',
+ mime: 'text/plain',
+ mtime: 1730000000000,
+ content: 'foo personal'
+ }
+ ]
+ ])
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo', 10)
+
+ expect(res).toHaveLength(2)
+ expect(res.map(({ name, score }) => [name, score])).toEqual([
+ ['personal.txt', 10],
+ ['shared.txt', 5]
+ ])
+ })
+
it('should search across tables, sort by score, and highlight matches', async () => {
// fabricate records returned by DB. Only first array (rows) is used.
const rows = [
@@ -245,12 +350,30 @@ describe(FilesContentStoreMySQL.name, () => {
score: 5
}
]
- db.execute.mockResolvedValueOnce([rows])
+ db.execute.mockResolvedValueOnce([
+ [
+ { id: 1, sourceIndex: 'files_content_u_1', score: 10 },
+ { id: 2, sourceIndex: 'files_content_s_2', score: 5 }
+ ]
+ ])
+ db.execute.mockResolvedValueOnce([
+ [
+ { ...rows[1], sourceIndex: 'files_content_s_2' },
+ { ...rows[0], sourceIndex: 'files_content_u_1' }
+ ]
+ ])
const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo bar', 10)
- expect(db.execute).toHaveBeenCalledTimes(1)
+ expect(db.execute).toHaveBeenCalledTimes(2)
+ const candidatesQuery = sqlText(db.execute.mock.calls[0][0])
+ const recordsQuery = sqlText(db.execute.mock.calls[1][0])
+ expect(candidatesQuery).not.toContain('path')
+ expect(candidatesQuery).not.toContain('content,')
+ expect(recordsQuery).toContain('path')
+ expect(recordsQuery).toContain('content')
expect(res.length).toBe(2)
+ expect(res.map((record) => record.id)).toEqual([1, 2])
// content must be cleared
expect(res[0].content).toBeUndefined()
expect(Array.isArray(res[0].matches)).toBe(true)
@@ -258,6 +381,29 @@ describe(FilesContentStoreMySQL.name, () => {
// highlighted with <mark> tags
expect(res[0].matches!.join(' ')).toMatch(/<mark>foo<\/mark>|<mark>bar<\/mark>/i)
})
+
+ it('should highlight CJK content returned by the fallback search', async () => {
+ db.execute.mockResolvedValueOnce([[{ id: 3, sourceIndex: 'files_content_u_1', score: 1 }]])
+ db.execute.mockResolvedValueOnce([
+ [
+ {
+ id: 3,
+ sourceIndex: 'files_content_u_1',
+ path: '/docs',
+ name: '日本語.txt',
+ mime: 'text/plain',
+ mtime: 1730000000002,
+ content: 'これは日本語の文書です。'
+ }
+ ]
+ ])
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], '日本語', 10)
+
+ expect(db.execute).toHaveBeenCalledTimes(2)
+ expect(res[0].content).toBeUndefined()
+ expect(res[0].matches).toEqual(['<mark>日本語</mark>の文書です。'])
+ })
})
describe('cleanIndexes', () => {
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index f5b06fa9..dca5be1a 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -6,7 +6,18 @@ import { DBSchema } from '../../../infrastructure/database/interfaces/database.i
import { FilesContentStore } from '../models/files-content-store'
import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'
import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'
-import { analyzeTerms, genTermsPattern, MaxSortedList } from '../utils/files-search'
+import {
+ analyzeTerms,
+ genTermsPattern,
+ likeSearchTermStartPattern,
+ MaxSortedList,
+ parseSearchTerms,
+ requiresLikeSearch,
+ SearchTerm
+} from '../utils/files-search'
+
+type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }
+type SearchRecord = FileContent & { sourceIndex: string }
@Injectable()
export class FilesContentStoreMySQL implements FilesContentStore {
@@ -118,34 +129,88 @@ export class FilesContentStoreMySQL implements FilesContentStore {
async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {
const terms: string[] = analyzeTerms(search)
+ const searchTerms = parseSearchTerms(search)
+ const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')
+ const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')
+ const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')
+ const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')
+ const useLikeSearch = searchTerms.some(({ value }) => requiresLikeSearch(value))
this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
if (!terms.length) {
return []
}
- // todo: use row iterator for better performance
- // mysql does not calculate MATCH results twice, can be used with select without worrying about performance
+ // Searches containing a script unsupported by FULLTEXT tokenization use LIKE for every term.
+ // Other searches keep the indexed FULLTEXT path.
const q: SQL = sql
.join(
- tableNames.map(
- (tableName) =>
- sql`(SELECT id, path, name, mime, mtime, content, MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE ) as score
+ tableNames.map((tableName) => {
+ if (useLikeSearch) {
+ const requiredMatch = createContentMatch(requiredTerms, ' AND ')
+ const optionalMatch = createContentMatch(optionalTerms, ' OR ')
+ const positiveMatch = requiredMatch || optionalMatch
+ const excludedMatch = createContentMatch(excludedTerms, ' AND ', true)
+ const score = positiveTerms.reduce<SQL>(
+ (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.value)} ESCAPE '=', 1, 0)`,
+ sql.raw('0')
+ )
+ return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as score
+ FROM ${sql.raw(tableName)}
+ WHERE (${positiveMatch})
+ ${excludedMatch ? sql`AND ${excludedMatch}` : sql``}
+ ORDER BY score DESC
+ LIMIT ${limit})`
+ }
+
+ const fullTextMatch = sql`MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE )`
+ return sql`(SELECT ${tableName} as sourceIndex, id, ${fullTextMatch} as score
FROM ${sql.raw(tableName)}
- WHERE MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE ) LIMIT ${limit})`
- ),
+ WHERE ${fullTextMatch}
+ ORDER BY score DESC
+ LIMIT ${limit})`
+ }),
sql.raw(' UNION ALL ')
)
.append(sql` ORDER BY score DESC LIMIT ${limit}`)
- const [records]: FileContent[][] = (await this.db.execute(q)) as MySqlQueryResult
- if (!records.length) {
+ const [candidateRecords]: SearchCandidate[][] = (await this.db.execute(q)) as MySqlQueryResult
+ const selectedCandidates = candidateRecords.slice(0, limit)
+ if (!selectedCandidates.length) {
return []
}
+ // Load LONGTEXT only for the final candidates to avoid carrying it through UNION and ORDER BY.
+ const idsByIndex = new Map<string, number[]>()
+ for (const candidate of selectedCandidates) {
+ const ids = idsByIndex.get(candidate.sourceIndex) || []
+ ids.push(candidate.id)
+ idsByIndex.set(candidate.sourceIndex, ids)
+ }
+ const recordsQuery = sql.join(
+ tableNames.flatMap((tableName) => {
+ const ids = idsByIndex.get(tableName)
+ if (!ids?.length) return []
+ return [
+ sql`SELECT ${tableName} as sourceIndex, id, path, name, mime, mtime, content
+ FROM ${sql.raw(tableName)}
+ WHERE id IN (${sql.join(
+ ids.map((id) => sql`${id}`),
+ sql.raw(', ')
+ )})`
+ ]
+ }),
+ sql.raw(' UNION ALL ')
+ )
+ const [loadedRecords]: SearchRecord[][] = (await this.db.execute(recordsQuery)) as MySqlQueryResult
+ const recordsByKey = new Map(loadedRecords.map((record) => [`${record.sourceIndex}:${record.id}`, record]))
+ const records = selectedCandidates.flatMap((candidate) => {
+ const record = recordsByKey.get(`${candidate.sourceIndex}:${candidate.id}`)
+ return record ? [{ ...record, score: candidate.score }] : []
+ })
+
const termsPattern = `(${genTermsPattern(terms)})`
- // const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]){0,20}\\b${termsPattern}(?:\\s*\\S*){0,20}`, 'gi') // best performance
- const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]{0,4}){0,10}\\b${termsPattern}(?:\\s*\\S*){0,15}`, 'gi')
+ const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`, 'giu')
- const termsHighlightRegexp = new RegExp(termsPattern, 'gi')
+ const termsHighlightRegexp = new RegExp(termsPattern, 'giu')
for (const r of records) {
const maxSortedList = new MaxSortedList(5)
for (const i of r.content.matchAll(termsRegexp)) {
@@ -153,11 +218,11 @@ export class FilesContentStoreMySQL implements FilesContentStore {
const nbDifferentWords: number = matches.length === 1 ? 1 : parseFloat(`${new Set(matches).size}.${matches.length}`)
maxSortedList.insert([nbDifferentWords, i[0]])
}
- // clear content
+ // Do not expose the full indexed content in search results.
r.content = undefined
r.matches = maxSortedList.data.map(([_nb, content]) => content.replace(termsHighlightRegexp, '<mark>$1</mark>'))
}
- return records
+ return records.map(({ sourceIndex: _sourceIndex, ...record }) => record)
}
async cleanIndexes(tableSuffixes: string[]): Promise<void> {
@@ -190,3 +255,17 @@ export class FilesContentStoreMySQL implements FilesContentStore {
await this.db.execute(sql`ALTER TABLE ${sql.raw(tableName)} ADD COLUMN seen_run_id varchar(64), ADD INDEX seen_run_id (seen_run_id)`)
}
}
+
+function toLikePattern(term: string): string {
+ return `%${term.replaceAll('=', '==').replaceAll('%', '=%').replaceAll('_', '=_')}%`
+}
+
+function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {
+ if (!terms.length) return null
+ return sql.join(
+ terms.map(({ value }) =>
+ negate ? sql`content NOT LIKE ${toLikePattern(value)} ESCAPE '='` : sql`content LIKE ${toLikePattern(value)} ESCAPE '='`
+ ),
+ sql.raw(separator)
+ )
+}
diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.ts
index 64ac66a9..4c2289f2 100644
--- a/backend/src/applications/files/constants/indexing.ts
+++ b/backend/src/applications/files/constants/indexing.ts
@@ -1,4 +1,4 @@
-export const MIN_CHARS_TO_SEARCH = 3
+export const MIN_CHARS_TO_SEARCH = 2
export const INDEXABLE_EXTENSIONS = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
export const CACHE_INDEXING_EVENT_PREFIX = 'event-update-indexing' as const
export const CACHE_INDEXING_EVENT_LAST_RUN_KEY = 'indexing-update-last-run' as const
diff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.ts
new file mode 100644
index 00000000..9b1391ad
--- /dev/null
+++ b/backend/src/applications/files/utils/files-search.spec.ts
@@ -0,0 +1,155 @@
+import {
+ analyzeTerms,
+ genRegexPositiveAndNegativeTerms,
+ genTermsPattern,
+ likeSearchTermStartPattern,
+ MaxSortedList,
+ parseSearchTerms,
+ requiresLikeSearch
+} from './files-search'
+
+describe('files search utilities', () => {
+ describe(MaxSortedList.name, () => {
+ it('should keep the highest scores in descending order', () => {
+ const list = new MaxSortedList(3)
+
+ list.insert([2, 'second'])
+ list.insert([1, 'first'])
+ list.insert([4, 'fourth'])
+ list.insert([3, 'third'])
+
+ expect(list.data).toEqual([
+ [4, 'fourth'],
+ [3, 'third'],
+ [2, 'second']
+ ])
+ })
+
+ it('should ignore lower and duplicate scores when full', () => {
+ const list = new MaxSortedList(2)
+ list.insert([3, 'first'])
+ list.insert([2, 'second'])
+
+ list.insert([1, 'lower'])
+ list.insert([2, 'duplicate'])
+
+ expect(list.data).toEqual([
+ [3, 'first'],
+ [2, 'second']
+ ])
+ })
+ })
+
+ describe(analyzeTerms.name, () => {
+ it('should return no terms for an empty or too short search', () => {
+ expect(analyzeTerms('')).toEqual([])
+ expect(analyzeTerms('a')).toEqual([])
+ })
+
+ it('should accept two-character CJK terms', () => {
+ expect(analyzeTerms('中文')).toEqual(['中文'])
+ })
+
+ it('should separate positive and negative terms', () => {
+ expect(analyzeTerms('+report -draft optional')).toEqual(['report', 'optional'])
+ expect(analyzeTerms('+report -draft optional', true)).toEqual(['draft'])
+ })
+
+ it('should remove boolean modifiers and trailing wildcards', () => {
+ expect(analyzeTerms('<report >draft ~lower *prefix file*')).toEqual(['report', 'draft', 'lower', 'prefix', 'file'])
+ })
+
+ it('should optionally escape regular expression characters', () => {
+ expect(analyzeTerms('file.txt')).toEqual(['file\\.txt'])
+ expect(analyzeTerms('file.txt', false, false)).toEqual(['file.txt'])
+ })
+ })
+
+ describe(genTermsPattern.name, () => {
+ it('should generate an accent-insensitive alternative pattern', () => {
+ const regexp = new RegExp(`^(${genTermsPattern(['resume', 'canyon'])})$`, 'iu')
+
+ expect(regexp.test('résumé')).toBe(true)
+ expect(regexp.test('cañyon')).toBe(true)
+ expect(regexp.test('other')).toBe(false)
+ })
+ })
+
+ describe(genRegexPositiveAndNegativeTerms.name, () => {
+ it('should match CJK terms without requiring ASCII word boundaries', () => {
+ const regexp = genRegexPositiveAndNegativeTerms('日本語')
+
+ expect(regexp.test('資料_日本語版.pdf')).toBe(true)
+ })
+
+ it('should use Unicode-aware boundaries for non-CJK terms', () => {
+ const regexp = genRegexPositiveAndNegativeTerms('файл')
+
+ expect(regexp.test('мой_файл.txt')).toBe(true)
+ expect(regexp.test('суперфайл.txt')).toBe(false)
+ })
+
+ it('should exclude negative CJK terms', () => {
+ const regexp = genRegexPositiveAndNegativeTerms('文書 -秘密')
+
+ expect(regexp.test('公開文書.txt')).toBe(true)
+ expect(regexp.test('秘密文書.txt')).toBe(false)
+ })
+
+ it('should exclude complete non-CJK terms without excluding partial words', () => {
+ const regexp = genRegexPositiveAndNegativeTerms('report -draft')
+
+ expect(regexp.test('report final.txt')).toBe(true)
+ expect(regexp.test('report draft.txt')).toBe(false)
+ expect(regexp.test('report drafting.txt')).toBe(true)
+ })
+ })
+
+ describe(requiresLikeSearch.name, () => {
+ it('should detect scripts requiring the LIKE fallback', () => {
+ expect(requiresLikeSearch('中文')).toBe(true)
+ expect(requiresLikeSearch('ภาษาไทย')).toBe(true)
+ expect(requiresLikeSearch('ພາສາລາວ')).toBe(true)
+ expect(requiresLikeSearch('ភាសាខ្មែរ')).toBe(true)
+ expect(requiresLikeSearch('မြန်မာစာ')).toBe(true)
+ })
+
+ it('should keep space-separated scripts on FULLTEXT', () => {
+ expect(requiresLikeSearch('खाते के मिलान में त्रुटि')).toBe(false)
+ expect(requiresLikeSearch('русский текст')).toBe(false)
+ expect(requiresLikeSearch('report')).toBe(false)
+ })
+ })
+
+ describe(parseSearchTerms.name, () => {
+ it('should classify boolean search terms and exact phrases', () => {
+ expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([
+ { value: '中文', operator: 'required' },
+ { value: '秘密', operator: 'excluded' },
+ { value: '文档', operator: 'optional' },
+ { value: '全文 搜索', operator: 'optional' }
+ ])
+ })
+
+ it('should remove nested modifiers and trailing wildcards', () => {
+ expect(parseSearchTerms('++report file*')).toEqual([
+ { value: 'report', operator: 'required' },
+ { value: 'file', operator: 'optional' }
+ ])
+ })
+
+ it('should ignore terms below the minimum length', () => {
+ expect(parseSearchTerms('+a valid')).toEqual([{ value: 'valid', operator: 'optional' }])
+ })
+ })
+
+ describe(likeSearchTermStartPattern.name, () => {
+ it('should match only before a character requiring the LIKE fallback', () => {
+ const regexp = new RegExp(`${likeSearchTermStartPattern()}.`, 'u')
+
+ expect(regexp.test('中文')).toBe(true)
+ expect(regexp.test('ภาษาไทย')).toBe(true)
+ expect(regexp.test('report')).toBe(false)
+ })
+ })
+})
diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.ts
index 3714f723..77da931e 100644
--- a/backend/src/applications/files/utils/files-search.ts
+++ b/backend/src/applications/files/utils/files-search.ts
@@ -4,7 +4,10 @@ import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'
const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)
const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')
const booleanOperators = new Set(['+', '-', '<', '>', '~', '*'])
-const BOUNDARY = '(?:\\b|_)'
+const UNICODE_WORD_CHAR = '[\\p{L}\\p{N}]'
+const LIKE_SEARCH_CHAR =
+ '[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]'
+const regexRequiresLikeSearch = new RegExp(LIKE_SEARCH_CHAR, 'u')
const accentToBaseMap = new Map<string, string>([
['a', '[aàáâä]'],
['e', '[eèéêë]'],
@@ -18,6 +21,11 @@ const accentToBaseMap = new Map<string, string>([
['y', '[yýÿ]']
])
+export interface SearchTerm {
+ value: string
+ operator: 'required' | 'excluded' | 'optional'
+}
+
export class MaxSortedList {
public data: [number, string][] = []
public nbItems: number
@@ -32,7 +40,7 @@ export class MaxSortedList {
return
}
// if score is smaller or the score already stored for another string ignore it and keep the first matches.
- if (item[0] < this.data[this.data.length - 1][0] || (this.data.length === this.nbItems && this.data.find(([num]) => num === item[0]))) {
+ if (this.data.length === this.nbItems && (item[0] < this.data[this.data.length - 1][0] || this.data.some(([num]) => num === item[0]))) {
return
}
// insert data ordered by highest score
@@ -49,7 +57,7 @@ export class MaxSortedList {
}
}
-export function analyzeTerms(search: string, onlyAllowNegative = false): string[] {
+export function analyzeTerms(search: string, onlyAllowNegative = false, escapeForRegexp = true): string[] {
/* Get the positive or negative terms list */
const matches: RegExpMatchArray | [] = search.match(regexMatchesSearchBoolean) || []
if (!matches.length) {
@@ -72,7 +80,7 @@ export function analyzeTerms(search: string, onlyAllowNegative = false): string[
term = term.substring(0, term.length - 1)
}
- return escapeString(term)
+ return escapeForRegexp ? escapeString(term) : term
})
.filter(Boolean)
}
@@ -86,14 +94,52 @@ export function genRegexPositiveAndNegativeTerms(search: string): RegExp {
const negativeTerms = analyzeTerms(search, true)
const p = positiveTerms
.map((t) => genAccentInsensitiveRegexpPattern(t))
- .map((t) => `(?=.*${BOUNDARY}${t})`)
+ .map((t) => `(?=.*${termBoundaryPattern(t)})`)
.join('')
- if (!negativeTerms.length) return new RegExp(p, 'i')
+ if (!negativeTerms.length) return new RegExp(p, 'iu')
const n = negativeTerms
.map((t) => genAccentInsensitiveRegexpPattern(t))
- .map((t) => `${BOUNDARY}${t}${BOUNDARY}`)
+ .map((t) => termBoundaryPattern(t, true))
.join('|')
- return new RegExp(`^${p}(?!.*(${n})).*$`, 'i')
+ return new RegExp(`^${p}(?!.*(${n})).*$`, 'iu')
+}
+
+export function requiresLikeSearch(input: string): boolean {
+ return regexRequiresLikeSearch.test(input)
+}
+
+export function parseSearchTerms(search: string): SearchTerm[] {
+ return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {
+ const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)
+ let value = (quoted || unquoted).trim()
+ while (booleanOperators.has(value[0])) {
+ value = value.substring(1)
+ }
+ if (value[value.length - 1] === '*') {
+ value = value.substring(0, value.length - 1)
+ }
+ if (value.length < MIN_CHARS_TO_SEARCH) {
+ return []
+ }
+ const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'
+ return [
+ {
+ value,
+ operator: searchOperator
+ }
+ ]
+ })
+}
+
+export function likeSearchTermStartPattern(): string {
+ return `(?=${LIKE_SEARCH_CHAR})`
+}
+
+function termBoundaryPattern(term: string, endBoundary = false): string {
+ if (requiresLikeSearch(term)) {
+ return term
+ }
+ return `(?<!${UNICODE_WORD_CHAR})${term}${endBoundary ? `(?!${UNICODE_WORD_CHAR})` : ''}`
}
function genAccentInsensitiveRegexpPattern(input: string): string {