Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): improve full-text search

johaven7ff6a8b7005c
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.md b/backend/src/applications/files/adapters/files-content-store-mysql.service.md
new file mode 100644
index 00000000..d6e971db
--- /dev/null
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.md
@@ -0,0 +1,73 @@
+# Content search with MariaDB
+
+`FilesContentStoreMySQL` adapts user input to the Boolean mode of
+`MATCH ... AGAINST`, then builds snippets from the selected documents.
+
+## Search syntax
+
+| Input | Behavior |
+|--------------------|--------------------------------------------------------------------------|
+| `report budget` | Optional terms: at least one of them must match. |
+| `+report` | Required term. |
+| `-draft` | Excluded term. A search must still contain at least one positive term. |
+| `test+` or `test-` | The invalid trailing operator is removed before querying MariaDB. |
+| `set*` | Prefix search; `setting` may match. Only a suffix wildcard is supported. |
+
+Sequences of `+` and `-` that do not target a term are removed outside quoted phrases. For example, `test+` becomes `test` and `test+,` becomes
+`test,`. Operators placed at the beginning of a term retain their Boolean meaning.
+
+## Compound terms and quoted phrases
+
+An unquoted term containing a separator between multiple tokens is automatically converted into a MariaDB phrase:
+
+| Input | Normalized query |
+|----------------------|------------------------|
+| `set-variable` | `"set-variable"` |
+| `2017-03-05` | `"2017-03-05"` |
+| `+set-variable` | `+"set-variable"` |
+
+An already quoted phrase, such as `"configure set-variable now"`, remains unchanged. In MariaDB, quotes require a sequence of **tokens**, not a
+character-for-character match: `"set-variable"` may therefore match `set variable`. Punctuation (`-`, `@`, `.`, etc.) generally acts as a separator
+for the FULLTEXT tokenizer.
+
+## Scripts without FULLTEXT segmentation
+
+If at least one term contains a Han, Hiragana, Katakana, Hangul, Thai, Lao, Khmer, or Myanmar character, the entire query uses `LIKE`:
+
+- required terms are combined with `AND`;
+- optional terms are combined with `OR`;
+- exclusions use `NOT LIKE`;
+- `%`, `_`, and the escape character are escaped.
+
+This fallback enables searches without an `ngram` tokenizer, but it scans content and is more expensive than a FULLTEXT index.
+
+## SQL execution
+
+The search runs in two phases:
+
+1. MariaDB selects only `sourceIndex`, `id`, and `score` from each table, then applies sorting and the limit to the `UNION ALL` result.
+2. A second query loads metadata and `LONGTEXT` only for the selected identifiers. The order calculated during the first phase is restored in memory.
+
+The full content is then used to generate snippets, but it is never returned in the search result.
+
+## Snippets and highlighting
+
+- Only positive terms are highlighted with `<mark>`.
+- A document returns at most 5 snippets.
+- A snippet keeps at most 10 words before and 15 words after the match, with safety limits of 512 and 768 characters.
+- Nearby matches are grouped. Snippets containing the most distinct terms, followed by the highest number of occurrences, are prioritized.
+- A compound term finds and highlights the separators actually present in the content: `set-variable` may produce
+ `<mark>set variable</mark>`.
+- Highlighting handles common Latin variants with or without accents, such as `resume` and `résumé`. SQL search also depends on the table collation
+ (`utf8mb4_general_ci`).
+- With a suffix wildcard, the end-of-word boundary is removed from highlighting.
+
+## Known limitations
+
+- The HTTP query is limited to 512 characters; the result limit defaults to 100 and cannot exceed 100.
+- Terms shorter than 2 characters are ignored by the application.
+- Stopwords and the minimum indexed token length depend on the MariaDB configuration and storage engine. A term accepted by the application may
+ therefore be absent from the index.
+- FULLTEXT scores depend on each table's statistics. A score from a user index is not strictly comparable with one from a space or share index.
+- A MariaDB phrase compares consecutive tokens; it does not guarantee that the punctuation entered matches the document's punctuation.
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
index a84bf836..5effb56f 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
@@ -18,6 +18,14 @@ describe(FilesContentStoreMySQL.name, () => {
return ''
}
+ const mockSingleSearchRecord = (content: string): void => {
+ const id = 1
+ const sourceIndex = 'files_content_u_1'
+ db.execute
+ .mockResolvedValueOnce([[{ id, sourceIndex, score: 1 }]])
+ .mockResolvedValueOnce([[{ id, sourceIndex, path: '/docs', name: 'document.txt', mime: 'text/plain', mtime: 1730000000000, content }]])
+ }
+
beforeAll(async () => {
db = { execute: vi.fn() }
@@ -255,28 +263,29 @@ describe(FilesContentStoreMySQL.name, () => {
expect(res).toEqual([])
})
- it('should use only LIKE for a CJK-only search', async () => {
+ it('should remove trailing boolean operators before querying FULLTEXT', async () => {
db.execute.mockResolvedValueOnce([[]])
- await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文', 10)
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+required optional+ excluded- "C++ guide"', 10)
- const query = sqlText(db.execute.mock.calls[0][0])
- expect(query).toContain('content LIKE')
- expect(query).not.toContain('MATCH (content)')
+ const query = dialect.sqlToQuery(db.execute.mock.calls[0][0])
+ expect(query.params).toContain('+required optional excluded "C++ guide"')
+ expect(query.params).not.toContain('+required optional+ excluded- "C++ guide"')
})
- it('should use LIKE for non-segmented scripts and MATCH for Devanagari', async () => {
- db.execute.mockResolvedValue([[]])
+ it.each([
+ ['report -set-variable', 'report -"set-variable"'],
+ ['2017-03-05', '"2017-03-05"'],
+ ['"configure set-variable now"', '"configure set-variable now"'],
+ ['test+,', 'test,']
+ ])('should normalize the FULLTEXT search %s as %s', async (search, normalizedSearch) => {
+ db.execute.mockResolvedValueOnce([[]])
- await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'ภาษาไทย', 10)
- await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते मिलान', 10)
+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], search, 10)
- const thaiQuery = sqlText(db.execute.mock.calls[0][0])
- const devanagariQuery = sqlText(db.execute.mock.calls[1][0])
- expect(thaiQuery).toContain('content LIKE')
- expect(thaiQuery).not.toContain('MATCH (content)')
- expect(devanagariQuery).toContain('MATCH (content)')
- expect(devanagariQuery).not.toContain('content LIKE')
+ const query = dialect.sqlToQuery(db.execute.mock.calls[0][0])
+ expect(query.params.filter((param) => param === normalizedSearch)).toHaveLength(2)
})
it('should use only LIKE when a mixed search contains CJK', async () => {
@@ -289,68 +298,24 @@ describe(FilesContentStoreMySQL.name, () => {
expect(query).not.toContain('MATCH (content)')
})
- it('should translate required, optional, excluded and exact CJK terms', async () => {
+ it('should translate required, optional and excluded CJK terms', async () => {
db.execute.mockResolvedValue([[]])
await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+中文 +文档 -秘密', 10)
await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文 文档', 10)
- await filesIndexerMySQL.searchRecords(['files_content_u_1'], '"中文 文档"', 10)
const requiredQuery = sqlText(db.execute.mock.calls[0][0])
const optionalQuery = sqlText(db.execute.mock.calls[1][0])
- const exactPhraseQuery = sqlText(db.execute.mock.calls[2][0])
expect(requiredQuery).toContain('WHERE (content LIKE')
expect(requiredQuery).toContain("ESCAPE '=' AND content LIKE")
expect(requiredQuery).toContain('content NOT LIKE')
expect(optionalQuery).toContain("ESCAPE '=' OR content LIKE")
- // One LIKE in the score and one in WHERE means the quoted phrase was kept as one term.
- expect(exactPhraseQuery.match(/content LIKE/g)).toHaveLength(2)
- })
-
- it('should keep identical ids from different indexes with their score order', async () => {
- db.execute.mockResolvedValueOnce([
- [
- { id: 3, sourceIndex: 'files_content_u_1', score: 10 },
- { id: 3, sourceIndex: 'files_content_s_2', score: 5 }
- ]
- ])
- db.execute.mockResolvedValueOnce([
- [
- {
- id: 3,
- sourceIndex: 'files_content_s_2',
- path: '/shared',
- name: 'shared.txt',
- mime: 'text/plain',
- mtime: 1730000000001,
- content: 'foo shared'
- },
- {
- id: 3,
- sourceIndex: 'files_content_u_1',
- path: '/personal',
- name: 'personal.txt',
- mime: 'text/plain',
- mtime: 1730000000000,
- content: 'foo personal'
- }
- ]
- ])
-
- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo', 10)
-
- expect(res).toHaveLength(2)
- expect(res.map(({ name, score }) => [name, score])).toEqual([
- ['personal.txt', 10],
- ['shared.txt', 5]
- ])
})
- it('should search across tables, sort by score, and highlight matches', async () => {
- // fabricate records returned by DB. Only first array (rows) is used.
+ it('should load final content across indexes, preserve score order, and highlight matches', async () => {
const rows = [
{
- id: 1,
+ id: 3,
path: '/docs',
name: 'alpha.txt',
mime: 'text/plain',
@@ -359,7 +324,7 @@ describe(FilesContentStoreMySQL.name, () => {
score: 10
},
{
- id: 2,
+ id: 3,
path: '/docs',
name: 'beta.txt',
mime: 'text/plain',
@@ -370,8 +335,8 @@ describe(FilesContentStoreMySQL.name, () => {
]
db.execute.mockResolvedValueOnce([
[
- { id: 1, sourceIndex: 'files_content_u_1', score: 10 },
- { id: 2, sourceIndex: 'files_content_s_2', score: 5 }
+ { id: 3, sourceIndex: 'files_content_u_1', score: 10 },
+ { id: 3, sourceIndex: 'files_content_s_2', score: 5 }
]
])
db.execute.mockResolvedValueOnce([
@@ -391,30 +356,18 @@ describe(FilesContentStoreMySQL.name, () => {
expect(recordsQuery).toContain('path')
expect(recordsQuery).toContain('content')
expect(res.length).toBe(2)
- expect(res.map((record) => record.id)).toEqual([1, 2])
- // content must be cleared
+ expect(res.map(({ name, score }) => [name, score])).toEqual([
+ ['alpha.txt', 10],
+ ['beta.txt', 5]
+ ])
expect(res[0].content).toBeUndefined()
expect(Array.isArray(res[0].matches)).toBe(true)
expect(res[0].matches!.length).toBeGreaterThan(0)
- // highlighted with <mark> tags
expect(res[0].matches!.join(' ')).toMatch(/<mark>foo<\/mark>|<mark>bar<\/mark>/i)
})
it('should highlight CJK content returned by the fallback search', async () => {
- db.execute.mockResolvedValueOnce([[{ id: 3, sourceIndex: 'files_content_u_1', score: 1 }]])
- db.execute.mockResolvedValueOnce([
- [
- {
- id: 3,
- sourceIndex: 'files_content_u_1',
- path: '/docs',
- name: '日本語.txt',
- mime: 'text/plain',
- mtime: 1730000000002,
- content: 'これは日本語の文書です。'
- }
- ]
- ])
+ mockSingleSearchRecord('これは日本語の文書です。')
const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], '日本語', 10)
@@ -423,50 +376,73 @@ describe(FilesContentStoreMySQL.name, () => {
expect(res[0].matches).toEqual(['<mark>日本語</mark>の文書です。'])
})
- it('should highlight hyphenated full-text matches', async () => {
- db.execute.mockResolvedValueOnce([[{ id: 4, sourceIndex: 'files_content_u_1', score: 1 }]])
- db.execute.mockResolvedValueOnce([
- [
- {
- id: 4,
- sourceIndex: 'files_content_u_1',
- path: '/docs',
- name: 'euro-office.txt',
- mime: 'text/plain',
- mtime: 1730000000003,
- content: 'Le dossier euro-office est prêt.'
- }
- ]
- ])
+ it('should highlight a tokenized compound before its shorter alternative', async () => {
+ mockSingleSearchRecord('The command can set variable values.')
- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'euro-office', 10)
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'set set-variable', 10)
- expect(db.execute).toHaveBeenCalledTimes(2)
- expect(res[0].content).toBeUndefined()
- expect(res[0].matches).toEqual(['Le dossier <mark>euro-office</mark> est prêt.'])
+ expect(res[0].matches).toEqual(['The command can <mark>set variable</mark> values.'])
})
- it('should keep Unicode words in the highlighted match context', async () => {
- db.execute.mockResolvedValueOnce([[{ id: 5, sourceIndex: 'files_content_u_1', score: 1 }]])
- db.execute.mockResolvedValueOnce([
- [
- {
- id: 5,
- sourceIndex: 'files_content_u_1',
- path: '/docs',
- name: 'resume.txt',
- mime: 'text/plain',
- mtime: 1730000000004,
- content: 'Le résumé final mentionne budget.'
- }
- ]
- ])
+ it('should bound the highlighted context around a match', async () => {
+ const before = Array.from({ length: 12 }, (_, index) => `before${index}`).join(' ')
+ const after = Array.from({ length: 17 }, (_, index) => `after${index}`).join(' ')
+ mockSingleSearchRecord(`${before} target ${after}.`)
- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'budget', 10)
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'target', 10)
+ const [match] = res[0].matches
- expect(db.execute).toHaveBeenCalledTimes(2)
- expect(res[0].content).toBeUndefined()
- expect(res[0].matches).toEqual(['Le résumé final mentionne <mark>budget</mark>.'])
+ expect(match).toMatch(/^before2 /)
+ expect(match).toContain('<mark>target</mark>')
+ expect(match).toContain('after14')
+ expect(match).not.toContain('after15')
+ })
+
+ it('should highlight every occurrence inside one context', async () => {
+ mockSingleSearchRecord('target target target')
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'target', 10)
+
+ expect(res[0].matches).toEqual(['<mark>target</mark> <mark>target</mark> <mark>target</mark>'])
+ })
+
+ it('should highlight the FULLTEXT token when trailing operators precede punctuation', async () => {
+ mockSingleSearchRecord('The test is available.')
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'test+,', 10)
+
+ expect(res[0].matches).toEqual(['The <mark>test</mark> is available.'])
+ })
+
+ it('should highlight an unaccented match from an accented search', async () => {
+ mockSingleSearchRecord('Le resume final est disponible.')
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'résumé', 10)
+
+ expect(res[0].matches).toEqual(['Le <mark>resume</mark> final est disponible.'])
+ })
+
+ it('should highlight a Unicode term containing combining marks', async () => {
+ mockSingleSearchRecord('यह खाते का विवरण है।')
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते', 10)
+
+ expect(res[0].matches[0]).toContain('<mark>खाते</mark>')
+ })
+
+ it.each([
+ ['set', '<mark>set</mark> value', '<mark>set</mark>ting'],
+ ['set*', '<mark>set</mark>ting', null]
+ ] as const)('should respect the FULLTEXT wildcard boundary for %s', async (search, highlightedTerm, rejectedHighlight) => {
+ mockSingleSearchRecord('A setting followed by a set value.')
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], search, 10)
+ const matches = res[0].matches.join(' ')
+
+ expect(matches).toContain(highlightedTerm)
+ if (rejectedHighlight) {
+ expect(matches).not.toContain(rejectedHighlight)
+ }
})
})
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index ba710a23..240a1ae6 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -4,19 +4,42 @@ import { MySqlQueryResult } from 'drizzle-orm/mysql2'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
import { FILE_REPOSITORY } from '../constants/operations'
+import type { FilesSearchTerm } from '../interfaces/files-search-query.interface'
import { FilesContentStore } from '../models/files-content-store'
import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'
import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'
-import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseSearchTerms, SearchTerm } from '../utils/files-search'
+import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseFilesSearchQuery } from '../utils/files-search'
type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }
type SearchRecord = FileContent & { sourceIndex: string }
-const HIGHLIGHT_CONTEXT_WORD = '(?<![\\p{L}\\p{N}])[\\p{L}\\p{N}]+(?![\\p{L}\\p{N}])'
-const HIGHLIGHT_CONTEXT_SEPARATOR = '[^\\p{L}\\p{N}]'
+interface HighlightMatch {
+ index: number
+ value: string
+}
+interface HighlightContext {
+ start: number
+ end: number
+ matches: HighlightMatch[]
+}
+const HIGHLIGHT_WORD_CHAR = '[\\p{L}\\p{M}\\p{N}]'
+const HIGHLIGHT_TERM_START = `(?<!${HIGHLIGHT_WORD_CHAR})`
+const HIGHLIGHT_TERM_END = `(?!${HIGHLIGHT_WORD_CHAR})`
+const HIGHLIGHT_CONTEXT_SEPARATOR = '[^\\p{L}\\p{M}\\p{N}]'
+const HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN = /[\p{L}\p{M}\p{N}]/uy
+const HIGHLIGHT_CONTEXT_WORDS_BEFORE = 10
+const HIGHLIGHT_CONTEXT_WORDS_AFTER = 15
+const HIGHLIGHT_CONTEXT_CHARS_BEFORE = 512
+const HIGHLIGHT_CONTEXT_CHARS_AFTER = 768
const FILES_CONTENT_TABLE_PATTERN = new RegExp(`^${FILES_CONTENT_TABLE_PREFIX}(?:${Object.values(FILE_REPOSITORY).join('|')})_[0-9]+$`)
@Injectable()
export class FilesContentStoreMySQL implements FilesContentStore {
+ private static readonly INVALID_TRAILING_FULL_TEXT_OPERATORS = new Set(['+', '-'])
+ private static readonly FULL_TEXT_WHITESPACE_PATTERN = /\s/u
+ private static readonly FULL_TEXT_TERM_CHARACTER_PATTERN = /[\p{L}\p{M}\p{N}]/u
+ private static readonly FULL_TEXT_COMPOUND_TERM_PATTERN =
+ /(^|[\s(])([+-]?)([\p{L}\p{M}\p{N}]+(?:[^\p{L}\p{M}\p{N}\s"]+[\p{L}\p{M}\p{N}]+)+)(?=[\s),.!?;:]|$)/gu
+ private static readonly FULL_TEXT_TERM_TOKEN_PATTERN = /[\p{L}\p{M}\p{N}]+/gu
private readonly logger = new Logger(FilesContentStoreMySQL.name)
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
@@ -36,8 +59,8 @@ export class FilesContentStoreMySQL implements FilesContentStore {
}
async existingIndexes(tableSuffixes: string[]): Promise<string[]> {
- const currentTables = await this.indexesList()
- return tableSuffixes.map((suffix) => this.getIndexName(suffix)).filter((table) => currentTables.indexOf(table) > -1)
+ const currentTables = new Set(await this.indexesList())
+ return tableSuffixes.map((suffix) => this.getIndexName(suffix)).filter((table) => currentTables.has(table))
}
async createIndex(tableName: string): Promise<boolean> {
@@ -130,14 +153,11 @@ export class FilesContentStoreMySQL implements FilesContentStore {
}
async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {
- const searchTerms = parseSearchTerms(search)
- const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')
- const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')
- const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')
- const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')
+ const normalizedSearch = FilesContentStoreMySQL.normalizeFullTextSearch(search)
+ const { terms: searchTerms, positiveTerms, requiredTerms, optionalTerms, excludedTerms } = parseFilesSearchQuery(normalizedSearch)
const terms = positiveTerms.map(({ regexpValue }) => regexpValue)
const useLikeSearch = searchTerms.some(({ requiresLike }) => requiresLike)
- this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
+ this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${normalizedSearch} -> ${JSON.stringify(terms)}` })
if (!terms.length) {
return []
}
@@ -151,9 +171,9 @@ export class FilesContentStoreMySQL implements FilesContentStore {
const optionalMatch = createContentMatch(optionalTerms, ' OR ')
const positiveMatch = requiredMatch || optionalMatch
const excludedMatch = createContentMatch(excludedTerms, ' AND ', true)
- const score = positiveTerms.reduce<SQL>(
+ const score = optionalTerms.reduce<SQL>(
(value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.rawValue)} ESCAPE '=', 1, 0)`,
- sql`0`
+ sql`${requiredTerms.length}`
)
return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as score
FROM ${sql.identifier(tableName)}
@@ -163,7 +183,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
LIMIT ${limit})`
}
- const fullTextMatch = sql`MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE )`
+ const fullTextMatch = sql`MATCH (content) AGAINST ( ${normalizedSearch} IN BOOLEAN MODE )`
return sql`(SELECT ${tableName} as sourceIndex, id, ${fullTextMatch} as score
FROM ${sql.identifier(tableName)}
WHERE ${fullTextMatch}
@@ -175,14 +195,13 @@ export class FilesContentStoreMySQL implements FilesContentStore {
.append(sql` ORDER BY score DESC LIMIT ${limit}`)
const [candidateRecords]: SearchCandidate[][] = (await this.db.execute(q)) as MySqlQueryResult
- const selectedCandidates = candidateRecords.slice(0, limit)
- if (!selectedCandidates.length) {
+ if (!candidateRecords.length) {
return []
}
// Load LONGTEXT only for the final candidates to avoid carrying it through UNION and ORDER BY.
const idsByIndex = new Map<string, number[]>()
- for (const candidate of selectedCandidates) {
+ for (const candidate of candidateRecords) {
const ids = idsByIndex.get(candidate.sourceIndex) || []
ids.push(candidate.id)
idsByIndex.set(candidate.sourceIndex, ids)
@@ -201,28 +220,16 @@ export class FilesContentStoreMySQL implements FilesContentStore {
)
const [loadedRecords]: SearchRecord[][] = (await this.db.execute(recordsQuery)) as MySqlQueryResult
const recordsByKey = new Map(loadedRecords.map((record) => [`${record.sourceIndex}:${record.id}`, record]))
- const records = selectedCandidates.flatMap((candidate) => {
+ const records = candidateRecords.flatMap((candidate) => {
const record = recordsByKey.get(`${candidate.sourceIndex}:${candidate.id}`)
return record ? [{ ...record, score: candidate.score }] : []
})
- const termsPattern = `(${genTermsPattern(terms)})`
- const termsRegexp = new RegExp(
- `(?:${HIGHLIGHT_CONTEXT_WORD}${HIGHLIGHT_CONTEXT_SEPARATOR}{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`,
- 'giu'
- )
-
- const termsHighlightRegexp = new RegExp(termsPattern, 'giu')
+ const termsHighlightRegexp = new RegExp(`(${FilesContentStoreMySQL.genFullTextHighlightPattern(positiveTerms)})`, 'giu')
for (const r of records) {
- const maxSortedList = new MaxSortedList(5)
- for (const i of r.content.matchAll(termsRegexp)) {
- const matches: string[] = i[0].match(termsHighlightRegexp).map((term) => term.toLowerCase())
- const nbDifferentWords: number = matches.length === 1 ? 1 : parseFloat(`${new Set(matches).size}.${matches.length}`)
- maxSortedList.insert([nbDifferentWords, i[0]])
- }
+ r.matches = FilesContentStoreMySQL.extractHighlightedMatches(r.content, termsHighlightRegexp)
// Do not expose the full indexed content in search results.
r.content = undefined
- r.matches = maxSortedList.data.map(([_nb, content]) => content.replace(termsHighlightRegexp, '<mark>$1</mark>'))
}
return records.map(({ sourceIndex: _sourceIndex, ...record }) => record)
}
@@ -244,6 +251,169 @@ export class FilesContentStoreMySQL implements FilesContentStore {
}
}
+ private static normalizeFullTextSearch(search: string): string {
+ const searchWithQuotedCompounds = this.quoteUnquotedFullTextCompoundTerms(search)
+ let normalizedSearch = ''
+ let quoted = false
+ for (let index = 0; index < searchWithQuotedCompounds.length; index++) {
+ const character = searchWithQuotedCompounds[index]
+ if (character === '"') {
+ quoted = !quoted
+ normalizedSearch += character
+ continue
+ }
+ if (!quoted && this.INVALID_TRAILING_FULL_TEXT_OPERATORS.has(character)) {
+ let operatorEnd = index + 1
+ while (this.INVALID_TRAILING_FULL_TEXT_OPERATORS.has(searchWithQuotedCompounds[operatorEnd])) {
+ operatorEnd++
+ }
+ const previousCharacter = searchWithQuotedCompounds[index - 1]
+ const isLeadingOperator = index === 0 || previousCharacter === '(' || this.FULL_TEXT_WHITESPACE_PATTERN.test(previousCharacter)
+ const nextCodePoint = searchWithQuotedCompounds.codePointAt(operatorEnd)
+ const nextCharacter = nextCodePoint === undefined ? undefined : String.fromCodePoint(nextCodePoint)
+ const hasValidTarget =
+ nextCharacter !== undefined &&
+ (this.FULL_TEXT_TERM_CHARACTER_PATTERN.test(nextCharacter) || (isLeadingOperator && (nextCharacter === '(' || nextCharacter === '"')))
+ if (!hasValidTarget) {
+ index = operatorEnd - 1
+ continue
+ }
+ normalizedSearch += isLeadingOperator ? character : searchWithQuotedCompounds.slice(index, operatorEnd)
+ index = operatorEnd - 1
+ continue
+ }
+ normalizedSearch += character
+ }
+ return normalizedSearch
+ }
+
+ private static quoteUnquotedFullTextCompoundTerms(search: string): string {
+ let normalizedSearch = ''
+ let quoted = false
+ let segmentStart = 0
+ for (let index = 0; index < search.length; index++) {
+ if (search[index] !== '"') continue
+ const segment = search.slice(segmentStart, index)
+ normalizedSearch += quoted ? segment : segment.replace(this.FULL_TEXT_COMPOUND_TERM_PATTERN, '$1$2"$3"')
+ normalizedSearch += '"'
+ quoted = !quoted
+ segmentStart = index + 1
+ }
+ const segment = search.slice(segmentStart)
+ return normalizedSearch + (quoted ? segment : segment.replace(this.FULL_TEXT_COMPOUND_TERM_PATTERN, '$1$2"$3"'))
+ }
+
+ private static genFullTextHighlightPattern(terms: FilesSearchTerm[]): string {
+ const termStart = `(?:${HIGHLIGHT_TERM_START}|${likeSearchTermStartPattern()})`
+ const patterns = terms.map(({ rawValue, regexpValue, requiresLike, wildcard }) => {
+ const tokens = rawValue.match(this.FULL_TEXT_TERM_TOKEN_PATTERN) || []
+ const termPattern = tokens.length
+ ? tokens.map((token) => genTermsPattern([token])).join(`${HIGHLIGHT_CONTEXT_SEPARATOR}+`)
+ : genTermsPattern([regexpValue])
+ return `${termStart}${termPattern}${requiresLike || wildcard ? '' : HIGHLIGHT_TERM_END}`
+ })
+ return [...new Set(patterns)].sort((left, right) => right.length - left.length).join('|')
+ }
+
+ private static extractHighlightedMatches(content: string, termsRegexp: RegExp): string[] {
+ const maxSortedList = new MaxSortedList<HighlightContext>(5)
+ let previousContextEnd = 0
+ termsRegexp.lastIndex = 0
+ let match = termsRegexp.exec(content)
+ while (match !== null) {
+ const contextStart = Math.max(this.findHighlightContextStart(content, match.index), previousContextEnd)
+ let contextEnd = this.findHighlightContextEnd(content, match.index + match[0].length)
+ const matches: HighlightMatch[] = [{ index: match.index, value: match[0] }]
+ const distinctTerms = new Set([match[0].toLowerCase()])
+
+ let nextMatch = termsRegexp.exec(content)
+ while (nextMatch !== null && nextMatch.index < contextEnd) {
+ matches.push({ index: nextMatch.index, value: nextMatch[0] })
+ distinctTerms.add(nextMatch[0].toLowerCase())
+ contextEnd = Math.max(contextEnd, nextMatch.index + nextMatch[0].length)
+ nextMatch = termsRegexp.exec(content)
+ }
+ const score = distinctTerms.size * 1000 + Math.min(matches.length, 999)
+ maxSortedList.insert([score, { start: contextStart, end: contextEnd, matches }])
+ previousContextEnd = contextEnd
+ match = nextMatch
+ }
+ termsRegexp.lastIndex = 0
+ return maxSortedList.data.map(([_score, context]) => this.renderHighlightedContext(content, context))
+ }
+
+ private static renderHighlightedContext(content: string, context: HighlightContext): string {
+ const highlightedParts: string[] = []
+ let cursor = context.start
+ for (const match of context.matches) {
+ highlightedParts.push(content.slice(cursor, match.index), '<mark>', match.value, '</mark>')
+ cursor = match.index + match.value.length
+ }
+ highlightedParts.push(content.slice(cursor, context.end))
+ return highlightedParts.join('')
+ }
+
+ private static findHighlightContextStart(content: string, matchStart: number): number {
+ if (matchStart === 0) return 0
+ const previousIndex = this.previousCharacterIndex(content, matchStart)
+ if (this.isHighlightWordCharacterAt(content, previousIndex)) return matchStart
+
+ const minIndex = Math.max(0, matchStart - HIGHLIGHT_CONTEXT_CHARS_BEFORE)
+ let index = matchStart
+ let insideWord = false
+ let words = 0
+ while (index > minIndex) {
+ const characterIndex = this.previousCharacterIndex(content, index)
+ const isWordCharacter = this.isHighlightWordCharacterAt(content, characterIndex)
+ if (!isWordCharacter && insideWord && ++words === HIGHLIGHT_CONTEXT_WORDS_BEFORE) break
+ index = characterIndex
+ insideWord = isWordCharacter
+ }
+ return index
+ }
+
+ private static findHighlightContextEnd(content: string, matchEnd: number): number {
+ const maxIndex = Math.min(content.length, matchEnd + HIGHLIGHT_CONTEXT_CHARS_AFTER)
+ let index = matchEnd
+ let insideWord = matchEnd > 0 && this.isHighlightWordCharacterAt(content, this.previousCharacterIndex(content, matchEnd))
+ let words = 0
+ while (index < maxIndex) {
+ const characterEnd = this.nextCharacterIndex(content, index)
+ const isWordCharacter = this.isHighlightWordCharacterAt(content, index)
+ if (isWordCharacter && !insideWord) {
+ if (words === HIGHLIGHT_CONTEXT_WORDS_AFTER) break
+ words++
+ }
+ index = characterEnd
+ insideWord = isWordCharacter
+ }
+ return index
+ }
+
+ private static previousCharacterIndex(value: string, index: number): number {
+ const characterIndex = index - 1
+ if (characterIndex === 0) return 0
+ const character = value.charCodeAt(characterIndex)
+ const previousCharacter = value.charCodeAt(characterIndex - 1)
+ return character >= 0xdc00 && character <= 0xdfff && previousCharacter >= 0xd800 && previousCharacter <= 0xdbff
+ ? characterIndex - 1
+ : characterIndex
+ }
+
+ private static nextCharacterIndex(value: string, index: number): number {
+ const codePoint = value.codePointAt(index)
+ return index + (codePoint !== undefined && codePoint > 0xffff ? 2 : 1)
+ }
+
+ private static isHighlightWordCharacterAt(value: string, index: number): boolean {
+ const codeUnit = value.charCodeAt(index)
+ if (codeUnit <= 0x7f) {
+ return (codeUnit >= 0x30 && codeUnit <= 0x39) || (codeUnit >= 0x41 && codeUnit <= 0x5a) || (codeUnit >= 0x61 && codeUnit <= 0x7a)
+ }
+ HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN.lastIndex = index
+ return HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN.test(value)
+ }
+
private async getIndexes(): Promise<Record<string, string>[]> {
return (await this.db.execute(sql`SHOW TABLES LIKE ${`${FILES_CONTENT_TABLE_PREFIX}%`}`))[0] as any
}
@@ -262,7 +432,7 @@ function toLikePattern(term: string): string {
return `%${term.replaceAll('=', '==').replaceAll('%', '=%').replaceAll('_', '=_')}%`
}
-function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {
+function createContentMatch(terms: FilesSearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {
if (!terms.length) return null
return sql.join(
terms.map(({ rawValue }) =>
diff --git a/backend/src/applications/files/constants/search.ts b/backend/src/applications/files/constants/search.ts
index 28775a30..6ea777da 100644
--- a/backend/src/applications/files/constants/search.ts
+++ b/backend/src/applications/files/constants/search.ts
@@ -1,3 +1,4 @@
export const SEARCH_FILES_MIN_LIMIT = 1
export const SEARCH_FILES_DEFAULT_LIMIT = 100
export const SEARCH_FILES_MAX_LIMIT = 100
+export const SEARCH_FILES_MAX_QUERY_LENGTH = 512
diff --git a/backend/src/applications/files/dto/file-operations.dto.ts b/backend/src/applications/files/dto/file-operations.dto.ts
index 1fe721fd..9a243d37 100644
--- a/backend/src/applications/files/dto/file-operations.dto.ts
+++ b/backend/src/applications/files/dto/file-operations.dto.ts
@@ -1,9 +1,23 @@
import { Transform } from 'class-transformer'
-import { ArrayMinSize, IsArray, IsBoolean, IsDefined, IsIn, IsInt, IsNotEmpty, IsOptional, IsString, IsUrl, Max, Min } from 'class-validator'
+import {
+ ArrayMinSize,
+ IsArray,
+ IsBoolean,
+ IsDefined,
+ IsIn,
+ IsInt,
+ IsNotEmpty,
+ IsOptional,
+ IsString,
+ IsUrl,
+ Max,
+ MaxLength,
+ Min
+} from 'class-validator'
import { RejectIfMatch } from '../../../common/decorators'
import { regExpInvalidFileName } from '../../../common/shared'
import { TAR_EXTENSION, ZIP_EXTENSION } from '../constants/compress'
-import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MIN_LIMIT } from '../constants/search'
+import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MAX_QUERY_LENGTH, SEARCH_FILES_MIN_LIMIT } from '../constants/search'
export class CopyMoveFileDto {
@IsNotEmpty()
@@ -70,6 +84,7 @@ export class CompressFileDto {
export class SearchFilesDto {
@IsNotEmpty()
@IsString()
+ @MaxLength(SEARCH_FILES_MAX_QUERY_LENGTH)
@Transform(({ value }) => (value ? value.trim() : null))
content: string
diff --git a/backend/src/applications/files/interfaces/files-search-query.interface.ts b/backend/src/applications/files/interfaces/files-search-query.interface.ts
new file mode 100644
index 00000000..9bbbfb4d
--- /dev/null
+++ b/backend/src/applications/files/interfaces/files-search-query.interface.ts
@@ -0,0 +1,18 @@
+export type FilesSearchTermOperator = 'required' | 'excluded' | 'optional'
+
+export interface FilesSearchTerm {
+ rawValue: string
+ regexpValue: string
+ operator: FilesSearchTermOperator
+ requiresLike: boolean
+ quoted: boolean
+ wildcard: boolean
+}
+
+export interface FilesSearchQuery {
+ terms: FilesSearchTerm[]
+ positiveTerms: FilesSearchTerm[]
+ requiredTerms: FilesSearchTerm[]
+ optionalTerms: FilesSearchTerm[]
+ excludedTerms: FilesSearchTerm[]
+}
diff --git a/backend/src/applications/files/services/files-search-manager.service.spec.ts b/backend/src/applications/files/services/files-search-manager.service.spec.ts
index abae101a..8a6c5f21 100644
--- a/backend/src/applications/files/services/files-search-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.spec.ts
@@ -162,6 +162,20 @@ describe(FilesSearchManager.name, () => {
expect(error.message).toBe('SyntaxError (check special characters)')
})
+ it('should keep storage error details in logs but not in the HTTP error', async () => {
+ const storageError = new Error('Failed query: SELECT private_data')
+ const loggerSpy = vi.spyOn((service as any).logger, 'error').mockImplementation(() => undefined)
+ filesIndexer.existingIndexes.mockResolvedValueOnce(['user_5'])
+ filesIndexer.searchRecords.mockRejectedValueOnce(storageError)
+
+ const error = await (service as any).searchFullText(5, [], [], 'report', 5).catch((e: HttpException) => e)
+
+ expect(loggerSpy).toHaveBeenCalledWith(expect.objectContaining({ err: storageError }))
+ expect(error.getStatus()).toBe(HttpStatus.BAD_REQUEST)
+ expect(error.message).toBe('Unable to perform full-text search')
+ expect(error.message).not.toContain('private_data')
+ })
+
it('should stop filename search when the limit is reached', async () => {
filesParser.allPaths.mockResolvedValue([
{
diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.ts
index df78f4ca..2a0c4586 100644
--- a/backend/src/applications/files/services/files-search-manager.service.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.ts
@@ -57,12 +57,13 @@ export class FilesSearchManager {
try {
return await this.filesIndexer.searchRecords(indexNames, search, limit)
} catch (e) {
- this.logger.error({ tag: this.searchFullText.name, msg: `${JSON.stringify(indexNames)} - ${search} : ${e}` })
+ const error = e instanceof Error ? e : new Error(`${e}`)
+ this.logger.error({ tag: this.searchFullText.name, msg: `${JSON.stringify(indexNames)} - ${search}`, err: error })
let msg: string
- if (/Invalid regular expression/.test(e.message)) {
+ if (/Invalid regular expression/.test(error.message)) {
msg = 'SyntaxError (check special characters)'
} else {
- msg = e.message
+ msg = 'Unable to perform full-text search'
}
throw new HttpException(msg, HttpStatus.BAD_REQUEST)
}
diff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.ts
index 1070c175..27984077 100644
--- a/backend/src/applications/files/utils/files-search.spec.ts
+++ b/backend/src/applications/files/utils/files-search.spec.ts
@@ -5,6 +5,7 @@ import {
likeSearchTermStartPattern,
MaxSortedList,
normalizeSearchLimit,
+ parseFilesSearchQuery,
parseSearchTerms,
requiresLikeSearch
} from './files-search'
@@ -80,12 +81,9 @@ describe('files search utilities', () => {
})
describe(genTermsPattern.name, () => {
- it('should generate an accent-insensitive alternative pattern', () => {
- const regexp = new RegExp(`^(${genTermsPattern(['resume', 'canyon'])})$`, 'iu')
-
- expect(regexp.test('résumé')).toBe(true)
- expect(regexp.test('cañyon')).toBe(true)
- expect(regexp.test('other')).toBe(false)
+ it('should match accented and unaccented terms in both directions', () => {
+ expect(new RegExp(`^(${genTermsPattern(['resume'])})$`, 'iu').test('résumé')).toBe(true)
+ expect(new RegExp(`^(${genTermsPattern(['résumé'])})$`, 'iu').test('resume')).toBe(true)
})
it('should generate a Unicode regular expression compatible pattern with hyphenated terms', () => {
@@ -152,32 +150,45 @@ describe('files search utilities', () => {
describe(parseSearchTerms.name, () => {
it('should classify boolean search terms and exact phrases', () => {
expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([
- { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true },
- { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true },
- { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true },
- { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true }
+ { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true, quoted: false, wildcard: false },
+ { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true, quoted: false, wildcard: false },
+ { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true, quoted: false, wildcard: false },
+ { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true, quoted: true, wildcard: false }
])
})
it('should remove nested modifiers and trailing wildcards', () => {
expect(parseSearchTerms('++report file*')).toEqual([
- { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false },
- { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false }
+ { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false, quoted: false, wildcard: false },
+ { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false, quoted: false, wildcard: true }
])
})
it('should ignore terms below the minimum length', () => {
- expect(parseSearchTerms('+a valid')).toEqual([{ rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false }])
+ expect(parseSearchTerms('+a valid')).toEqual([
+ { rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false, quoted: false, wildcard: false }
+ ])
})
it('should expose raw and regular expression values', () => {
expect(parseSearchTerms('file.txt euro-office')).toEqual([
- { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false },
- { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false }
+ { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false, quoted: false, wildcard: false },
+ { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false, quoted: false, wildcard: false }
])
})
})
+ describe(parseFilesSearchQuery.name, () => {
+ it('should group generic search terms by operator', () => {
+ const query = parseFilesSearchQuery('+required optional -excluded')
+
+ expect(query.positiveTerms.map(({ rawValue }) => rawValue)).toEqual(['required', 'optional'])
+ expect(query.requiredTerms.map(({ rawValue }) => rawValue)).toEqual(['required'])
+ expect(query.optionalTerms.map(({ rawValue }) => rawValue)).toEqual(['optional'])
+ expect(query.excludedTerms.map(({ rawValue }) => rawValue)).toEqual(['excluded'])
+ })
+ })
+
describe(likeSearchTermStartPattern.name, () => {
it('should match only before a character requiring the LIKE fallback', () => {
const regexp = new RegExp(`${likeSearchTermStartPattern()}.`, 'u')
diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.ts
index ba6644cc..84a368c9 100644
--- a/backend/src/applications/files/utils/files-search.ts
+++ b/backend/src/applications/files/utils/files-search.ts
@@ -1,48 +1,36 @@
import { regexpEscape } from '../../../common/functions'
import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'
import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MIN_LIMIT } from '../constants/search'
+import type { FilesSearchQuery, FilesSearchTerm } from '../interfaces/files-search-query.interface'
-const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)
-const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')
-const booleanOperators = new Set(['+', '-', '<', '>', '~', '*'])
-const UNICODE_WORD_CHAR = '[\\p{L}\\p{N}]'
+const SEARCH_TERMS_PATTERN = /([+-]?)(?:"([^"]+)"|(\S+))/g
+const LEADING_BOOLEAN_OPERATORS = new Set(['+', '-', '<', '>', '~', '*'])
+const UNICODE_WORD_CHAR = '[\\p{L}\\p{M}\\p{N}]'
const LIKE_SEARCH_CHAR =
'[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]'
-const regexRequiresLikeSearch = new RegExp(LIKE_SEARCH_CHAR, 'u')
-const accentToBaseMap = new Map<string, string>([
- ['a', '[aàáâä]'],
- ['e', '[eèéêë]'],
- ['i', '[iìíîï]'],
- ['o', '[oòóôö]'],
- ['u', '[uùúûü]'],
- ['c', '[cç]'],
- ['n', '[nñ]'],
- ['s', '[sš]'],
- ['z', '[zž]'],
- ['y', '[yýÿ]']
-])
-
-export interface SearchTerm {
- rawValue: string
- regexpValue: string
- operator: 'required' | 'excluded' | 'optional'
- requiresLike: boolean
-}
+const REQUIRES_LIKE_SEARCH_PATTERN = new RegExp(LIKE_SEARCH_CHAR, 'u')
+const ACCENT_INSENSITIVE_CHARACTER_GROUPS = ['aàáâä', 'eèéêë', 'iìíîï', 'oòóôö', 'uùúûü', 'cç', 'nñ', 'sš', 'zž', 'yýÿ']
+const ACCENT_INSENSITIVE_PATTERN_BY_CHARACTER = new Map<string, string>(
+ ACCENT_INSENSITIVE_CHARACTER_GROUPS.flatMap((characters) => {
+ const pattern = `[${characters}]`
+ return [...characters].map((character) => [character, pattern] as const)
+ })
+)
-export class MaxSortedList {
- public data: [number, string][] = []
+export class MaxSortedList<T = string> {
+ public data: [number, T][] = []
public nbItems: number
constructor(nbItems: number) {
this.nbItems = nbItems
}
- insert(item: [number, string]) {
+ insert(item: [number, T]) {
if (this.data.length === 0) {
this.data.push(item)
return
}
- // if score is smaller or the score already stored for another string ignore it and keep the first matches.
+ // If the score is smaller or already stored for another item, keep the first matches.
if (this.data.length === this.nbItems && (item[0] < this.data[this.data.length - 1][0] || this.data.some(([num]) => num === item[0]))) {
return
}
@@ -86,33 +74,47 @@ export function genRegexPositiveAndNegativeTerms(search: string): RegExp {
return new RegExp(`^${p}(?!.*(${n})).*$`, 'iu')
}
-export function requiresLikeSearch(input: string): boolean {
- return regexRequiresLikeSearch.test(input)
+export function parseFilesSearchQuery(search: string): FilesSearchQuery {
+ const terms = parseSearchTerms(search)
+ return {
+ terms,
+ positiveTerms: terms.filter(({ operator }) => operator !== 'excluded'),
+ requiredTerms: terms.filter(({ operator }) => operator === 'required'),
+ optionalTerms: terms.filter(({ operator }) => operator === 'optional'),
+ excludedTerms: terms.filter(({ operator }) => operator === 'excluded')
+ }
}
-export function parseSearchTerms(search: string): SearchTerm[] {
- return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {
- const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)
- let rawValue = (quoted || unquoted).trim()
- while (booleanOperators.has(rawValue[0])) {
+export function parseSearchTerms(search: string): FilesSearchTerm[] {
+ const terms: FilesSearchTerm[] = []
+ for (const [, operator, quotedValue, unquotedValue] of search.matchAll(SEARCH_TERMS_PATTERN)) {
+ const quoted = quotedValue !== undefined
+ let rawValue = (quotedValue || unquotedValue).trim()
+ while (LEADING_BOOLEAN_OPERATORS.has(rawValue[0])) {
rawValue = rawValue.substring(1)
}
- if (rawValue[rawValue.length - 1] === '*') {
+ const wildcard = rawValue.endsWith('*')
+ if (wildcard) {
rawValue = rawValue.substring(0, rawValue.length - 1)
}
if (rawValue.length < MIN_CHARS_TO_SEARCH) {
- return []
+ continue
}
- const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'
- return [
- {
- rawValue,
- regexpValue: escapeSearchTermRegexp(rawValue),
- operator: searchOperator,
- requiresLike: requiresLikeSearch(rawValue)
- }
- ]
- })
+ const searchOperator: FilesSearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'
+ terms.push({
+ rawValue,
+ regexpValue: escapeSearchTermRegexp(rawValue),
+ operator: searchOperator,
+ requiresLike: requiresLikeSearch(rawValue),
+ quoted,
+ wildcard
+ })
+ }
+ return terms
+}
+
+export function requiresLikeSearch(input: string): boolean {
+ return REQUIRES_LIKE_SEARCH_PATTERN.test(input)
}
export function likeSearchTermStartPattern(): string {
@@ -132,11 +134,11 @@ function termBoundaryPattern(term: string, endBoundary = false): string {
}
function genAccentInsensitiveRegexpPattern(input: string): string {
- /* Allow to catch all terms with accents or not */
- return input
- .split('')
- .map((char: string) => accentToBaseMap.get(char) || char)
- .join('')
+ let pattern = ''
+ for (const character of input) {
+ pattern += ACCENT_INSENSITIVE_PATTERN_BY_CHARACTER.get(character.toLowerCase()) || character
+ }
+ return pattern
}
function escapeSearchTermRegexp(input: string): string {