Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): unify search term parsing
johaven3a262c713777
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.tsindex dca5be1a..61c6d767 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts@@ -6,15 +6,7 @@ import { DBSchema } from '../../../infrastructure/database/interfaces/database.iimport { FilesContentStore } from '../models/files-content-store'import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'-import {- analyzeTerms,- genTermsPattern,- likeSearchTermStartPattern,- MaxSortedList,- parseSearchTerms,- requiresLikeSearch,- SearchTerm-} from '../utils/files-search'+import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseSearchTerms, SearchTerm } from '../utils/files-search'type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }type SearchRecord = FileContent & { sourceIndex: string }@@ -88,7 +80,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {return new Map()}const [r]: { id: number; path: string; name: string; size: number }[][] = (await this.db.execute(- sql`SELECT id, path, name, size FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`+ sql`SELECT id, path, name, size FROM ${sql.raw(tableName)} WHERE id IN (${idsSqlList(ids)})`)) as MySqlQueryResultreturn new Map(r.map((row) => [row.id, { path: row.path, name: row.name, size: row.size }] satisfies [FileContent['id'], FileContentRecordMetadata])@@ -98,7 +90,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {async markRecordsSeen(tableName: string, ids: number[], runId: string): Promise<boolean> {if (!ids.length) return truetry {- await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${sql.raw(ids.join(','))})`)+ await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${idsSqlList(ids)})`)return true} catch (e) {this.logger.error({ tag: this.markRecordsSeen.name, msg: `${tableName} : ${e}` })@@ -108,7 +100,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {async deleteRecords(tableName: string, ids: number[]): Promise<void> {try {- const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`)+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${idsSqlList(ids)})`)if (r.affectedRows !== ids.length) {this.logger.warn({ tag: this.deleteRecords.name, msg: `${tableName} - deleted : ${r.affectedRows}/${ids.length}` })}@@ -128,13 +120,13 @@ export class FilesContentStoreMySQL implements FilesContentStore {}async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {- const terms: string[] = analyzeTerms(search)const searchTerms = parseSearchTerms(search)const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')- const useLikeSearch = searchTerms.some(({ value }) => requiresLikeSearch(value))+ const terms = positiveTerms.map(({ regexpValue }) => regexpValue)+ const useLikeSearch = searchTerms.some(({ requiresLike }) => requiresLike)this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })if (!terms.length) {return []@@ -150,7 +142,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {const positiveMatch = requiredMatch || optionalMatchconst excludedMatch = createContentMatch(excludedTerms, ' AND ', true)const score = positiveTerms.reduce<SQL>(- (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.value)} ESCAPE '=', 1, 0)`,+ (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.rawValue)} ESCAPE '=', 1, 0)`,sql.raw('0'))return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as score@@ -192,10 +184,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {return [sql`SELECT ${tableName} as sourceIndex, id, path, name, mime, mtime, contentFROM ${sql.raw(tableName)}- WHERE id IN (${sql.join(- ids.map((id) => sql`${id}`),- sql.raw(', ')- )})`+ WHERE id IN (${idsSqlList(ids)})`]}),sql.raw(' UNION ALL ')@@ -263,9 +252,16 @@ function toLikePattern(term: string): string {function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {if (!terms.length) return nullreturn sql.join(- terms.map(({ value }) =>- negate ? sql`content NOT LIKE ${toLikePattern(value)} ESCAPE '='` : sql`content LIKE ${toLikePattern(value)} ESCAPE '='`+ terms.map(({ rawValue }) =>+ negate ? sql`content NOT LIKE ${toLikePattern(rawValue)} ESCAPE '='` : sql`content LIKE ${toLikePattern(rawValue)} ESCAPE '='`),sql.raw(separator))}++function idsSqlList(ids: number[]): SQL {+ return sql.join(+ ids.map((id) => sql`${id}`),+ sql.raw(', ')+ )+}diff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.tsindex bdc0d4c4..1070c175 100644--- a/backend/src/applications/files/utils/files-search.spec.ts+++ b/backend/src/applications/files/utils/files-search.spec.ts@@ -152,22 +152,29 @@ describe('files search utilities', () => {describe(parseSearchTerms.name, () => {it('should classify boolean search terms and exact phrases', () => {expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([- { value: '中文', operator: 'required' },- { value: '秘密', operator: 'excluded' },- { value: '文档', operator: 'optional' },- { value: '全文 搜索', operator: 'optional' }+ { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true },+ { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true },+ { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true },+ { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true }])})it('should remove nested modifiers and trailing wildcards', () => {expect(parseSearchTerms('++report file*')).toEqual([- { value: 'report', operator: 'required' },- { value: 'file', operator: 'optional' }+ { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false },+ { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false }])})it('should ignore terms below the minimum length', () => {- expect(parseSearchTerms('+a valid')).toEqual([{ value: 'valid', operator: 'optional' }])+ expect(parseSearchTerms('+a valid')).toEqual([{ rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false }])+ })++ it('should expose raw and regular expression values', () => {+ expect(parseSearchTerms('file.txt euro-office')).toEqual([+ { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false },+ { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false }+ ])})})diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.tsindex efcae78b..ba6644cc 100644--- a/backend/src/applications/files/utils/files-search.ts+++ b/backend/src/applications/files/utils/files-search.ts@@ -23,8 +23,10 @@ const accentToBaseMap = new Map<string, string>([])export interface SearchTerm {- value: string+ rawValue: string+ regexpValue: stringoperator: 'required' | 'excluded' | 'optional'+ requiresLike: boolean}export class MaxSortedList {@@ -59,31 +61,9 @@ export class MaxSortedList {}export function analyzeTerms(search: string, onlyAllowNegative = false, escapeForRegexp = true): string[] {- /* Get the positive or negative terms list */- const matches: RegExpMatchArray | [] = search.match(regexMatchesSearchBoolean) || []- if (!matches.length) {- return matches- }- return matches- .flatMap((match: string) => {- const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)- let term: string = (quoted || unquoted).trim()-- if (term.length < MIN_CHARS_TO_SEARCH) return null-- if ((onlyAllowNegative && operator !== '-') || (!onlyAllowNegative && (operator === '-' || operator === '~'))) return null-- if (booleanOperators.has(term[0])) {- term = term.substring(1)- }-- if (term[term.length - 1] === '*') {- term = term.substring(0, term.length - 1)- }-- return escapeForRegexp ? escapeSearchTermRegexp(term) : term- })- .filter(Boolean)+ return parseSearchTerms(search)+ .filter(({ operator }) => (onlyAllowNegative ? operator === 'excluded' : operator !== 'excluded'))+ .map((term) => (escapeForRegexp ? term.regexpValue : term.rawValue))}export function genTermsPattern(terms: string[]): string {@@ -91,8 +71,9 @@ export function genTermsPattern(terms: string[]): string {}export function genRegexPositiveAndNegativeTerms(search: string): RegExp {- const positiveTerms = analyzeTerms(search)- const negativeTerms = analyzeTerms(search, true)+ const searchTerms = parseSearchTerms(search)+ const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded').map(({ regexpValue }) => regexpValue)+ const negativeTerms = searchTerms.filter(({ operator }) => operator === 'excluded').map(({ regexpValue }) => regexpValue)const p = positiveTerms.map((t) => genAccentInsensitiveRegexpPattern(t)).map((t) => `(?=.*${termBoundaryPattern(t)})`)@@ -112,21 +93,23 @@ export function requiresLikeSearch(input: string): boolean {export function parseSearchTerms(search: string): SearchTerm[] {return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)- let value = (quoted || unquoted).trim()- while (booleanOperators.has(value[0])) {- value = value.substring(1)+ let rawValue = (quoted || unquoted).trim()+ while (booleanOperators.has(rawValue[0])) {+ rawValue = rawValue.substring(1)}- if (value[value.length - 1] === '*') {- value = value.substring(0, value.length - 1)+ if (rawValue[rawValue.length - 1] === '*') {+ rawValue = rawValue.substring(0, rawValue.length - 1)}- if (value.length < MIN_CHARS_TO_SEARCH) {+ if (rawValue.length < MIN_CHARS_TO_SEARCH) {return []}const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'return [{- value,- operator: searchOperator+ rawValue,+ regexpValue: escapeSearchTermRegexp(rawValue),+ operator: searchOperator,+ requiresLike: requiresLikeSearch(rawValue)}]})