Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): unify search term parsing

johaven3a262c713777
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index dca5be1a..61c6d767 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -6,15 +6,7 @@ import { DBSchema } from '../../../infrastructure/database/interfaces/database.i
import { FilesContentStore } from '../models/files-content-store'
import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'
import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'
-import {
- analyzeTerms,
- genTermsPattern,
- likeSearchTermStartPattern,
- MaxSortedList,
- parseSearchTerms,
- requiresLikeSearch,
- SearchTerm
-} from '../utils/files-search'
+import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseSearchTerms, SearchTerm } from '../utils/files-search'
type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }
type SearchRecord = FileContent & { sourceIndex: string }
@@ -88,7 +80,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
return new Map()
}
const [r]: { id: number; path: string; name: string; size: number }[][] = (await this.db.execute(
- sql`SELECT id, path, name, size FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`
+ sql`SELECT id, path, name, size FROM ${sql.raw(tableName)} WHERE id IN (${idsSqlList(ids)})`
)) as MySqlQueryResult
return new Map(
r.map((row) => [row.id, { path: row.path, name: row.name, size: row.size }] satisfies [FileContent['id'], FileContentRecordMetadata])
@@ -98,7 +90,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
async markRecordsSeen(tableName: string, ids: number[], runId: string): Promise<boolean> {
if (!ids.length) return true
try {
- await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${sql.raw(ids.join(','))})`)
+ await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${idsSqlList(ids)})`)
return true
} catch (e) {
this.logger.error({ tag: this.markRecordsSeen.name, msg: `${tableName} : ${e}` })
@@ -108,7 +100,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
async deleteRecords(tableName: string, ids: number[]): Promise<void> {
try {
- const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`)
+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${idsSqlList(ids)})`)
if (r.affectedRows !== ids.length) {
this.logger.warn({ tag: this.deleteRecords.name, msg: `${tableName} - deleted : ${r.affectedRows}/${ids.length}` })
}
@@ -128,13 +120,13 @@ export class FilesContentStoreMySQL implements FilesContentStore {
}
async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {
- const terms: string[] = analyzeTerms(search)
const searchTerms = parseSearchTerms(search)
const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')
const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')
const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')
const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')
- const useLikeSearch = searchTerms.some(({ value }) => requiresLikeSearch(value))
+ const terms = positiveTerms.map(({ regexpValue }) => regexpValue)
+ const useLikeSearch = searchTerms.some(({ requiresLike }) => requiresLike)
this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
if (!terms.length) {
return []
@@ -150,7 +142,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
const positiveMatch = requiredMatch || optionalMatch
const excludedMatch = createContentMatch(excludedTerms, ' AND ', true)
const score = positiveTerms.reduce<SQL>(
- (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.value)} ESCAPE '=', 1, 0)`,
+ (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.rawValue)} ESCAPE '=', 1, 0)`,
sql.raw('0')
)
return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as score
@@ -192,10 +184,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
return [
sql`SELECT ${tableName} as sourceIndex, id, path, name, mime, mtime, content
FROM ${sql.raw(tableName)}
- WHERE id IN (${sql.join(
- ids.map((id) => sql`${id}`),
- sql.raw(', ')
- )})`
+ WHERE id IN (${idsSqlList(ids)})`
]
}),
sql.raw(' UNION ALL ')
@@ -263,9 +252,16 @@ function toLikePattern(term: string): string {
function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {
if (!terms.length) return null
return sql.join(
- terms.map(({ value }) =>
- negate ? sql`content NOT LIKE ${toLikePattern(value)} ESCAPE '='` : sql`content LIKE ${toLikePattern(value)} ESCAPE '='`
+ terms.map(({ rawValue }) =>
+ negate ? sql`content NOT LIKE ${toLikePattern(rawValue)} ESCAPE '='` : sql`content LIKE ${toLikePattern(rawValue)} ESCAPE '='`
),
sql.raw(separator)
)
}
+
+function idsSqlList(ids: number[]): SQL {
+ return sql.join(
+ ids.map((id) => sql`${id}`),
+ sql.raw(', ')
+ )
+}
diff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.ts
index bdc0d4c4..1070c175 100644
--- a/backend/src/applications/files/utils/files-search.spec.ts
+++ b/backend/src/applications/files/utils/files-search.spec.ts
@@ -152,22 +152,29 @@ describe('files search utilities', () => {
describe(parseSearchTerms.name, () => {
it('should classify boolean search terms and exact phrases', () => {
expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([
- { value: '中文', operator: 'required' },
- { value: '秘密', operator: 'excluded' },
- { value: '文档', operator: 'optional' },
- { value: '全文 搜索', operator: 'optional' }
+ { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true },
+ { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true },
+ { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true },
+ { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true }
])
})
it('should remove nested modifiers and trailing wildcards', () => {
expect(parseSearchTerms('++report file*')).toEqual([
- { value: 'report', operator: 'required' },
- { value: 'file', operator: 'optional' }
+ { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false },
+ { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false }
])
})
it('should ignore terms below the minimum length', () => {
- expect(parseSearchTerms('+a valid')).toEqual([{ value: 'valid', operator: 'optional' }])
+ expect(parseSearchTerms('+a valid')).toEqual([{ rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false }])
+ })
+
+ it('should expose raw and regular expression values', () => {
+ expect(parseSearchTerms('file.txt euro-office')).toEqual([
+ { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false },
+ { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false }
+ ])
})
})
diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.ts
index efcae78b..ba6644cc 100644
--- a/backend/src/applications/files/utils/files-search.ts
+++ b/backend/src/applications/files/utils/files-search.ts
@@ -23,8 +23,10 @@ const accentToBaseMap = new Map<string, string>([
])
export interface SearchTerm {
- value: string
+ rawValue: string
+ regexpValue: string
operator: 'required' | 'excluded' | 'optional'
+ requiresLike: boolean
}
export class MaxSortedList {
@@ -59,31 +61,9 @@ export class MaxSortedList {
}
export function analyzeTerms(search: string, onlyAllowNegative = false, escapeForRegexp = true): string[] {
- /* Get the positive or negative terms list */
- const matches: RegExpMatchArray | [] = search.match(regexMatchesSearchBoolean) || []
- if (!matches.length) {
- return matches
- }
- return matches
- .flatMap((match: string) => {
- const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)
- let term: string = (quoted || unquoted).trim()
-
- if (term.length < MIN_CHARS_TO_SEARCH) return null
-
- if ((onlyAllowNegative && operator !== '-') || (!onlyAllowNegative && (operator === '-' || operator === '~'))) return null
-
- if (booleanOperators.has(term[0])) {
- term = term.substring(1)
- }
-
- if (term[term.length - 1] === '*') {
- term = term.substring(0, term.length - 1)
- }
-
- return escapeForRegexp ? escapeSearchTermRegexp(term) : term
- })
- .filter(Boolean)
+ return parseSearchTerms(search)
+ .filter(({ operator }) => (onlyAllowNegative ? operator === 'excluded' : operator !== 'excluded'))
+ .map((term) => (escapeForRegexp ? term.regexpValue : term.rawValue))
}
export function genTermsPattern(terms: string[]): string {
@@ -91,8 +71,9 @@ export function genTermsPattern(terms: string[]): string {
}
export function genRegexPositiveAndNegativeTerms(search: string): RegExp {
- const positiveTerms = analyzeTerms(search)
- const negativeTerms = analyzeTerms(search, true)
+ const searchTerms = parseSearchTerms(search)
+ const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded').map(({ regexpValue }) => regexpValue)
+ const negativeTerms = searchTerms.filter(({ operator }) => operator === 'excluded').map(({ regexpValue }) => regexpValue)
const p = positiveTerms
.map((t) => genAccentInsensitiveRegexpPattern(t))
.map((t) => `(?=.*${termBoundaryPattern(t)})`)
@@ -112,21 +93,23 @@ export function requiresLikeSearch(input: string): boolean {
export function parseSearchTerms(search: string): SearchTerm[] {
return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {
const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)
- let value = (quoted || unquoted).trim()
- while (booleanOperators.has(value[0])) {
- value = value.substring(1)
+ let rawValue = (quoted || unquoted).trim()
+ while (booleanOperators.has(rawValue[0])) {
+ rawValue = rawValue.substring(1)
}
- if (value[value.length - 1] === '*') {
- value = value.substring(0, value.length - 1)
+ if (rawValue[rawValue.length - 1] === '*') {
+ rawValue = rawValue.substring(0, rawValue.length - 1)
}
- if (value.length < MIN_CHARS_TO_SEARCH) {
+ if (rawValue.length < MIN_CHARS_TO_SEARCH) {
return []
}
const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'
return [
{
- value,
- operator: searchOperator
+ rawValue,
+ regexpValue: escapeSearchTermRegexp(rawValue),
+ operator: searchOperator,
+ requiresLike: requiresLikeSearch(rawValue)
}
]
})