Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): implement incremental indexing triggers for full-text search

johaven468c1c3d9480
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index d0704372..eb87dbb5 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -15,7 +15,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
- @CacheDecorator(900) // 15 mn
+ @CacheDecorator(300)
async indexesList(): Promise<string[]> {
return ((await this.db.execute(sql`SHOW TABLES LIKE '${sql.raw(FILES_CONTENT_TABLE_PREFIX)}%'`))[0] as any).flatMap((r: Record<string, string>) =>
Object.values(r)
@@ -93,7 +93,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {
async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {
const terms: string[] = analyzeTerms(search)
- this.logger.debug({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
+ this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })
if (!terms.length) {
return []
}
diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.ts
index 4012e5e3..1f3e6622 100644
--- a/backend/src/applications/files/services/files-content-indexer.service.ts
+++ b/backend/src/applications/files/services/files-content-indexer.service.ts
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'
import fs from 'fs/promises'
import { Stats } from 'node:fs'
import path from 'node:path'
-import { INDEXABLE_EXTENSIONS } from '../constants/indexing'
+import { CACHE_INDEXING_UPDATE_PREFIX, INDEXABLE_EXTENSIONS } from '../constants/indexing'
import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'
import { FilesContentStore } from '../models/files-content-store'
import { FileContent } from '../schemas/file-content.interface'
@@ -11,6 +11,8 @@ import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'
import { getExtensionWithoutDot, getMimeType } from '../utils/files'
import { FilesParser } from './files-parser.service'
import { genIndexingKey } from '../utils/indexing'
+import { FILE_REPOSITORY } from '../constants/operations'
+import { Cache } from '../../../infrastructure/cache/services/cache.service'
@Injectable()
export class FilesContentIndexer {
@@ -19,10 +21,46 @@ export class FilesContentIndexer {
private pdfOcrWorkerManager: PdfOCRWorkerManager | null = null
constructor(
+ private readonly cache: Cache,
private readonly filesIndexer: FilesContentStore,
private readonly filesParser: FilesParser
) {}
+ async updateIndexEntries() {
+ const cacheKeys: string[] = []
+ const [userIds, spaceIds, shareIds]: [number[], number[], number[]] = [[], [], []]
+ for (const k of await this.cache.keys(`${CACHE_INDEXING_UPDATE_PREFIX}-*`)) {
+ cacheKeys.push(k)
+ const keySegments = k.split('-')
+ const [repository, idPart] = keySegments.slice(-2)
+ const id = Number.parseInt(idPart ?? '', 10)
+
+ if (repository === FILE_REPOSITORY.USER) {
+ userIds.push(id)
+ } else if (repository === FILE_REPOSITORY.SPACE) {
+ spaceIds.push(id)
+ } else if (repository === FILE_REPOSITORY.SHARE) {
+ shareIds.push(id)
+ } else {
+ this.logger.warn({ tag: this.updateIndexEntries.name, msg: `Unknown type: ${repository}` })
+ }
+ }
+
+ if (userIds.length || spaceIds.length || shareIds.length) {
+ try {
+ await this.parseAndIndexAllFiles(userIds, spaceIds, shareIds)
+ } catch (e) {
+ this.logger.error({ tag: this.updateIndexEntries.name, msg: `${e}` })
+ }
+ }
+
+ // Clean up event keys even if incremental indexing fails.
+ // Ignore cache deletion errors, the full reindex restores consistency.
+ for (const k of cacheKeys) {
+ this.cache.del(k).catch((e) => this.logger.warn({ tag: this.updateIndexEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))
+ }
+ }
+
async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {
this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)
try {
diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.ts
index 5ae40432..ec2c7c47 100644
--- a/backend/src/applications/files/services/files-event-manager.service.ts
+++ b/backend/src/applications/files/services/files-event-manager.service.ts
@@ -5,19 +5,24 @@ import { ACTION } from '../../../common/constants'
import type { FileEventType } from '../interfaces/file-event.interface'
import { CACHE_QUOTA_TTL } from '../constants/cache'
import { quotaCacheKeyFromSpace } from '../utils/quota'
+import { getExtensionWithoutDot } from '../utils/files'
+import { CACHE_INDEXING_TTL, INDEXABLE_EXTENSIONS } from '../constants/indexing'
+import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'
+import { configuration } from '../../../configuration/config.environment'
@Injectable()
export class FilesEventManager implements OnModuleDestroy {
/* Used to:
- store cached events for storage usage updates
- - todo: handle indexing
+ - store indexing events for full-text index updates
- todo: handle versioning
*/
private readonly MAX_BUFFER_SIZE = 1_000
- private readonly MAX_BUFFER_DELAY_MS = 20_000
+ private readonly MAX_BUFFER_DELAY_MS = 30_000
private readonly logger = new Logger(FilesEventManager.name)
private readonly eventsBuffer: FileEventType[] = []
private quotaEvents: string[] = []
+ private indexingEvents: string[] = []
private flushTimer: NodeJS.Timeout | null = null
private isFlushing = false
private flushRequested = false
@@ -34,8 +39,8 @@ export class FilesEventManager implements OnModuleDestroy {
private readonly onFileEvent = (fileEvent: FileEventType): void => {
this.logger.verbose({
- tag: this.processEvents.name,
- msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} repository:${fileEvent.space.alias}`
+ tag: this.onFileEvent.name,
+ msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} on ${fileEvent.space.alias}`
})
this.eventsBuffer.push(fileEvent)
if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {
@@ -89,52 +94,82 @@ export class FilesEventManager implements OnModuleDestroy {
}
}
- private async processEvents(fileEvents: FileEventType[]) {
- this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })
- for (const event of fileEvents) {
+ private async processEvents(fEvents: FileEventType[]) {
+ this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fEvents.length} file event(s)` })
+ for (const event of fEvents) {
try {
this.processQuotaEvent(event)
+ if (configuration.applications.files.contentIndexing.enabled) {
+ this.processIndexingEvent(event)
+ }
} catch (e) {
- this.logger.warn({ tag: this.processEvents.name, msg: `Could not process quota event: ${JSON.stringify(event)} - ${e}` })
+ this.logger.warn({ tag: this.processEvents.name, msg: `Could not process event: ${JSON.stringify(event)} - ${e}` })
}
}
await this.storeEventsInCache()
}
private async storeEventsInCache() {
- if (!this.quotaEvents.length) return
+ const [failedQuotaKeys, failedIndexingKeys] = await Promise.all([
+ this.cacheKeysWithTTL(this.quotaEvents, CACHE_QUOTA_TTL, 'quota'),
+ this.cacheKeysWithTTL(this.indexingEvents, CACHE_INDEXING_TTL, 'indexing')
+ ])
+ this.quotaEvents = failedQuotaKeys
+ this.indexingEvents = failedIndexingKeys
+ }
+
+ private async cacheKeysWithTTL(events: string[], ttl: number, label: 'quota' | 'indexing'): Promise<string[]> {
+ if (!events.length) {
+ return []
+ }
+
const failedCacheKeys: string[] = []
- await Promise.all(
- this.quotaEvents.map(async (cacheKey) => {
- try {
- const ok = await this.cache.set(cacheKey, true, CACHE_QUOTA_TTL)
- if (!ok) {
- failedCacheKeys.push(cacheKey)
- }
- } catch {
+ for (const cacheKey of events) {
+ try {
+ const ok = await this.cache.set(cacheKey, true, ttl)
+ if (!ok) {
failedCacheKeys.push(cacheKey)
}
- })
- )
+ } catch {
+ failedCacheKeys.push(cacheKey)
+ }
+ }
+
if (failedCacheKeys.length) {
this.logger.warn({
- tag: this.storeEventsInCache.name,
- msg: `Could not cache quota keys (${failedCacheKeys.length})`
+ tag: this.cacheKeysWithTTL.name,
+ msg: `Could not cache ${label} keys (${failedCacheKeys.length})`
})
}
- this.quotaEvents = failedCacheKeys
+
+ return failedCacheKeys
}
- private processQuotaEvent(fileEvent: FileEventType) {
+ private processQuotaEvent(fEvent: FileEventType) {
// Ignore files moved to the trash; storage usage remains unchanged.
- if (fileEvent.action === ACTION.DELETE) return
- const cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space, true)
+ if (fEvent.action === ACTION.DELETE) return
+ const cacheKey = quotaCacheKeyFromSpace(fEvent.user.id, fEvent.space, true)
if (!cacheKey) {
- this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })
+ this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fEvent.space.id}` })
return
}
if (this.quotaEvents.indexOf(cacheKey) === -1) {
this.quotaEvents.push(cacheKey)
}
}
+
+ private processIndexingEvent(fEvent: FileEventType) {
+ const extension = getExtensionWithoutDot(fEvent.rPath)
+ if (!INDEXABLE_EXTENSIONS.has(extension)) return null
+ const cacheKeys = indexingUpdateCacheKeysFromSpace(fEvent.user.id, fEvent.space)
+ if (!cacheKeys.length) {
+ this.logger.warn({ tag: this.processIndexingEvent.name, msg: `Unable to determine space location: ${fEvent.space.id}` })
+ return
+ }
+ for (const key of cacheKeys) {
+ if (this.indexingEvents.indexOf(key) === -1) {
+ this.indexingEvents.push(key)
+ }
+ }
+ }
}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 2bf3cbfc..85b0d27e 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -24,7 +24,8 @@ import { FilesQuotaManager } from './files-quota-manager.service'
@Injectable()
export class FilesScheduler {
private readonly logger = new Logger(FilesScheduler.name)
- private isIndexContentFilesRunning = false
+ private isIndexContentRunning = false
+ private isIndexContentEntriesRunning = false
private isQuotaUpdateIsRunning = false
private isQuotaUpdateEntriesIsRunning = false
@@ -56,16 +57,25 @@ export class FilesScheduler {
}
@Interval(60_000)
- async updateStorageUsageEntries() {
+ async updateStorageAndIndexing() {
if (this.isQuotaUpdateIsRunning || this.isQuotaUpdateEntriesIsRunning) return
this.isQuotaUpdateEntriesIsRunning = true
try {
await this.filesQuotaManager.updateStorageUsageEntries()
} catch (e) {
- this.logger.error({ tag: this.updateQuotas.name, msg: `${e}` })
+ this.logger.error({ tag: this.updateStorageAndIndexing.name, msg: `update quota error: ${e}` })
} finally {
this.isQuotaUpdateEntriesIsRunning = false
}
+ if (!configuration.applications.files.contentIndexing.enabled || this.isIndexContentRunning || this.isIndexContentEntriesRunning) return
+ this.isIndexContentEntriesRunning = true
+ try {
+ await this.filesContentIndexer.updateIndexEntries()
+ } catch (e) {
+ this.logger.error({ tag: this.updateStorageAndIndexing.name, msg: `update indexing error: ${e}` })
+ } finally {
+ this.isIndexContentEntriesRunning = false
+ }
}
async cleanupInterruptedTasks(): Promise<void> {
@@ -154,17 +164,17 @@ export class FilesScheduler {
async indexContentFiles(): Promise<void> {
// Conditional loading of file content indexing
if (!configuration.applications.files.contentIndexing.enabled) return
- if (this.isIndexContentFilesRunning) {
+ if (this.isIndexContentRunning || this.isIndexContentEntriesRunning) {
this.logger.warn({ tag: this.indexContentFiles.name, msg: `SKIP (already running)` })
return
}
- this.isIndexContentFilesRunning = true
+ this.isIndexContentRunning = true
this.logger.log({ tag: this.indexContentFiles.name, msg: `START` })
try {
await this.filesContentIndexer.parseAndIndexAllFiles()
this.logger.log({ tag: this.indexContentFiles.name, msg: `END` })
} finally {
- this.isIndexContentFilesRunning = false
+ this.isIndexContentRunning = false
}
}
diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.ts
index b6f65a23..3705bd55 100644
--- a/backend/src/applications/files/services/files-search-manager.service.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.ts
@@ -27,11 +27,11 @@ export class FilesSearchManager {
) {}
async search(user: UserModel, search: SearchFilesDto): Promise<FileContent[]> {
+ if (search.fullText && !configuration.applications.files.contentIndexing.enabled) {
+ throw new HttpException('Full-text search is disabled', HttpStatus.BAD_REQUEST)
+ }
const [spaceIds, shareIds] = await Promise.all([this.spacesQueries.spaceIds(user.id), this.sharesQueries.shareIds(user.id, +user.isAdmin)])
if (search.fullText) {
- if (!configuration.applications.files.contentIndexing.enabled) {
- throw new HttpException('Full-text search is disabled', HttpStatus.BAD_REQUEST)
- }
return await this.searchFullText(user.id, spaceIds, shareIds, search.content, search.limit)
} else {
return await this.searchFileNames(user.id, spaceIds, shareIds, search.content, search.limit)
diff --git a/backend/src/applications/files/utils/indexing.ts b/backend/src/applications/files/utils/indexing.ts
index c2190f8e..79dace51 100644
--- a/backend/src/applications/files/utils/indexing.ts
+++ b/backend/src/applications/files/utils/indexing.ts
@@ -7,9 +7,13 @@ export function genIndexingKey(id: number, type: FILE_REPOSITORY, sep = '_'): st
return `${type}${sep}${id}`
}
-export function indexingUpdateCacheKeyFromSpace(userId: number, space: SpaceEnv): string | null {
+export function indexingUpdateCacheKeysFromSpace(userId: number, space: SpaceEnv): string[] {
+ const cacheKeys: string[] = []
+ if (space.inSharesRepository && !space.root.externalPath) {
+ // shares that are linked to a space must also be indexed
+ cacheKeys.push(`${CACHE_INDEXING_UPDATE_PREFIX}-${genIndexingKey(space.id, FILE_REPOSITORY.SHARE, '-')}`)
+ }
const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)
- if (repository === null) return null
- const indexingKey = genIndexingKey(repository.id, repository.type, '-')
- return `${CACHE_INDEXING_UPDATE_PREFIX}-${indexingKey}`
+ if (repository !== null) cacheKeys.push(`${CACHE_INDEXING_UPDATE_PREFIX}-${genIndexingKey(repository.id, repository.type, '-')}`)
+ return cacheKeys
}