Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): implement incremental indexing triggers for full-text search
johaven468c1c3d9480
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.tsindex d0704372..eb87dbb5 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts@@ -15,7 +15,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}- @CacheDecorator(900) // 15 mn+ @CacheDecorator(300)async indexesList(): Promise<string[]> {return ((await this.db.execute(sql`SHOW TABLES LIKE '${sql.raw(FILES_CONTENT_TABLE_PREFIX)}%'`))[0] as any).flatMap((r: Record<string, string>) =>Object.values(r)@@ -93,7 +93,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {const terms: string[] = analyzeTerms(search)- this.logger.debug({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })+ this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })if (!terms.length) {return []}diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.tsindex 4012e5e3..1f3e6622 100644--- a/backend/src/applications/files/services/files-content-indexer.service.ts+++ b/backend/src/applications/files/services/files-content-indexer.service.ts@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'import fs from 'fs/promises'import { Stats } from 'node:fs'import path from 'node:path'-import { INDEXABLE_EXTENSIONS } from '../constants/indexing'+import { CACHE_INDEXING_UPDATE_PREFIX, INDEXABLE_EXTENSIONS } from '../constants/indexing'import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'import { FilesContentStore } from '../models/files-content-store'import { FileContent } from '../schemas/file-content.interface'@@ -11,6 +11,8 @@ import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'import { getExtensionWithoutDot, getMimeType } from '../utils/files'import { FilesParser } from './files-parser.service'import { genIndexingKey } from '../utils/indexing'+import { FILE_REPOSITORY } from '../constants/operations'+import { Cache } from '../../../infrastructure/cache/services/cache.service'@Injectable()export class FilesContentIndexer {@@ -19,10 +21,46 @@ export class FilesContentIndexer {private pdfOcrWorkerManager: PdfOCRWorkerManager | null = nullconstructor(+ private readonly cache: Cache,private readonly filesIndexer: FilesContentStore,private readonly filesParser: FilesParser) {}+ async updateIndexEntries() {+ const cacheKeys: string[] = []+ const [userIds, spaceIds, shareIds]: [number[], number[], number[]] = [[], [], []]+ for (const k of await this.cache.keys(`${CACHE_INDEXING_UPDATE_PREFIX}-*`)) {+ cacheKeys.push(k)+ const keySegments = k.split('-')+ const [repository, idPart] = keySegments.slice(-2)+ const id = Number.parseInt(idPart ?? '', 10)++ if (repository === FILE_REPOSITORY.USER) {+ userIds.push(id)+ } else if (repository === FILE_REPOSITORY.SPACE) {+ spaceIds.push(id)+ } else if (repository === FILE_REPOSITORY.SHARE) {+ shareIds.push(id)+ } else {+ this.logger.warn({ tag: this.updateIndexEntries.name, msg: `Unknown type: ${repository}` })+ }+ }++ if (userIds.length || spaceIds.length || shareIds.length) {+ try {+ await this.parseAndIndexAllFiles(userIds, spaceIds, shareIds)+ } catch (e) {+ this.logger.error({ tag: this.updateIndexEntries.name, msg: `${e}` })+ }+ }++ // Clean up event keys even if incremental indexing fails.+ // Ignore cache deletion errors, the full reindex restores consistency.+ for (const k of cacheKeys) {+ this.cache.del(k).catch((e) => this.logger.warn({ tag: this.updateIndexEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))+ }+ }+async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)try {diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.tsindex 5ae40432..ec2c7c47 100644--- a/backend/src/applications/files/services/files-event-manager.service.ts+++ b/backend/src/applications/files/services/files-event-manager.service.ts@@ -5,19 +5,24 @@ import { ACTION } from '../../../common/constants'import type { FileEventType } from '../interfaces/file-event.interface'import { CACHE_QUOTA_TTL } from '../constants/cache'import { quotaCacheKeyFromSpace } from '../utils/quota'+import { getExtensionWithoutDot } from '../utils/files'+import { CACHE_INDEXING_TTL, INDEXABLE_EXTENSIONS } from '../constants/indexing'+import { indexingUpdateCacheKeysFromSpace } from '../utils/indexing'+import { configuration } from '../../../configuration/config.environment'@Injectable()export class FilesEventManager implements OnModuleDestroy {/* Used to:- store cached events for storage usage updates- - todo: handle indexing+ - store indexing events for full-text index updates- todo: handle versioning*/private readonly MAX_BUFFER_SIZE = 1_000- private readonly MAX_BUFFER_DELAY_MS = 20_000+ private readonly MAX_BUFFER_DELAY_MS = 30_000private readonly logger = new Logger(FilesEventManager.name)private readonly eventsBuffer: FileEventType[] = []private quotaEvents: string[] = []+ private indexingEvents: string[] = []private flushTimer: NodeJS.Timeout | null = nullprivate isFlushing = falseprivate flushRequested = false@@ -34,8 +39,8 @@ export class FilesEventManager implements OnModuleDestroy {private readonly onFileEvent = (fileEvent: FileEventType): void => {this.logger.verbose({- tag: this.processEvents.name,- msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} repository:${fileEvent.space.alias}`+ tag: this.onFileEvent.name,+ msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} on ${fileEvent.space.alias}`})this.eventsBuffer.push(fileEvent)if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {@@ -89,52 +94,82 @@ export class FilesEventManager implements OnModuleDestroy {}}- private async processEvents(fileEvents: FileEventType[]) {- this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })- for (const event of fileEvents) {+ private async processEvents(fEvents: FileEventType[]) {+ this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fEvents.length} file event(s)` })+ for (const event of fEvents) {try {this.processQuotaEvent(event)+ if (configuration.applications.files.contentIndexing.enabled) {+ this.processIndexingEvent(event)+ }} catch (e) {- this.logger.warn({ tag: this.processEvents.name, msg: `Could not process quota event: ${JSON.stringify(event)} - ${e}` })+ this.logger.warn({ tag: this.processEvents.name, msg: `Could not process event: ${JSON.stringify(event)} - ${e}` })}}await this.storeEventsInCache()}private async storeEventsInCache() {- if (!this.quotaEvents.length) return+ const [failedQuotaKeys, failedIndexingKeys] = await Promise.all([+ this.cacheKeysWithTTL(this.quotaEvents, CACHE_QUOTA_TTL, 'quota'),+ this.cacheKeysWithTTL(this.indexingEvents, CACHE_INDEXING_TTL, 'indexing')+ ])+ this.quotaEvents = failedQuotaKeys+ this.indexingEvents = failedIndexingKeys+ }++ private async cacheKeysWithTTL(events: string[], ttl: number, label: 'quota' | 'indexing'): Promise<string[]> {+ if (!events.length) {+ return []+ }+const failedCacheKeys: string[] = []- await Promise.all(- this.quotaEvents.map(async (cacheKey) => {- try {- const ok = await this.cache.set(cacheKey, true, CACHE_QUOTA_TTL)- if (!ok) {- failedCacheKeys.push(cacheKey)- }- } catch {+ for (const cacheKey of events) {+ try {+ const ok = await this.cache.set(cacheKey, true, ttl)+ if (!ok) {failedCacheKeys.push(cacheKey)}- })- )+ } catch {+ failedCacheKeys.push(cacheKey)+ }+ }+if (failedCacheKeys.length) {this.logger.warn({- tag: this.storeEventsInCache.name,- msg: `Could not cache quota keys (${failedCacheKeys.length})`+ tag: this.cacheKeysWithTTL.name,+ msg: `Could not cache ${label} keys (${failedCacheKeys.length})`})}- this.quotaEvents = failedCacheKeys++ return failedCacheKeys}- private processQuotaEvent(fileEvent: FileEventType) {+ private processQuotaEvent(fEvent: FileEventType) {// Ignore files moved to the trash; storage usage remains unchanged.- if (fileEvent.action === ACTION.DELETE) return- const cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space, true)+ if (fEvent.action === ACTION.DELETE) return+ const cacheKey = quotaCacheKeyFromSpace(fEvent.user.id, fEvent.space, true)if (!cacheKey) {- this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })+ this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fEvent.space.id}` })return}if (this.quotaEvents.indexOf(cacheKey) === -1) {this.quotaEvents.push(cacheKey)}}++ private processIndexingEvent(fEvent: FileEventType) {+ const extension = getExtensionWithoutDot(fEvent.rPath)+ if (!INDEXABLE_EXTENSIONS.has(extension)) return null+ const cacheKeys = indexingUpdateCacheKeysFromSpace(fEvent.user.id, fEvent.space)+ if (!cacheKeys.length) {+ this.logger.warn({ tag: this.processIndexingEvent.name, msg: `Unable to determine space location: ${fEvent.space.id}` })+ return+ }+ for (const key of cacheKeys) {+ if (this.indexingEvents.indexOf(key) === -1) {+ this.indexingEvents.push(key)+ }+ }+ }}diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 2bf3cbfc..85b0d27e 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -24,7 +24,8 @@ import { FilesQuotaManager } from './files-quota-manager.service'@Injectable()export class FilesScheduler {private readonly logger = new Logger(FilesScheduler.name)- private isIndexContentFilesRunning = false+ private isIndexContentRunning = false+ private isIndexContentEntriesRunning = falseprivate isQuotaUpdateIsRunning = falseprivate isQuotaUpdateEntriesIsRunning = false@@ -56,16 +57,25 @@ export class FilesScheduler {}@Interval(60_000)- async updateStorageUsageEntries() {+ async updateStorageAndIndexing() {if (this.isQuotaUpdateIsRunning || this.isQuotaUpdateEntriesIsRunning) returnthis.isQuotaUpdateEntriesIsRunning = truetry {await this.filesQuotaManager.updateStorageUsageEntries()} catch (e) {- this.logger.error({ tag: this.updateQuotas.name, msg: `${e}` })+ this.logger.error({ tag: this.updateStorageAndIndexing.name, msg: `update quota error: ${e}` })} finally {this.isQuotaUpdateEntriesIsRunning = false}+ if (!configuration.applications.files.contentIndexing.enabled || this.isIndexContentRunning || this.isIndexContentEntriesRunning) return+ this.isIndexContentEntriesRunning = true+ try {+ await this.filesContentIndexer.updateIndexEntries()+ } catch (e) {+ this.logger.error({ tag: this.updateStorageAndIndexing.name, msg: `update indexing error: ${e}` })+ } finally {+ this.isIndexContentEntriesRunning = false+ }}async cleanupInterruptedTasks(): Promise<void> {@@ -154,17 +164,17 @@ export class FilesScheduler {async indexContentFiles(): Promise<void> {// Conditional loading of file content indexingif (!configuration.applications.files.contentIndexing.enabled) return- if (this.isIndexContentFilesRunning) {+ if (this.isIndexContentRunning || this.isIndexContentEntriesRunning) {this.logger.warn({ tag: this.indexContentFiles.name, msg: `SKIP (already running)` })return}- this.isIndexContentFilesRunning = true+ this.isIndexContentRunning = truethis.logger.log({ tag: this.indexContentFiles.name, msg: `START` })try {await this.filesContentIndexer.parseAndIndexAllFiles()this.logger.log({ tag: this.indexContentFiles.name, msg: `END` })} finally {- this.isIndexContentFilesRunning = false+ this.isIndexContentRunning = false}}diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.tsindex b6f65a23..3705bd55 100644--- a/backend/src/applications/files/services/files-search-manager.service.ts+++ b/backend/src/applications/files/services/files-search-manager.service.ts@@ -27,11 +27,11 @@ export class FilesSearchManager {) {}async search(user: UserModel, search: SearchFilesDto): Promise<FileContent[]> {+ if (search.fullText && !configuration.applications.files.contentIndexing.enabled) {+ throw new HttpException('Full-text search is disabled', HttpStatus.BAD_REQUEST)+ }const [spaceIds, shareIds] = await Promise.all([this.spacesQueries.spaceIds(user.id), this.sharesQueries.shareIds(user.id, +user.isAdmin)])if (search.fullText) {- if (!configuration.applications.files.contentIndexing.enabled) {- throw new HttpException('Full-text search is disabled', HttpStatus.BAD_REQUEST)- }return await this.searchFullText(user.id, spaceIds, shareIds, search.content, search.limit)} else {return await this.searchFileNames(user.id, spaceIds, shareIds, search.content, search.limit)diff --git a/backend/src/applications/files/utils/indexing.ts b/backend/src/applications/files/utils/indexing.tsindex c2190f8e..79dace51 100644--- a/backend/src/applications/files/utils/indexing.ts+++ b/backend/src/applications/files/utils/indexing.ts@@ -7,9 +7,13 @@ export function genIndexingKey(id: number, type: FILE_REPOSITORY, sep = '_'): streturn `${type}${sep}${id}`}-export function indexingUpdateCacheKeyFromSpace(userId: number, space: SpaceEnv): string | null {+export function indexingUpdateCacheKeysFromSpace(userId: number, space: SpaceEnv): string[] {+ const cacheKeys: string[] = []+ if (space.inSharesRepository && !space.root.externalPath) {+ // shares that are linked to a space must also be indexed+ cacheKeys.push(`${CACHE_INDEXING_UPDATE_PREFIX}-${genIndexingKey(space.id, FILE_REPOSITORY.SHARE, '-')}`)+ }const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)- if (repository === null) return null- const indexingKey = genIndexingKey(repository.id, repository.type, '-')- return `${CACHE_INDEXING_UPDATE_PREFIX}-${indexingKey}`+ if (repository !== null) cacheKeys.push(`${CACHE_INDEXING_UPDATE_PREFIX}-${genIndexingKey(repository.id, repository.type, '-')}`)+ return cacheKeys}