Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): implement file event manager
johavenc9951d724645
diff --git a/backend/src/applications/files/events/file-events.ts b/backend/src/applications/files/events/file-events.tsnew file mode 100644index 00000000..00e47fe7--- /dev/null+++ b/backend/src/applications/files/events/file-events.ts@@ -0,0 +1,6 @@+import EventEmitter from 'node:events'+import type { FileEventEmit, FileTaskEventEmit } from '../interfaces/file-event.interface'++export const FileTaskEvent: EventEmitter<FileTaskEventEmit> = new EventEmitter<FileTaskEventEmit>()++export const FileEvent: EventEmitter<FileEventEmit> = new EventEmitter<FileEventEmit>()diff --git a/backend/src/applications/files/events/file-task-event.ts b/backend/src/applications/files/events/file-task-event.tsdeleted file mode 100644index 806d688b..00000000--- a/backend/src/applications/files/events/file-task-event.ts+++ /dev/null@@ -1,3 +0,0 @@-import EventEmitter from 'node:events'--export const FileTaskEvent: EventEmitter = new EventEmitter()diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex d1b8200a..111875d4 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -36,7 +36,8 @@ import { FilesQuotaManager } from './services/files-quota-manager.service'FilesParser,FilesContentManager,{ provide: FilesIndexer, useClass: FilesIndexerMySQL },- FilesSearchManager+ FilesSearchManager,+ FilesEventManager,FilesQuotaManager],exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.tsnew file mode 100644index 00000000..b5e5cc0a--- /dev/null+++ b/backend/src/applications/files/interfaces/file-event.interface.ts@@ -0,0 +1,20 @@+import type { SpaceEnv } from '../../spaces/models/space-env.model'+import type { FILE_OPERATION } from '../constants/operations'+import type { ACTION } from '../../../common/constants'+import type { UserModel } from '../../users/models/user.model'++export interface FileTaskEventEmit {+ startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string]+}++export interface FileEventType {+ user: UserModel+ space: SpaceEnv+ action: ACTION+ rPath: string+ oriPath?: string // src from copy/move operations+}++export interface FileEventEmit {+ event: [event: FileEventType]+}diff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-manager.service.tsindex c0af33df..616397e4 100644--- a/backend/src/applications/files/services/files-content-manager.service.ts+++ b/backend/src/applications/files/services/files-content-manager.service.ts@@ -102,7 +102,7 @@ export class FilesContentManager {this.filesIndexer.dropIndex(indexName).catch((e: Error) => this.logger.error({ tag: this.indexFiles.name, msg: `${indexSuffix} - unable to drop index : ${e}` }))- this.logger.log({ tag: this.indexFiles.name, msg: `${indexSuffix} - no data, index not stored` })+ this.logger.verbose({ tag: this.indexFiles.name, msg: `${indexSuffix} - no data, index not stored` })} else {// clean up old recordsconst recordsToDelete: number[] = [...context.db.keys()].filter((key) => !context.fs.has(key))@@ -112,7 +112,7 @@ export class FilesContentManager {.catch((e: Error) => this.logger.error({ tag: this.indexFiles.name, msg: `${indexSuffix} - unable to delete records : ${e}` }))}if (indexedRecords === 0 && errorRecords === 0 && recordsToDelete.length === 0) {- this.logger.log({ tag: this.indexFiles.name, msg: `${indexSuffix} - no new data` })+ this.logger.verbose({ tag: this.indexFiles.name, msg: `${indexSuffix} - no new data` })} else {this.logger.log({tag: this.indexFiles.name,diff --git a/backend/src/applications/files/services/files-event-manager.service.spec.ts b/backend/src/applications/files/services/files-event-manager.service.spec.tsnew file mode 100644index 00000000..4c1214f5--- /dev/null+++ b/backend/src/applications/files/services/files-event-manager.service.spec.ts@@ -0,0 +1,25 @@+import { Test, TestingModule } from '@nestjs/testing'+import { FilesEventManager } from './files-event-manager.service'+import { Cache } from '../../../infrastructure/cache/services/cache.service'++describe(FilesEventManager.name, () => {+ let service: FilesEventManager++ beforeAll(async () => {+ const module: TestingModule = await Test.createTestingModule({+ providers: [+ FilesEventManager,+ {+ provide: Cache,+ useValue: {}+ }+ ]+ }).compile()++ service = module.get<FilesEventManager>(FilesEventManager)+ })++ it('should be defined', () => {+ expect(service).toBeDefined()+ })+})diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.tsnew file mode 100644index 00000000..3fa7e234--- /dev/null+++ b/backend/src/applications/files/services/files-event-manager.service.ts@@ -0,0 +1,136 @@+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'+import { Cache } from '../../../infrastructure/cache/services/cache.service'+import { FileEvent } from '../events/file-events'+import { ACTION } from '../../../common/constants'+import type { FileEventType } from '../interfaces/file-event.interface'+import { CACHE_QUOTA_MUST_UPDATE, CACHE_QUOTA_TTL } from '../constants/cache'+import { quotaCacheKeyFromSpace } from '../utils/quota'++@Injectable()+export class FilesEventManager implements OnModuleDestroy {+ /* Used to:+ - process cache events (storage usage updates, new file indexing) in the main worker+ - handle versioning+ */+ private readonly MAX_BUFFER_SIZE = 1_000+ private readonly MAX_BUFFER_DELAY_MS = 20_000+ private readonly logger = new Logger(FilesEventManager.name)+ private readonly eventsBuffer: FileEventType[] = []+ private quotaEvents: string[] = []+ private flushTimer: NodeJS.Timeout | null = null+ private isFlushing = false+ private flushRequested = false++ constructor(private readonly cache: Cache) {+ FileEvent.on('event', this.onFileEvent)+ }++ async onModuleDestroy(): Promise<void> {+ FileEvent.off('event', this.onFileEvent)+ this.clearFlushTimer()+ await this.flushEvents()+ }++ private readonly onFileEvent = (fileEvent: FileEventType): void => {+ this.logger.verbose({+ tag: this.processEvents.name,+ msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} repository:${fileEvent.space.alias}`+ })+ this.eventsBuffer.push(fileEvent)+ if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {+ void this.flushEvents()+ return+ }+ this.startFlushTimer()+ }++ private startFlushTimer(): void {+ if (this.flushTimer) {+ return+ }+ this.flushTimer = setTimeout(() => void this.flushEvents(), this.MAX_BUFFER_DELAY_MS)+ }++ private clearFlushTimer(): void {+ if (!this.flushTimer) {+ return+ }+ clearTimeout(this.flushTimer)+ this.flushTimer = null+ }++ private async flushEvents() {+ this.clearFlushTimer()+ if (this.isFlushing) {+ this.flushRequested = true+ return+ }+ if (!this.eventsBuffer.length) {+ return+ }+ this.isFlushing = true+ const events = this.eventsBuffer.splice(0, this.eventsBuffer.length)+ try {+ await this.processEvents(events)+ } catch (e) {+ this.logger.error({ tag: this.flushEvents.name, msg: `Could not process buffered events: ${e}` })+ this.eventsBuffer.unshift(...events)+ } finally {+ this.isFlushing = false+ }+ if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE || this.flushRequested) {+ this.flushRequested = false+ await this.flushEvents()+ return+ }+ if (this.eventsBuffer.length) {+ this.startFlushTimer()+ }+ }++ private async processEvents(fileEvents: FileEventType[]) {+ this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })+ for (const event of fileEvents) {+ this.processQuotaEvent(event)+ }+ await this.storeEventsInCache()+ }++ private async storeEventsInCache() {+ if (!this.quotaEvents.length) return+ const failedCacheKeys: string[] = []+ await Promise.all(+ this.quotaEvents.map(async (cacheKey) => {+ try {+ const ok = await this.cache.set(cacheKey, true, CACHE_QUOTA_TTL)+ if (!ok) {+ failedCacheKeys.push(cacheKey)+ }+ } catch {+ failedCacheKeys.push(cacheKey)+ }+ })+ )+ if (failedCacheKeys.length) {+ this.logger.warn({+ tag: this.storeEventsInCache.name,+ msg: `Could not cache quota keys (${failedCacheKeys.length})`+ })+ }+ this.quotaEvents = failedCacheKeys+ }++ private processQuotaEvent(fileEvent: FileEventType) {+ // Ignore files moved to the trash; storage usage remains unchanged.+ if (fileEvent.action === ACTION.DELETE) return+ let cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space)+ if (!cacheKey) {+ this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })+ return+ }+ cacheKey = `${CACHE_QUOTA_MUST_UPDATE}-${cacheKey}`+ if (this.quotaEvents.indexOf(cacheKey) === -1) {+ this.quotaEvents.push(cacheKey)+ }+ }+}diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.tsindex bfb2d928..d157df4b 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts@@ -4,18 +4,17 @@ import crypto from 'node:crypto'import path from 'node:path'import { setTimeout } from 'node:timers/promises'import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'-import { currentTimeStamp } from '../../../common/shared'import { Cache } from '../../../infrastructure/cache/services/cache.service'import { SpaceEnv } from '../../spaces/models/space-env.model'import { UserModel } from '../../users/models/user.model'import { CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'import { FILE_OPERATION } from '../constants/operations'-import { FileTaskEvent } from '../events/file-task-event'import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'import { countDirEntries, dirName, fileName, fileSize, isPathIsDir, removeFiles } from '../utils/files'import { SendFile } from '../utils/send-file'import { FilesMethods } from './files-methods.service'+import { FileTaskEvent } from '../events/file-events'@Injectable()export class FilesTasksManager {