Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): implement file event manager

johavenc9951d724645
diff --git a/backend/src/applications/files/events/file-events.ts b/backend/src/applications/files/events/file-events.ts
new file mode 100644
index 00000000..00e47fe7
--- /dev/null
+++ b/backend/src/applications/files/events/file-events.ts
@@ -0,0 +1,6 @@
+import EventEmitter from 'node:events'
+import type { FileEventEmit, FileTaskEventEmit } from '../interfaces/file-event.interface'
+
+export const FileTaskEvent: EventEmitter<FileTaskEventEmit> = new EventEmitter<FileTaskEventEmit>()
+
+export const FileEvent: EventEmitter<FileEventEmit> = new EventEmitter<FileEventEmit>()
diff --git a/backend/src/applications/files/events/file-task-event.ts b/backend/src/applications/files/events/file-task-event.ts
deleted file mode 100644
index 806d688b..00000000
--- a/backend/src/applications/files/events/file-task-event.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-import EventEmitter from 'node:events'
-
-export const FileTaskEvent: EventEmitter = new EventEmitter()
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index d1b8200a..111875d4 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -36,7 +36,8 @@ import { FilesQuotaManager } from './services/files-quota-manager.service'
FilesParser,
FilesContentManager,
{ provide: FilesIndexer, useClass: FilesIndexerMySQL },
- FilesSearchManager
+ FilesSearchManager,
+ FilesEventManager,
FilesQuotaManager
],
exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.ts
new file mode 100644
index 00000000..b5e5cc0a
--- /dev/null
+++ b/backend/src/applications/files/interfaces/file-event.interface.ts
@@ -0,0 +1,20 @@
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+import type { FILE_OPERATION } from '../constants/operations'
+import type { ACTION } from '../../../common/constants'
+import type { UserModel } from '../../users/models/user.model'
+
+export interface FileTaskEventEmit {
+ startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string]
+}
+
+export interface FileEventType {
+ user: UserModel
+ space: SpaceEnv
+ action: ACTION
+ rPath: string
+ oriPath?: string // src from copy/move operations
+}
+
+export interface FileEventEmit {
+ event: [event: FileEventType]
+}
diff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-manager.service.ts
index c0af33df..616397e4 100644
--- a/backend/src/applications/files/services/files-content-manager.service.ts
+++ b/backend/src/applications/files/services/files-content-manager.service.ts
@@ -102,7 +102,7 @@ export class FilesContentManager {
this.filesIndexer
.dropIndex(indexName)
.catch((e: Error) => this.logger.error({ tag: this.indexFiles.name, msg: `${indexSuffix} - unable to drop index : ${e}` }))
- this.logger.log({ tag: this.indexFiles.name, msg: `${indexSuffix} - no data, index not stored` })
+ this.logger.verbose({ tag: this.indexFiles.name, msg: `${indexSuffix} - no data, index not stored` })
} else {
// clean up old records
const recordsToDelete: number[] = [...context.db.keys()].filter((key) => !context.fs.has(key))
@@ -112,7 +112,7 @@ export class FilesContentManager {
.catch((e: Error) => this.logger.error({ tag: this.indexFiles.name, msg: `${indexSuffix} - unable to delete records : ${e}` }))
}
if (indexedRecords === 0 && errorRecords === 0 && recordsToDelete.length === 0) {
- this.logger.log({ tag: this.indexFiles.name, msg: `${indexSuffix} - no new data` })
+ this.logger.verbose({ tag: this.indexFiles.name, msg: `${indexSuffix} - no new data` })
} else {
this.logger.log({
tag: this.indexFiles.name,
diff --git a/backend/src/applications/files/services/files-event-manager.service.spec.ts b/backend/src/applications/files/services/files-event-manager.service.spec.ts
new file mode 100644
index 00000000..4c1214f5
--- /dev/null
+++ b/backend/src/applications/files/services/files-event-manager.service.spec.ts
@@ -0,0 +1,25 @@
+import { Test, TestingModule } from '@nestjs/testing'
+import { FilesEventManager } from './files-event-manager.service'
+import { Cache } from '../../../infrastructure/cache/services/cache.service'
+
+describe(FilesEventManager.name, () => {
+ let service: FilesEventManager
+
+ beforeAll(async () => {
+ const module: TestingModule = await Test.createTestingModule({
+ providers: [
+ FilesEventManager,
+ {
+ provide: Cache,
+ useValue: {}
+ }
+ ]
+ }).compile()
+
+ service = module.get<FilesEventManager>(FilesEventManager)
+ })
+
+ it('should be defined', () => {
+ expect(service).toBeDefined()
+ })
+})
diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.ts
new file mode 100644
index 00000000..3fa7e234
--- /dev/null
+++ b/backend/src/applications/files/services/files-event-manager.service.ts
@@ -0,0 +1,136 @@
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'
+import { Cache } from '../../../infrastructure/cache/services/cache.service'
+import { FileEvent } from '../events/file-events'
+import { ACTION } from '../../../common/constants'
+import type { FileEventType } from '../interfaces/file-event.interface'
+import { CACHE_QUOTA_MUST_UPDATE, CACHE_QUOTA_TTL } from '../constants/cache'
+import { quotaCacheKeyFromSpace } from '../utils/quota'
+
+@Injectable()
+export class FilesEventManager implements OnModuleDestroy {
+ /* Used to:
+ - process cache events (storage usage updates, new file indexing) in the main worker
+ - handle versioning
+ */
+ private readonly MAX_BUFFER_SIZE = 1_000
+ private readonly MAX_BUFFER_DELAY_MS = 20_000
+ private readonly logger = new Logger(FilesEventManager.name)
+ private readonly eventsBuffer: FileEventType[] = []
+ private quotaEvents: string[] = []
+ private flushTimer: NodeJS.Timeout | null = null
+ private isFlushing = false
+ private flushRequested = false
+
+ constructor(private readonly cache: Cache) {
+ FileEvent.on('event', this.onFileEvent)
+ }
+
+ async onModuleDestroy(): Promise<void> {
+ FileEvent.off('event', this.onFileEvent)
+ this.clearFlushTimer()
+ await this.flushEvents()
+ }
+
+ private readonly onFileEvent = (fileEvent: FileEventType): void => {
+ this.logger.verbose({
+ tag: this.processEvents.name,
+ msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} repository:${fileEvent.space.alias}`
+ })
+ this.eventsBuffer.push(fileEvent)
+ if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {
+ void this.flushEvents()
+ return
+ }
+ this.startFlushTimer()
+ }
+
+ private startFlushTimer(): void {
+ if (this.flushTimer) {
+ return
+ }
+ this.flushTimer = setTimeout(() => void this.flushEvents(), this.MAX_BUFFER_DELAY_MS)
+ }
+
+ private clearFlushTimer(): void {
+ if (!this.flushTimer) {
+ return
+ }
+ clearTimeout(this.flushTimer)
+ this.flushTimer = null
+ }
+
+ private async flushEvents() {
+ this.clearFlushTimer()
+ if (this.isFlushing) {
+ this.flushRequested = true
+ return
+ }
+ if (!this.eventsBuffer.length) {
+ return
+ }
+ this.isFlushing = true
+ const events = this.eventsBuffer.splice(0, this.eventsBuffer.length)
+ try {
+ await this.processEvents(events)
+ } catch (e) {
+ this.logger.error({ tag: this.flushEvents.name, msg: `Could not process buffered events: ${e}` })
+ this.eventsBuffer.unshift(...events)
+ } finally {
+ this.isFlushing = false
+ }
+ if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE || this.flushRequested) {
+ this.flushRequested = false
+ await this.flushEvents()
+ return
+ }
+ if (this.eventsBuffer.length) {
+ this.startFlushTimer()
+ }
+ }
+
+ private async processEvents(fileEvents: FileEventType[]) {
+ this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })
+ for (const event of fileEvents) {
+ this.processQuotaEvent(event)
+ }
+ await this.storeEventsInCache()
+ }
+
+ private async storeEventsInCache() {
+ if (!this.quotaEvents.length) return
+ const failedCacheKeys: string[] = []
+ await Promise.all(
+ this.quotaEvents.map(async (cacheKey) => {
+ try {
+ const ok = await this.cache.set(cacheKey, true, CACHE_QUOTA_TTL)
+ if (!ok) {
+ failedCacheKeys.push(cacheKey)
+ }
+ } catch {
+ failedCacheKeys.push(cacheKey)
+ }
+ })
+ )
+ if (failedCacheKeys.length) {
+ this.logger.warn({
+ tag: this.storeEventsInCache.name,
+ msg: `Could not cache quota keys (${failedCacheKeys.length})`
+ })
+ }
+ this.quotaEvents = failedCacheKeys
+ }
+
+ private processQuotaEvent(fileEvent: FileEventType) {
+ // Ignore files moved to the trash; storage usage remains unchanged.
+ if (fileEvent.action === ACTION.DELETE) return
+ let cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space)
+ if (!cacheKey) {
+ this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })
+ return
+ }
+ cacheKey = `${CACHE_QUOTA_MUST_UPDATE}-${cacheKey}`
+ if (this.quotaEvents.indexOf(cacheKey) === -1) {
+ this.quotaEvents.push(cacheKey)
+ }
+ }
+}
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.ts
index bfb2d928..d157df4b 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts
@@ -4,18 +4,17 @@ import crypto from 'node:crypto'
import path from 'node:path'
import { setTimeout } from 'node:timers/promises'
import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'
-
import { currentTimeStamp } from '../../../common/shared'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { SpaceEnv } from '../../spaces/models/space-env.model'
import { UserModel } from '../../users/models/user.model'
import { CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
import { FILE_OPERATION } from '../constants/operations'
-import { FileTaskEvent } from '../events/file-task-event'
import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'
import { countDirEntries, dirName, fileName, fileSize, isPathIsDir, removeFiles } from '../utils/files'
import { SendFile } from '../utils/send-file'
import { FilesMethods } from './files-methods.service'
+import { FileTaskEvent } from '../events/file-events'
@Injectable()
export class FilesTasksManager {