Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): normalize quota cache keys and content-index naming
johaven7fdf46a98cdb
diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.tsindex d5e2c0d1..86738ec7 100644--- a/backend/src/applications/files/constants/cache.ts+++ b/backend/src/applications/files/constants/cache.ts@@ -6,8 +6,6 @@ export const CACHE_LOCK_PREFIX = 'flock' as constexport const CACHE_LOCK_DEFAULT_TTL = 28800 as const // 8 hours in secondsexport const CACHE_LOCK_FILE_TTL = 3600 as const// cache quota key = `(quota-user|quota-space)-${id}` => number-export const CACHE_QUOTA_USER_PREFIX = 'quota-user'-export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'-export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'-export const CACHE_QUOTA_MUST_UPDATE = 'must-update'+export const CACHE_QUOTA_PREFIX = 'quota' as const+export const CACHE_QUOTA_EVENT_UPDATE_PREFIX = 'event-update-quota' as constexport const CACHE_QUOTA_TTL = 86400 // 1 daydiff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.tsindex b688da7c..fdf8cc99 100644--- a/backend/src/applications/files/constants/indexing.ts+++ b/backend/src/applications/files/constants/indexing.ts@@ -1,5 +1,2 @@-export const userIndexPrefix = 'user_'-export const spaceIndexPrefix = 'space_'-export const shareIndexPrefix = 'share_'export const minCharsToSearch = 3export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])diff --git a/backend/src/applications/files/constants/operations.ts b/backend/src/applications/files/constants/operations.tsindex ce800d45..9956ce63 100644--- a/backend/src/applications/files/constants/operations.ts+++ b/backend/src/applications/files/constants/operations.ts@@ -20,4 +20,10 @@ export enum FILE_MODE {EDIT = 'edit'}+export enum FILE_REPOSITORY {+ USER = 'user',+ SPACE = 'space',+ SHARE = 'share'+}+export const FORCE_AS_FILE_OWNER = 'forceAsFileOwner' as constdiff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.tsindex eb825908..06d7845e 100644--- a/backend/src/applications/files/interfaces/file-parse-index.ts+++ b/backend/src/applications/files/interfaces/file-parse-index.ts@@ -1,5 +1,3 @@-export type FileParseType = 'user' | 'space' | 'share'-export interface FileParseContext {realPath: stringpathPrefix: stringdiff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-manager.service.tsindex 616397e4..c609aacd 100644--- a/backend/src/applications/files/services/files-content-manager.service.ts+++ b/backend/src/applications/files/services/files-content-manager.service.ts@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'import fs from 'fs/promises'import { Stats } from 'node:fs'import path from 'node:path'-import { indexableExtensions, shareIndexPrefix, spaceIndexPrefix, userIndexPrefix } from '../constants/indexing'+import { indexableExtensions } from '../constants/indexing'import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'import { FilesIndexer } from '../models/files-indexer'import { FileContent } from '../schemas/file-content.interface'@@ -10,6 +10,7 @@ import { docTextify } from '../utils/doc-textify/doc-textify'import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'import { getMimeType } from '../utils/files'import { FilesParser } from './files-parser.service'+import { FILE_REPOSITORY } from '../constants/operations'@Injectable()export class FilesContentManager {@@ -34,14 +35,14 @@ export class FilesContentManager {for await (const [id, type, paths] of this.filesParser.allPaths()) {let indexSuffix: stringswitch (type) {- case 'user':- indexSuffix = `${userIndexPrefix}${id}`+ case FILE_REPOSITORY.USER:+ indexSuffix = `${FILE_REPOSITORY.USER}_${id}`break- case 'space':- indexSuffix = `${spaceIndexPrefix}${id}`+ case FILE_REPOSITORY.SPACE:+ indexSuffix = `${FILE_REPOSITORY.SPACE}_${id}`break- case 'share':- indexSuffix = `${shareIndexPrefix}${id}`+ case FILE_REPOSITORY.SHARE:+ indexSuffix = `${FILE_REPOSITORY.SHARE}_${id}`}try {await this.indexFiles(indexSuffix, paths)diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.tsindex 3fa7e234..9c1f9b58 100644--- a/backend/src/applications/files/services/files-event-manager.service.ts+++ b/backend/src/applications/files/services/files-event-manager.service.ts@@ -3,7 +3,7 @@ import { Cache } from '../../../infrastructure/cache/services/cache.service'import { FileEvent } from '../events/file-events'import { ACTION } from '../../../common/constants'import type { FileEventType } from '../interfaces/file-event.interface'-import { CACHE_QUOTA_MUST_UPDATE, CACHE_QUOTA_TTL } from '../constants/cache'+import { CACHE_QUOTA_TTL } from '../constants/cache'import { quotaCacheKeyFromSpace } from '../utils/quota'@Injectable()@@ -91,7 +91,11 @@ export class FilesEventManager implements OnModuleDestroy {private async processEvents(fileEvents: FileEventType[]) {this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })for (const event of fileEvents) {- this.processQuotaEvent(event)+ try {+ this.processQuotaEvent(event)+ } catch (e) {+ this.logger.warn({ tag: this.processEvents.name, msg: `Could not process quota event: ${JSON.stringify(event)} - ${e}` })+ }}await this.storeEventsInCache()}@@ -123,12 +127,11 @@ export class FilesEventManager implements OnModuleDestroy {private processQuotaEvent(fileEvent: FileEventType) {// Ignore files moved to the trash; storage usage remains unchanged.if (fileEvent.action === ACTION.DELETE) return- let cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space)+ const cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space, true)if (!cacheKey) {this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })return}- cacheKey = `${CACHE_QUOTA_MUST_UPDATE}-${cacheKey}`if (this.quotaEvents.indexOf(cacheKey) === -1) {this.quotaEvents.push(cacheKey)}diff --git a/backend/src/applications/files/services/files-parser.service.ts b/backend/src/applications/files/services/files-parser.service.tsindex e0c5cd82..aee1ad26 100644--- a/backend/src/applications/files/services/files-parser.service.ts+++ b/backend/src/applications/files/services/files-parser.service.ts@@ -13,9 +13,10 @@ import { spaces } from '../../spaces/schemas/spaces.schema'import { USER_ROLE } from '../../users/constants/user'import { UserModel } from '../../users/models/user.model'import { users } from '../../users/schemas/users.schema'-import { FileParseContext, FileParseType } from '../interfaces/file-parse-index'+import { FileParseContext } from '../interfaces/file-parse-index'import { filePathSQL, files } from '../schemas/files.schema'import { isPathExists } from '../utils/files'+import { FILE_REPOSITORY } from '../constants/operations'@Injectable()export class FilesParser {@@ -23,13 +24,13 @@ export class FilesParser {constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}- async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {+ async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {yield* this.userPaths(userId)yield* this.spacePaths(spaceIds)yield* this.sharePaths(shareIds)}- async *userPaths(userId?: number): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {+ async *userPaths(userId?: number): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {for (const user of await this.db.select({id: users.id,@@ -42,11 +43,15 @@ export class FilesParser {this.logger.warn({ tag: this.userPaths.name, msg: `user path does not exist : ${userFilesPath}` })continue}- yield [user.id, 'user', [{ realPath: userFilesPath, pathPrefix: `${SPACE_REPOSITORY.FILES}/${SPACE_ALIAS.PERSONAL}`, isDir: true }]]+ yield [+ user.id,+ FILE_REPOSITORY.USER,+ [{ realPath: userFilesPath, pathPrefix: `${SPACE_REPOSITORY.FILES}/${SPACE_ALIAS.PERSONAL}`, isDir: true }]+ ]}}- async *spacePaths(spaceIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {+ async *spacePaths(spaceIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {for (const space of await this.db.select({id: spaces.id,@@ -87,11 +92,11 @@ export class FilesParser {isDir: r.isDir})- yield [space.id, 'space', [...spacePath, ...rootPaths]]+ yield [space.id, FILE_REPOSITORY.SPACE, [...spacePath, ...rootPaths]]}}- async *sharePaths(shareIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {+ async *sharePaths(shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {for (const share of await this.db.select({id: shares.id,@@ -129,7 +134,11 @@ export class FilesParser {this.logger.warn({ tag: this.sharePaths.name, msg: `share path does not exist : ${shareFilesPath}` })continue}- yield [share.id, 'share', [{ realPath: shareFilesPath, pathPrefix: `${SPACE_REPOSITORY.SHARES}/${share.alias}`, isDir: share.isDir }]]+ yield [+ share.id,+ FILE_REPOSITORY.SHARE,+ [{ realPath: shareFilesPath, pathPrefix: `${SPACE_REPOSITORY.SHARES}/${share.alias}`, isDir: share.isDir }]+ ]}}}diff --git a/backend/src/applications/files/services/files-quota-manager.service.ts b/backend/src/applications/files/services/files-quota-manager.service.tsindex 13c2b19f..0a79957d 100644--- a/backend/src/applications/files/services/files-quota-manager.service.ts+++ b/backend/src/applications/files/services/files-quota-manager.service.ts@@ -11,14 +11,9 @@ import { dirSize, isPathExists } from '../utils/files'import { SpaceModel } from '../../spaces/models/space.model'import { SharesQueries } from '../../shares/services/shares-queries.service'import { SpaceEnv } from '../../spaces/models/space-env.model'-import {- CACHE_QUOTA_MUST_UPDATE,- CACHE_QUOTA_SHARE_PREFIX,- CACHE_QUOTA_SPACE_PREFIX,- CACHE_QUOTA_TTL,- CACHE_QUOTA_USER_PREFIX-} from '../constants/cache'-import { quotaCacheKeyFromSpace } from '../utils/quota'+import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_TTL } from '../constants/cache'+import { genQuotaCacheKey, quotaCacheKeyFromSpace } from '../utils/quota'+import { FILE_REPOSITORY } from '../constants/operations'@Injectable()export class FilesQuotaManager {@@ -62,16 +57,16 @@ export class FilesQuotaManager {}async updateQuotaEntries() {- for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_MUST_UPDATE}-quota-*`)) {+ for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_EVENT_UPDATE_PREFIX}-*`)) {try {const keySegments = k.split('-')const [repository, idPart] = keySegments.slice(-2)const id = Number.parseInt(idPart ?? '', 10)- if (repository === 'user') {+ if (repository === FILE_REPOSITORY.USER) {await this.updatePersonalSpacesQuota(id)- } else if (repository === 'space') {+ } else if (repository === FILE_REPOSITORY.SPACE) {await this.updateSpacesQuota(id)- } else if (repository === 'share') {+ } else if (repository === FILE_REPOSITORY.SHARE) {await this.updateSharesExternalPathQuota(id)} else {this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unknown type: ${repository}` })@@ -100,7 +95,7 @@ export class FilesQuotaManager {}const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }this.spacesQueries.cache- .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)+ .set(genQuotaCacheKey(user.id, FILE_REPOSITORY.USER), spaceQuota, CACHE_QUOTA_TTL).catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))if (user.storageUsage !== spaceQuota.storageUsage) {this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(@@ -137,7 +132,7 @@ export class FilesQuotaManager {}const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }this.spacesQueries.cache- .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)+ .set(genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE), spaceQuota, CACHE_QUOTA_TTL).catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))if (space.storageUsage !== spaceQuota.storageUsage) {this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(@@ -169,7 +164,7 @@ export class FilesQuotaManager {}const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }this.sharesQueries.cache- .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)+ .set(genQuotaCacheKey(share.id, FILE_REPOSITORY.SHARE), shareQuota, CACHE_QUOTA_TTL).catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))if (share.storageUsage !== shareQuota.storageUsage) {this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.tsindex 8a7e4865..765029d5 100644--- a/backend/src/applications/files/services/files-search-manager.service.ts+++ b/backend/src/applications/files/services/files-search-manager.service.ts@@ -6,13 +6,13 @@ import { configuration } from '../../../configuration/config.environment'import { SharesQueries } from '../../shares/services/shares-queries.service'import { SpacesQueries } from '../../spaces/services/spaces-queries.service'import { UserModel } from '../../users/models/user.model'-import { shareIndexPrefix, spaceIndexPrefix, userIndexPrefix } from '../constants/indexing'import { SearchFilesDto } from '../dto/file-operations.dto'import { FilesIndexer } from '../models/files-indexer'import { FileContent } from '../schemas/file-content.interface'import { dirName, fileName, getMimeType } from '../utils/files'import { genRegexPositiveAndNegativeTerms } from '../utils/files-search'import { FilesParser } from './files-parser.service'+import { FILE_REPOSITORY } from '../constants/operations'@Injectable()export class FilesSearchManager {@@ -39,9 +39,9 @@ export class FilesSearchManager {private async searchFullText(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {const indexNames = await this.filesIndexer.existingIndexes([- `${userIndexPrefix}${userId}`,- ...spaceIds.map((id) => `${spaceIndexPrefix}${id}`),- ...shareIds.map((id) => `${shareIndexPrefix}${id}`)+ `${FILE_REPOSITORY.USER}_${userId}`,+ ...spaceIds.map((id) => `${FILE_REPOSITORY.SPACE}_${id}`),+ ...shareIds.map((id) => `${FILE_REPOSITORY.SHARE}_${id}`)])if (indexNames.length === 0) {return []diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.tsindex 1f085d3c..3f8093e9 100644--- a/backend/src/applications/files/utils/quota.ts+++ b/backend/src/applications/files/utils/quota.ts@@ -1,23 +1,28 @@import { SpaceEnv } from '../../spaces/models/space-env.model'-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'+import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_PREFIX } from '../constants/cache'+import { FILE_REPOSITORY } from '../constants/operations'-export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv) {+export function genQuotaCacheKey(id: number, type: FILE_REPOSITORY, isEventUpdate = false): string {+ return `${isEventUpdate ? CACHE_QUOTA_EVENT_UPDATE_PREFIX : CACHE_QUOTA_PREFIX}-${type}-${id}`+}++export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string {if (space.inPersonalSpace) {// Personal user space- return `${CACHE_QUOTA_USER_PREFIX}-${userId}`+ return genQuotaCacheKey(userId, FILE_REPOSITORY.USER, isEventUpdate)} else if (space.root?.externalPath) {// External paths used as shares or as space roots share the same quota as their originif (space.inSharesRepository) {- return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`+ return genQuotaCacheKey(space.root?.externalParentShareId || space.id, FILE_REPOSITORY.SHARE, isEventUpdate)}- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`- } else if (space.root.file?.path && space.root.owner?.login) {+ return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)+ } else if (space.root?.file?.path && space.root.owner?.login) {// Space root is linked to a user file- return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`- } else if (space.root.file?.space?.id) {- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`+ return genQuotaCacheKey(space.root.owner.id, FILE_REPOSITORY.USER, isEventUpdate)+ } else if (space.root?.file?.space?.id) {+ return genQuotaCacheKey(space.root.file.space.id, FILE_REPOSITORY.SPACE, isEventUpdate)} else if (space.id) {- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`+ return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)} else {return null}diff --git a/backend/src/applications/spaces/services/spaces-manager.service.ts b/backend/src/applications/spaces/services/spaces-manager.service.tsindex 9e71059f..3b9b6371 100644--- a/backend/src/applications/spaces/services/spaces-manager.service.ts+++ b/backend/src/applications/spaces/services/spaces-manager.service.ts@@ -46,7 +46,8 @@ import { spaces } from '../schemas/spaces.schema'import { haveSpacePermission } from '../utils/permissions'import { SpacesQueries } from './spaces-queries.service'import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'-import { CACHE_QUOTA_SPACE_PREFIX } from '../../files/constants/cache'+import { genQuotaCacheKey } from '../../files/utils/quota'+import { FILE_REPOSITORY } from '../../files/constants/operations'@Injectable()export class SpacesManager {@@ -666,7 +667,7 @@ export class SpacesManager {await this.spacesQueries.deleteSpace(space.id, deleteNow)if (deleteNow) {this.spacesQueries.cache- .del(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`)+ .del(genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE)).catch((e: Error) => this.logger.error({ tag: this.deleteOrDisableSpace.name, msg: `${e}` }))await this.deleteSpaceLocation(space.alias)}