Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): normalize quota cache keys and content-index naming

johaven7fdf46a98cdb
diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.ts
index d5e2c0d1..86738ec7 100644
--- a/backend/src/applications/files/constants/cache.ts
+++ b/backend/src/applications/files/constants/cache.ts
@@ -6,8 +6,6 @@ export const CACHE_LOCK_PREFIX = 'flock' as const
export const CACHE_LOCK_DEFAULT_TTL = 28800 as const // 8 hours in seconds
export const CACHE_LOCK_FILE_TTL = 3600 as const
// cache quota key = `(quota-user|quota-space)-${id}` => number
-export const CACHE_QUOTA_USER_PREFIX = 'quota-user'
-export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'
-export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'
-export const CACHE_QUOTA_MUST_UPDATE = 'must-update'
+export const CACHE_QUOTA_PREFIX = 'quota' as const
+export const CACHE_QUOTA_EVENT_UPDATE_PREFIX = 'event-update-quota' as const
export const CACHE_QUOTA_TTL = 86400 // 1 day
diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.ts
index b688da7c..fdf8cc99 100644
--- a/backend/src/applications/files/constants/indexing.ts
+++ b/backend/src/applications/files/constants/indexing.ts
@@ -1,5 +1,2 @@
-export const userIndexPrefix = 'user_'
-export const spaceIndexPrefix = 'space_'
-export const shareIndexPrefix = 'share_'
export const minCharsToSearch = 3
export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
diff --git a/backend/src/applications/files/constants/operations.ts b/backend/src/applications/files/constants/operations.ts
index ce800d45..9956ce63 100644
--- a/backend/src/applications/files/constants/operations.ts
+++ b/backend/src/applications/files/constants/operations.ts
@@ -20,4 +20,10 @@ export enum FILE_MODE {
EDIT = 'edit'
}
+export enum FILE_REPOSITORY {
+ USER = 'user',
+ SPACE = 'space',
+ SHARE = 'share'
+}
+
export const FORCE_AS_FILE_OWNER = 'forceAsFileOwner' as const
diff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.ts
index eb825908..06d7845e 100644
--- a/backend/src/applications/files/interfaces/file-parse-index.ts
+++ b/backend/src/applications/files/interfaces/file-parse-index.ts
@@ -1,5 +1,3 @@
-export type FileParseType = 'user' | 'space' | 'share'
-
export interface FileParseContext {
realPath: string
pathPrefix: string
diff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-manager.service.ts
index 616397e4..c609aacd 100644
--- a/backend/src/applications/files/services/files-content-manager.service.ts
+++ b/backend/src/applications/files/services/files-content-manager.service.ts
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common'
import fs from 'fs/promises'
import { Stats } from 'node:fs'
import path from 'node:path'
-import { indexableExtensions, shareIndexPrefix, spaceIndexPrefix, userIndexPrefix } from '../constants/indexing'
+import { indexableExtensions } from '../constants/indexing'
import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'
import { FilesIndexer } from '../models/files-indexer'
import { FileContent } from '../schemas/file-content.interface'
@@ -10,6 +10,7 @@ import { docTextify } from '../utils/doc-textify/doc-textify'
import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'
import { getMimeType } from '../utils/files'
import { FilesParser } from './files-parser.service'
+import { FILE_REPOSITORY } from '../constants/operations'
@Injectable()
export class FilesContentManager {
@@ -34,14 +35,14 @@ export class FilesContentManager {
for await (const [id, type, paths] of this.filesParser.allPaths()) {
let indexSuffix: string
switch (type) {
- case 'user':
- indexSuffix = `${userIndexPrefix}${id}`
+ case FILE_REPOSITORY.USER:
+ indexSuffix = `${FILE_REPOSITORY.USER}_${id}`
break
- case 'space':
- indexSuffix = `${spaceIndexPrefix}${id}`
+ case FILE_REPOSITORY.SPACE:
+ indexSuffix = `${FILE_REPOSITORY.SPACE}_${id}`
break
- case 'share':
- indexSuffix = `${shareIndexPrefix}${id}`
+ case FILE_REPOSITORY.SHARE:
+ indexSuffix = `${FILE_REPOSITORY.SHARE}_${id}`
}
try {
await this.indexFiles(indexSuffix, paths)
diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.ts
index 3fa7e234..9c1f9b58 100644
--- a/backend/src/applications/files/services/files-event-manager.service.ts
+++ b/backend/src/applications/files/services/files-event-manager.service.ts
@@ -3,7 +3,7 @@ import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { FileEvent } from '../events/file-events'
import { ACTION } from '../../../common/constants'
import type { FileEventType } from '../interfaces/file-event.interface'
-import { CACHE_QUOTA_MUST_UPDATE, CACHE_QUOTA_TTL } from '../constants/cache'
+import { CACHE_QUOTA_TTL } from '../constants/cache'
import { quotaCacheKeyFromSpace } from '../utils/quota'
@Injectable()
@@ -91,7 +91,11 @@ export class FilesEventManager implements OnModuleDestroy {
private async processEvents(fileEvents: FileEventType[]) {
this.logger.verbose({ tag: this.processEvents.name, msg: `Processing ${fileEvents.length} file event(s)` })
for (const event of fileEvents) {
- this.processQuotaEvent(event)
+ try {
+ this.processQuotaEvent(event)
+ } catch (e) {
+ this.logger.warn({ tag: this.processEvents.name, msg: `Could not process quota event: ${JSON.stringify(event)} - ${e}` })
+ }
}
await this.storeEventsInCache()
}
@@ -123,12 +127,11 @@ export class FilesEventManager implements OnModuleDestroy {
private processQuotaEvent(fileEvent: FileEventType) {
// Ignore files moved to the trash; storage usage remains unchanged.
if (fileEvent.action === ACTION.DELETE) return
- let cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space)
+ const cacheKey = quotaCacheKeyFromSpace(fileEvent.user.id, fileEvent.space, true)
if (!cacheKey) {
this.logger.warn({ tag: this.processQuotaEvent.name, msg: `Unable to determine space location: ${fileEvent.space.id}` })
return
}
- cacheKey = `${CACHE_QUOTA_MUST_UPDATE}-${cacheKey}`
if (this.quotaEvents.indexOf(cacheKey) === -1) {
this.quotaEvents.push(cacheKey)
}
diff --git a/backend/src/applications/files/services/files-parser.service.ts b/backend/src/applications/files/services/files-parser.service.ts
index e0c5cd82..aee1ad26 100644
--- a/backend/src/applications/files/services/files-parser.service.ts
+++ b/backend/src/applications/files/services/files-parser.service.ts
@@ -13,9 +13,10 @@ import { spaces } from '../../spaces/schemas/spaces.schema'
import { USER_ROLE } from '../../users/constants/user'
import { UserModel } from '../../users/models/user.model'
import { users } from '../../users/schemas/users.schema'
-import { FileParseContext, FileParseType } from '../interfaces/file-parse-index'
+import { FileParseContext } from '../interfaces/file-parse-index'
import { filePathSQL, files } from '../schemas/files.schema'
import { isPathExists } from '../utils/files'
+import { FILE_REPOSITORY } from '../constants/operations'
@Injectable()
export class FilesParser {
@@ -23,13 +24,13 @@ export class FilesParser {
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
- async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {
+ async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
yield* this.userPaths(userId)
yield* this.spacePaths(spaceIds)
yield* this.sharePaths(shareIds)
}
- async *userPaths(userId?: number): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {
+ async *userPaths(userId?: number): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
for (const user of await this.db
.select({
id: users.id,
@@ -42,11 +43,15 @@ export class FilesParser {
this.logger.warn({ tag: this.userPaths.name, msg: `user path does not exist : ${userFilesPath}` })
continue
}
- yield [user.id, 'user', [{ realPath: userFilesPath, pathPrefix: `${SPACE_REPOSITORY.FILES}/${SPACE_ALIAS.PERSONAL}`, isDir: true }]]
+ yield [
+ user.id,
+ FILE_REPOSITORY.USER,
+ [{ realPath: userFilesPath, pathPrefix: `${SPACE_REPOSITORY.FILES}/${SPACE_ALIAS.PERSONAL}`, isDir: true }]
+ ]
}
}
- async *spacePaths(spaceIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {
+ async *spacePaths(spaceIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
for (const space of await this.db
.select({
id: spaces.id,
@@ -87,11 +92,11 @@ export class FilesParser {
isDir: r.isDir
}
)
- yield [space.id, 'space', [...spacePath, ...rootPaths]]
+ yield [space.id, FILE_REPOSITORY.SPACE, [...spacePath, ...rootPaths]]
}
}
- async *sharePaths(shareIds?: number[]): AsyncGenerator<[number, FileParseType, FileParseContext[]]> {
+ async *sharePaths(shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
for (const share of await this.db
.select({
id: shares.id,
@@ -129,7 +134,11 @@ export class FilesParser {
this.logger.warn({ tag: this.sharePaths.name, msg: `share path does not exist : ${shareFilesPath}` })
continue
}
- yield [share.id, 'share', [{ realPath: shareFilesPath, pathPrefix: `${SPACE_REPOSITORY.SHARES}/${share.alias}`, isDir: share.isDir }]]
+ yield [
+ share.id,
+ FILE_REPOSITORY.SHARE,
+ [{ realPath: shareFilesPath, pathPrefix: `${SPACE_REPOSITORY.SHARES}/${share.alias}`, isDir: share.isDir }]
+ ]
}
}
}
diff --git a/backend/src/applications/files/services/files-quota-manager.service.ts b/backend/src/applications/files/services/files-quota-manager.service.ts
index 13c2b19f..0a79957d 100644
--- a/backend/src/applications/files/services/files-quota-manager.service.ts
+++ b/backend/src/applications/files/services/files-quota-manager.service.ts
@@ -11,14 +11,9 @@ import { dirSize, isPathExists } from '../utils/files'
import { SpaceModel } from '../../spaces/models/space.model'
import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpaceEnv } from '../../spaces/models/space-env.model'
-import {
- CACHE_QUOTA_MUST_UPDATE,
- CACHE_QUOTA_SHARE_PREFIX,
- CACHE_QUOTA_SPACE_PREFIX,
- CACHE_QUOTA_TTL,
- CACHE_QUOTA_USER_PREFIX
-} from '../constants/cache'
-import { quotaCacheKeyFromSpace } from '../utils/quota'
+import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_TTL } from '../constants/cache'
+import { genQuotaCacheKey, quotaCacheKeyFromSpace } from '../utils/quota'
+import { FILE_REPOSITORY } from '../constants/operations'
@Injectable()
export class FilesQuotaManager {
@@ -62,16 +57,16 @@ export class FilesQuotaManager {
}
async updateQuotaEntries() {
- for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_MUST_UPDATE}-quota-*`)) {
+ for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_EVENT_UPDATE_PREFIX}-*`)) {
try {
const keySegments = k.split('-')
const [repository, idPart] = keySegments.slice(-2)
const id = Number.parseInt(idPart ?? '', 10)
- if (repository === 'user') {
+ if (repository === FILE_REPOSITORY.USER) {
await this.updatePersonalSpacesQuota(id)
- } else if (repository === 'space') {
+ } else if (repository === FILE_REPOSITORY.SPACE) {
await this.updateSpacesQuota(id)
- } else if (repository === 'share') {
+ } else if (repository === FILE_REPOSITORY.SHARE) {
await this.updateSharesExternalPathQuota(id)
} else {
this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unknown type: ${repository}` })
@@ -100,7 +95,7 @@ export class FilesQuotaManager {
}
const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }
this.spacesQueries.cache
- .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)
+ .set(genQuotaCacheKey(user.id, FILE_REPOSITORY.USER), spaceQuota, CACHE_QUOTA_TTL)
.catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))
if (user.storageUsage !== spaceQuota.storageUsage) {
this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(
@@ -137,7 +132,7 @@ export class FilesQuotaManager {
}
const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }
this.spacesQueries.cache
- .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)
+ .set(genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE), spaceQuota, CACHE_QUOTA_TTL)
.catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))
if (space.storageUsage !== spaceQuota.storageUsage) {
this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(
@@ -169,7 +164,7 @@ export class FilesQuotaManager {
}
const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }
this.sharesQueries.cache
- .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)
+ .set(genQuotaCacheKey(share.id, FILE_REPOSITORY.SHARE), shareQuota, CACHE_QUOTA_TTL)
.catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))
if (share.storageUsage !== shareQuota.storageUsage) {
this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(
diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.ts
index 8a7e4865..765029d5 100644
--- a/backend/src/applications/files/services/files-search-manager.service.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.ts
@@ -6,13 +6,13 @@ import { configuration } from '../../../configuration/config.environment'
import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { UserModel } from '../../users/models/user.model'
-import { shareIndexPrefix, spaceIndexPrefix, userIndexPrefix } from '../constants/indexing'
import { SearchFilesDto } from '../dto/file-operations.dto'
import { FilesIndexer } from '../models/files-indexer'
import { FileContent } from '../schemas/file-content.interface'
import { dirName, fileName, getMimeType } from '../utils/files'
import { genRegexPositiveAndNegativeTerms } from '../utils/files-search'
import { FilesParser } from './files-parser.service'
+import { FILE_REPOSITORY } from '../constants/operations'
@Injectable()
export class FilesSearchManager {
@@ -39,9 +39,9 @@ export class FilesSearchManager {
private async searchFullText(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {
const indexNames = await this.filesIndexer.existingIndexes([
- `${userIndexPrefix}${userId}`,
- ...spaceIds.map((id) => `${spaceIndexPrefix}${id}`),
- ...shareIds.map((id) => `${shareIndexPrefix}${id}`)
+ `${FILE_REPOSITORY.USER}_${userId}`,
+ ...spaceIds.map((id) => `${FILE_REPOSITORY.SPACE}_${id}`),
+ ...shareIds.map((id) => `${FILE_REPOSITORY.SHARE}_${id}`)
])
if (indexNames.length === 0) {
return []
diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.ts
index 1f085d3c..3f8093e9 100644
--- a/backend/src/applications/files/utils/quota.ts
+++ b/backend/src/applications/files/utils/quota.ts
@@ -1,23 +1,28 @@
import { SpaceEnv } from '../../spaces/models/space-env.model'
-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'
+import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_PREFIX } from '../constants/cache'
+import { FILE_REPOSITORY } from '../constants/operations'
-export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv) {
+export function genQuotaCacheKey(id: number, type: FILE_REPOSITORY, isEventUpdate = false): string {
+ return `${isEventUpdate ? CACHE_QUOTA_EVENT_UPDATE_PREFIX : CACHE_QUOTA_PREFIX}-${type}-${id}`
+}
+
+export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string {
if (space.inPersonalSpace) {
// Personal user space
- return `${CACHE_QUOTA_USER_PREFIX}-${userId}`
+ return genQuotaCacheKey(userId, FILE_REPOSITORY.USER, isEventUpdate)
} else if (space.root?.externalPath) {
// External paths used as shares or as space roots share the same quota as their origin
if (space.inSharesRepository) {
- return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`
+ return genQuotaCacheKey(space.root?.externalParentShareId || space.id, FILE_REPOSITORY.SHARE, isEventUpdate)
}
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
- } else if (space.root.file?.path && space.root.owner?.login) {
+ return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
+ } else if (space.root?.file?.path && space.root.owner?.login) {
// Space root is linked to a user file
- return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`
- } else if (space.root.file?.space?.id) {
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`
+ return genQuotaCacheKey(space.root.owner.id, FILE_REPOSITORY.USER, isEventUpdate)
+ } else if (space.root?.file?.space?.id) {
+ return genQuotaCacheKey(space.root.file.space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
} else if (space.id) {
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
+ return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
} else {
return null
}
diff --git a/backend/src/applications/spaces/services/spaces-manager.service.ts b/backend/src/applications/spaces/services/spaces-manager.service.ts
index 9e71059f..3b9b6371 100644
--- a/backend/src/applications/spaces/services/spaces-manager.service.ts
+++ b/backend/src/applications/spaces/services/spaces-manager.service.ts
@@ -46,7 +46,8 @@ import { spaces } from '../schemas/spaces.schema'
import { haveSpacePermission } from '../utils/permissions'
import { SpacesQueries } from './spaces-queries.service'
import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'
-import { CACHE_QUOTA_SPACE_PREFIX } from '../../files/constants/cache'
+import { genQuotaCacheKey } from '../../files/utils/quota'
+import { FILE_REPOSITORY } from '../../files/constants/operations'
@Injectable()
export class SpacesManager {
@@ -666,7 +667,7 @@ export class SpacesManager {
await this.spacesQueries.deleteSpace(space.id, deleteNow)
if (deleteNow) {
this.spacesQueries.cache
- .del(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`)
+ .del(genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE))
.catch((e: Error) => this.logger.error({ tag: this.deleteOrDisableSpace.name, msg: `${e}` }))
await this.deleteSpaceLocation(space.alias)
}