Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): move quota functions to the new quota manager
johaven53e00272df04
diff --git a/backend/src/applications/comments/comments.controller.spec.ts b/backend/src/applications/comments/comments.controller.spec.tsindex fc9c98d3..37b4cd54 100644--- a/backend/src/applications/comments/comments.controller.spec.ts+++ b/backend/src/applications/comments/comments.controller.spec.ts@@ -13,6 +13,7 @@ import { UsersQueries } from '../users/services/users-queries.service'import { CommentsController } from './comments.controller'import { CommentsManager } from './services/comments-manager.service'import { CommentsQueries } from './services/comments-queries.service'+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'describe(CommentsController.name, () => {let commentsController: CommentsController@@ -41,6 +42,7 @@ describe(CommentsController.name, () => {{ provide: Cache, useValue: {} },ContextManager,{ provide: CommentsManager, useValue: commentsManagerMock },+ { provide: FilesQuotaManager, useValue: {} },CommentsQueries,SpacesManager,SpacesQueries,diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.tsindex cd06483a..d5e2c0d1 100644--- a/backend/src/applications/files/constants/cache.ts+++ b/backend/src/applications/files/constants/cache.ts@@ -5,4 +5,9 @@ export const CACHE_TASK_TTL = 86400 as const // one dayexport const CACHE_LOCK_PREFIX = 'flock' as constexport const CACHE_LOCK_DEFAULT_TTL = 28800 as const // 8 hours in secondsexport const CACHE_LOCK_FILE_TTL = 3600 as const-// cache only office = `office|${fileId}` => docKey+// cache quota key = `(quota-user|quota-space)-${id}` => number+export const CACHE_QUOTA_USER_PREFIX = 'quota-user'+export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'+export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'+export const CACHE_QUOTA_MUST_UPDATE = 'must-update'+export const CACHE_QUOTA_TTL = 86400 // 1 daydiff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex e10f9b7a..d1b8200a 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -16,6 +16,8 @@ import { FilesRecents } from './services/files-recents.service'import { FilesScheduler } from './services/files-scheduler.service'import { FilesSearchManager } from './services/files-search-manager.service'import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesEventManager } from './services/files-event-manager.service'+import { FilesQuotaManager } from './services/files-quota-manager.service'@Module({imports: [@@ -35,7 +37,8 @@ import { FilesTasksManager } from './services/files-tasks-manager.service'FilesContentManager,{ provide: FilesIndexer, useClass: FilesIndexerMySQL },FilesSearchManager+ FilesQuotaManager],- exports: [FilesManager, FilesQueries, FilesLockManager, FilesMethods, FilesRecents]+ exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]})export class FilesModule {}diff --git a/backend/src/applications/files/modules/only-office/only-office.constants.ts b/backend/src/applications/files/modules/only-office/only-office.constants.tsindex 0b970a82..0bde9848 100644--- a/backend/src/applications/files/modules/only-office/only-office.constants.ts+++ b/backend/src/applications/files/modules/only-office/only-office.constants.ts@@ -2,8 +2,8 @@ export const ONLY_OFFICE_INTERNAL_URI = '/onlyoffice' // used by nginx as a proxexport const ONLY_OFFICE_CONTEXT = 'OnlyOfficeEnvironment' as constexport const ONLY_OFFICE_TOKEN_QUERY_PARAM_NAME = 'token' as constexport const ONLY_OFFICE_APP_LOCK = 'OnlyOffice' as const+// cache only office = `office|${fileId}` => docKeyexport const ONLY_OFFICE_CACHE_KEY = 'foffice' as const-export const ONLY_OFFICE_EXTENSIONS = new Map<string, 'word' | 'cell' | 'slide' | 'pdf' | 'diagram'>([// ─────────────// WORDdiff --git a/backend/src/applications/files/services/files-methods.service.spec.ts b/backend/src/applications/files/services/files-methods.service.spec.tsindex 5c32debb..3aae7a7d 100644--- a/backend/src/applications/files/services/files-methods.service.spec.ts+++ b/backend/src/applications/files/services/files-methods.service.spec.ts@@ -18,10 +18,12 @@ import { tarExtension } from '../constants/compress'import { CompressFileDto, CopyMoveFileDto } from '../dto/file-operations.dto'import { FilesManager } from './files-manager.service'import { FilesMethods } from './files-methods.service'+import { FilesQuotaManager } from './files-quota-manager.service'describe(FilesMethods.name, () => {let filesMethods: FilesMethodslet spacesManager: SpacesManager+ let filesQuotaManager: FilesQuotaManagerlet userTest: UserModelconst spaceEnv = {id: 1,@@ -53,6 +55,7 @@ describe(FilesMethods.name, () => {},{ provide: UsersQueries, useValue: {} },{ provide: SharesManager, useValue: {} },+ { provide: FilesQuotaManager, useValue: {} },{provide: SpacesQueries,useValue: {@@ -71,9 +74,10 @@ describe(FilesMethods.name, () => {module.useLogger(['fatal'])filesMethods = module.get<FilesMethods>(FilesMethods)spacesManager = module.get<SpacesManager>(SpacesManager)+ filesQuotaManager = module.get<FilesQuotaManager>(FilesQuotaManager)userTest = new UserModel(generateUserTest())// mock- spacesManager.updateSpacesQuota = jest.fn().mockReturnValue(undefined)+ filesQuotaManager.updateSpacesQuota = jest.fn().mockReturnValue(undefined)})it('should be defined', () => {diff --git a/backend/src/applications/files/services/files-quota-manager.service.spec.ts b/backend/src/applications/files/services/files-quota-manager.service.spec.tsnew file mode 100644index 00000000..9850130f--- /dev/null+++ b/backend/src/applications/files/services/files-quota-manager.service.spec.ts@@ -0,0 +1,26 @@+import { Test, TestingModule } from '@nestjs/testing'+import { FilesQuotaManager } from './files-quota-manager.service'+import { SpacesQueries } from '../../spaces/services/spaces-queries.service'+import { UsersQueries } from '../../users/services/users-queries.service'+import { SharesQueries } from '../../shares/services/shares-queries.service'++describe(FilesQuotaManager.name, () => {+ let service: FilesQuotaManager++ beforeAll(async () => {+ const module: TestingModule = await Test.createTestingModule({+ providers: [+ FilesQuotaManager,+ { provide: SpacesQueries, useValue: {} },+ { provide: UsersQueries, useValue: {} },+ { provide: SharesQueries, useValue: {} }+ ]+ }).compile()++ service = module.get<FilesQuotaManager>(FilesQuotaManager)+ })++ it('should be defined', () => {+ expect(service).toBeDefined()+ })+})diff --git a/backend/src/applications/files/services/files-quota-manager.service.ts b/backend/src/applications/files/services/files-quota-manager.service.tsnew file mode 100644index 00000000..13c2b19f--- /dev/null+++ b/backend/src/applications/files/services/files-quota-manager.service.ts@@ -0,0 +1,189 @@+import { Injectable, Logger } from '@nestjs/common'+import { SpacesQueries } from '../../spaces/services/spaces-queries.service'+import { UsersQueries } from '../../users/services/users-queries.service'+import type { StorageQuota } from '../../../common/interfaces'+import type { User } from '../../users/schemas/user.interface'+import { eq, lte } from 'drizzle-orm'+import { users } from '../../users/schemas/users.schema'+import { USER_ROLE } from '../../users/constants/user'+import { UserModel } from '../../users/models/user.model'+import { dirSize, isPathExists } from '../utils/files'+import { SpaceModel } from '../../spaces/models/space.model'+import { SharesQueries } from '../../shares/services/shares-queries.service'+import { SpaceEnv } from '../../spaces/models/space-env.model'+import {+ CACHE_QUOTA_MUST_UPDATE,+ CACHE_QUOTA_SHARE_PREFIX,+ CACHE_QUOTA_SPACE_PREFIX,+ CACHE_QUOTA_TTL,+ CACHE_QUOTA_USER_PREFIX+} from '../constants/cache'+import { quotaCacheKeyFromSpace } from '../utils/quota'++@Injectable()+export class FilesQuotaManager {+ private readonly logger = new Logger(FilesQuotaManager.name)++ constructor(+ private readonly spacesQueries: SpacesQueries,+ private readonly usersQueries: UsersQueries,+ private readonly sharesQueries: SharesQueries+ ) {}++ async setQuotaExceeded(user: UserModel, space: SpaceEnv) {+ /* extract quota from spaces|shares|roots */+ if (space.inSharesList) {+ return+ }+ const cacheQuotaKey = quotaCacheKeyFromSpace(user.id, space)+ if (!cacheQuotaKey) {+ this.logger.warn({ tag: this.setQuotaExceeded.name, msg: `quota was ignored for space : *${space.alias}* (${space.id})` })+ return+ }+ let quota: StorageQuota = await this.spacesQueries.cache.get(cacheQuotaKey)+ if (!quota) {+ // the quota scheduler has not started yet or the cache has been cleared+ if (space.inPersonalSpace) {+ quota = await this.updatePersonalSpacesQuota(user.id)+ } else if (space.inSharesRepository) {+ // Shares with external paths+ quota = await this.updateSharesExternalPathQuota(space.id)+ } else {+ quota = await this.updateSpacesQuota(space.id)+ }+ }+ if (quota) {+ space.storageUsage = quota.storageUsage+ space.storageQuota = quota.storageQuota+ space.quotaIsExceeded = quota.storageQuota !== null && quota.storageUsage >= quota.storageQuota+ } else {+ this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota not found for space : *${space.alias}* (${space.id})` })+ }+ }++ async updateQuotaEntries() {+ for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_MUST_UPDATE}-quota-*`)) {+ try {+ const keySegments = k.split('-')+ const [repository, idPart] = keySegments.slice(-2)+ const id = Number.parseInt(idPart ?? '', 10)+ if (repository === 'user') {+ await this.updatePersonalSpacesQuota(id)+ } else if (repository === 'space') {+ await this.updateSpacesQuota(id)+ } else if (repository === 'share') {+ await this.updateSharesExternalPathQuota(id)+ } else {+ this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unknown type: ${repository}` })+ }+ } catch (e) {+ this.logger.error({ tag: this.updateQuotaEntries.name, msg: `${e}` })+ } finally {+ this.spacesQueries.cache.del(k).catch((e) => this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))+ }+ }+ }++ async updatePersonalSpacesQuota(): Promise<void>+ async updatePersonalSpacesQuota(userId: number): Promise<StorageQuota>+ async updatePersonalSpacesQuota(userId?: number): Promise<void | StorageQuota> {+ const props: (keyof User)[] = ['id', 'login', 'storageUsage', 'storageQuota']+ for (const user of await this.usersQueries.selectUsers(props, [lte(users.role, USER_ROLE.USER), ...(userId ? [eq(users.id, userId)] : [])])) {+ const userPath = UserModel.getHomePath(user.login)+ if (!(await isPathExists(userPath))) {+ this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `*${user.login}* home path does not exist` })+ continue+ }+ const [size, errors] = await dirSize(userPath)+ for (const [path, error] of Object.entries(errors)) {+ this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `unable to get size for *${user.login}* on ${path} : ${error}` })+ }+ const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }+ this.spacesQueries.cache+ .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)+ .catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))+ if (user.storageUsage !== spaceQuota.storageUsage) {+ this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(+ (updated: boolean) =>+ updated &&+ this.logger.log({+ tag: this.updatePersonalSpacesQuota.name,+ msg: `user *${user.login}* (${user.id}) - storage usage updated: ${spaceQuota.storageUsage}`+ })+ )+ }+ if (userId) {+ return spaceQuota+ }+ }+ }++ async updateSpacesQuota(): Promise<void>+ async updateSpacesQuota(spaceId: number): Promise<StorageQuota>+ async updateSpacesQuota(spaceId?: number): Promise<void | StorageQuota> {+ for (const space of await this.spacesQueries.spacesQuotaPaths(spaceId)) {+ const spacePath = SpaceModel.getHomePath(space.alias)+ if (!(await isPathExists(spacePath))) {+ this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `*${space.alias}* home path does not exist` })+ continue+ }+ let size = 0+ for (const rPath of [spacePath, ...space.externalPaths.filter(Boolean)]) {+ const [rPathSize, errors] = await dirSize(rPath)+ size += rPathSize+ for (const [path, error] of Object.entries(errors)) {+ this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `unable to get size for *${space.alias}* on ${path} : ${error}` })+ }+ }+ const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }+ this.spacesQueries.cache+ .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)+ .catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))+ if (space.storageUsage !== spaceQuota.storageUsage) {+ this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(+ (updated: boolean) =>+ updated &&+ this.logger.log({+ tag: this.updateSpacesQuota.name,+ msg: `space *${space.alias}* (${space.id}) - storage usage updated : ${spaceQuota.storageUsage}`+ })+ )+ }+ if (spaceId) {+ return spaceQuota+ }+ }+ }++ async updateSharesExternalPathQuota(): Promise<void>+ async updateSharesExternalPathQuota(shareId: number): Promise<StorageQuota>+ async updateSharesExternalPathQuota(shareId?: number): Promise<void | StorageQuota> {+ for (const share of await this.sharesQueries.sharesQuotaExternalPaths(shareId)) {+ if (!(await isPathExists(share.externalPath))) {+ this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `*${share.alias}* home path does not exist` })+ continue+ }+ const [size, errors] = await dirSize(share.externalPath)+ for (const [path, error] of Object.entries(errors)) {+ this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `unable to get size for *${share.alias}* on ${path} : ${error}` })+ }+ const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }+ this.sharesQueries.cache+ .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)+ .catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))+ if (share.storageUsage !== shareQuota.storageUsage) {+ this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(+ (updated: boolean) =>+ updated &&+ this.logger.log({+ tag: this.updateSharesExternalPathQuota.name,+ msg: `share *${share.alias}* (${share.id}) - storage usage updated : ${shareQuota.storageUsage}`+ })+ )+ }+ if (shareId) {+ return shareQuota+ }+ }+ }+}diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 80f46428..13f65e7f 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -1,5 +1,5 @@import { Inject, Injectable, Logger } from '@nestjs/common'-import { Cron, CronExpression, Timeout } from '@nestjs/schedule'+import { Cron, CronExpression, Interval, Timeout } from '@nestjs/schedule'import { isNotNull, sql } from 'drizzle-orm'import { unionAll } from 'drizzle-orm/mysql-core'import fs from 'node:fs/promises'@@ -19,16 +19,20 @@ import { files } from '../schemas/files.schema'import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'import { FilesContentManager } from './files-content-manager.service'import { FilesTasksManager } from './files-tasks-manager.service'+import { FilesQuotaManager } from './files-quota-manager.service'@Injectable()export class FilesScheduler {private readonly logger = new Logger(FilesScheduler.name)private isIndexContentFilesRunning = false+ private isQuotaUpdateIsRunning = false+ private isQuotaUpdateEntriesIsRunning = falseconstructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,private readonly cache: Cache,- private readonly filesContentManager: FilesContentManager+ private readonly filesContentManager: FilesContentManager,+ private readonly filesQuotaManager: FilesQuotaManager) {}@Timeout(10_000)@@ -36,6 +40,7 @@ export class FilesScheduler {try {await this.cleanupInterruptedTasks()await this.clearRecentFiles()+ await this.updateQuotas()} catch (e) {this.logger.error(e)}@@ -50,6 +55,19 @@ export class FilesScheduler {}}+ @Interval(60_000)+ async updateQuotaEntries() {+ if (this.isQuotaUpdateIsRunning || this.isQuotaUpdateEntriesIsRunning) return+ this.isQuotaUpdateEntriesIsRunning = true+ try {+ await this.filesQuotaManager.updateQuotaEntries()+ } catch (e) {+ this.logger.error({ tag: this.updateQuotas.name, msg: `${e}` })+ } finally {+ this.isQuotaUpdateEntriesIsRunning = false+ }+ }+async cleanupInterruptedTasks(): Promise<void> {this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `START` })try {@@ -186,4 +204,32 @@ export class FilesScheduler {}this.logger.log({ tag: this.deleteOrphanFiles.name, msg: `END` })}++ @Cron(CronExpression.EVERY_HOUR)+ async updateQuotas() {+ if (this.isQuotaUpdateIsRunning) return+ this.isQuotaUpdateIsRunning = true+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - START' })+ try {+ await this.filesQuotaManager.updatePersonalSpacesQuota()+ } catch (e) {+ this.logger.error({ tag: this.updateQuotas.name, msg: `Personals - ${e}` })+ }+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - END' })+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - START' })+ try {+ await this.filesQuotaManager.updateSpacesQuota()+ } catch (e) {+ this.logger.error({ tag: this.updateQuotas.name, msg: `Spaces - ${e}` })+ }+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - END' })+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - START' })+ try {+ await this.filesQuotaManager.updateSharesExternalPathQuota()+ } catch (e) {+ this.logger.error({ tag: this.updateQuotas.name, msg: `Share External Paths - ${e}` })+ }+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })+ this.isQuotaUpdateIsRunning = false+ }}diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.tsnew file mode 100644index 00000000..1f085d3c--- /dev/null+++ b/backend/src/applications/files/utils/quota.ts@@ -0,0 +1,24 @@+import { SpaceEnv } from '../../spaces/models/space-env.model'+import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'++export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv) {+ if (space.inPersonalSpace) {+ // Personal user space+ return `${CACHE_QUOTA_USER_PREFIX}-${userId}`+ } else if (space.root?.externalPath) {+ // External paths used as shares or as space roots share the same quota as their origin+ if (space.inSharesRepository) {+ return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`+ }+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`+ } else if (space.root.file?.path && space.root.owner?.login) {+ // Space root is linked to a user file+ return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`+ } else if (space.root.file?.space?.id) {+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`+ } else if (space.id) {+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`+ } else {+ return null+ }+}diff --git a/backend/src/applications/shares/services/shares-manager.service.ts b/backend/src/applications/shares/services/shares-manager.service.tsindex 13bb7823..533713ab 100644--- a/backend/src/applications/shares/services/shares-manager.service.ts+++ b/backend/src/applications/shares/services/shares-manager.service.ts@@ -9,12 +9,12 @@ import {generateShortUUID,hashPassword} from '../../../common/functions'-import type { Entries, StorageQuota } from '../../../common/interfaces'+import type { Entries } from '../../../common/interfaces'import { intersectPermissions } from '../../../common/shared'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'import type { FileProps } from '../../files/interfaces/file-props.interface'import { FileError } from '../../files/models/file-error'-import { checkExternalPath, dirSize, getProps, isPathExists } from '../../files/utils/files'+import { checkExternalPath, getProps, isPathExists } from '../../files/utils/files'import { LINK_TYPE } from '../../links/constants/links'import type { CreateOrUpdateLinkDto } from '../../links/dto/create-or-update-link.dto'import type { LinkGuest } from '../../links/interfaces/link-guest.interface'@@ -24,7 +24,6 @@ import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/coimport type { NotificationContent, NotificationOptions } from '../../notifications/interfaces/notification-properties.interface'import type { UserMailNotification } from '../../notifications/interfaces/user-mail-notification.interface'import { NotificationsManager } from '../../notifications/services/notifications-manager.service'-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_TTL } from '../../spaces/constants/cache'import { SPACE_OPERATION, SPACE_PERMS_SEP, SPACE_REPOSITORY, SPACE_ROLE } from '../../spaces/constants/spaces'import type { SpaceMemberDto } from '../../spaces/dto/create-or-update-space.dto'import { SpaceEnv } from '../../spaces/models/space-env.model'@@ -736,38 +735,6 @@ export class SharesManager {await this.deleteGuestLinks(members)}- async updateSharesExternalPathQuota(): Promise<void>- async updateSharesExternalPathQuota(shareId: number): Promise<StorageQuota>- async updateSharesExternalPathQuota(shareId?: number): Promise<void | StorageQuota> {- for (const share of await this.sharesQueries.sharesQuotaExternalPaths(shareId)) {- if (!(await isPathExists(share.externalPath))) {- this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `*${share.alias}* home path does not exist` })- continue- }- const [size, errors] = await dirSize(share.externalPath)- for (const [path, error] of Object.entries(errors)) {- this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `unable to get size for *${share.alias}* on ${path} : ${error}` })- }- const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }- this.sharesQueries.cache- .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)- .catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))- if (share.storageUsage !== shareQuota.storageUsage) {- this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(- (updated: boolean) =>- updated &&- this.logger.log({- tag: this.updateSharesExternalPathQuota.name,- msg: `share *${share.alias}* (${share.id}) - storage usage updated : ${shareQuota.storageUsage}`- })- )- }- if (shareId) {- return shareQuota- }- }- }-private async updateMembers(user: UserModel, share: Partial<Share>, oldMembers: ShareMemberDto[], currentMembers: ShareMemberDto[]) {if (oldMembers.length === 0 && currentMembers.length === 0) {returndiff --git a/backend/src/applications/spaces/constants/cache.ts b/backend/src/applications/spaces/constants/cache.tsdeleted file mode 100644index bf57bff2..00000000--- a/backend/src/applications/spaces/constants/cache.ts+++ /dev/null@@ -1,5 +0,0 @@-// cache quota key = `(quota-user|quota-space)-${id}` => number-export const CACHE_QUOTA_USER_PREFIX = 'quota-user'-export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'-export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'-export const CACHE_QUOTA_TTL = 86400 // 1 daydiff --git a/backend/src/applications/spaces/guards/space.guard.spec.ts b/backend/src/applications/spaces/guards/space.guard.spec.tsindex ec39d9e7..7a50c309 100644--- a/backend/src/applications/spaces/guards/space.guard.spec.ts+++ b/backend/src/applications/spaces/guards/space.guard.spec.ts@@ -25,6 +25,7 @@ import { SpaceEnv } from '../models/space-env.model'import { SpacesManager } from '../services/spaces-manager.service'import { SpacesQueries } from '../services/spaces-queries.service'import { SpaceGuard } from './space.guard'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'describe(SpaceGuard.name, () => {let spacesGuard: SpaceGuard@@ -49,6 +50,10 @@ describe(SpaceGuard.name, () => {provide: NotificationsManager,useValue: {}},+ {+ provide: FilesQuotaManager,+ useValue: { setQuotaExceeded: () => jest.fn() }+ },SpaceGuard,SpacesManager,SpacesQueries,@@ -64,7 +69,6 @@ describe(SpaceGuard.name, () => {spacesQueries = module.get<SpacesQueries>(SpacesQueries)spacesGuard = module.get<SpaceGuard>(SpaceGuard)// mocks- spacesManager['setQuotaExceeded'] = jest.fn()userTest = new UserModel(generateUserTest())})diff --git a/backend/src/applications/spaces/services/spaces-browser.service.spec.ts b/backend/src/applications/spaces/services/spaces-browser.service.spec.tsindex 726fdb32..4017f60a 100644--- a/backend/src/applications/spaces/services/spaces-browser.service.spec.ts+++ b/backend/src/applications/spaces/services/spaces-browser.service.spec.ts@@ -15,6 +15,7 @@ import { UsersQueries } from '../../users/services/users-queries.service'import { SpacesBrowser } from './spaces-browser.service'import { SpacesManager } from './spaces-manager.service'import { SpacesQueries } from './spaces-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'describe(SpacesBrowser.name, () => {let spacesBrowserService: SpacesBrowser@@ -33,6 +34,7 @@ describe(SpacesBrowser.name, () => {provide: NotificationsManager,useValue: {}},+ { provide: FilesQuotaManager, useValue: {} },SpacesManager,SpacesBrowser,SpacesQueries,diff --git a/backend/src/applications/spaces/services/spaces-manager.service.spec.ts b/backend/src/applications/spaces/services/spaces-manager.service.spec.tsindex ab5c5598..a31a9f1c 100644--- a/backend/src/applications/spaces/services/spaces-manager.service.spec.ts+++ b/backend/src/applications/spaces/services/spaces-manager.service.spec.ts@@ -29,6 +29,7 @@ import { SpaceModel } from '../models/space.model'import { IsRealPathIsDirAndExists } from '../utils/paths'import { SpacesManager } from './spaces-manager.service'import { SpacesQueries } from './spaces-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'describe(SpacesManager.name, () => {let filesConfig: FilesConfig@@ -52,6 +53,10 @@ describe(SpacesManager.name, () => {provide: NotificationsManager,useValue: {}},+ {+ provide: FilesQuotaManager,+ useValue: { setQuotaExceeded: () => jest.fn() }+ },SpacesManager,SpacesQueries,UsersQueries,@@ -66,9 +71,7 @@ describe(SpacesManager.name, () => {filesConfig = module.get<ConfigService>(ConfigService).get('applications.files')spacesManager = module.get<SpacesManager>(SpacesManager)spacesQueries = module.get<SpacesQueries>(SpacesQueries)- spacesManager['setQuotaExceeded'] = jest.fn()userTest = new UserModel(generateUserTest())- // todo: validate shares, permissions})afterAll(async () => {diff --git a/backend/src/applications/spaces/services/spaces-manager.service.ts b/backend/src/applications/spaces/services/spaces-manager.service.tsindex a8e59b81..9e71059f 100644--- a/backend/src/applications/spaces/services/spaces-manager.service.ts+++ b/backend/src/applications/spaces/services/spaces-manager.service.ts@@ -1,15 +1,15 @@import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'-import { eq, isNotNull, lte } from 'drizzle-orm'+import { eq, isNotNull } from 'drizzle-orm'import fs from 'node:fs/promises'import path from 'node:path'import { ACTION } from '../../../common/constants'import { convertDiffUpdate, diffCollection, differencePermissions } from '../../../common/functions'-import type { Entries, StorageQuota } from '../../../common/interfaces'+import type { Entries } from '../../../common/interfaces'import { createSlug, regExpNumberSuffix } from '../../../common/shared'import { configuration } from '../../../configuration/config.environment'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'import { FileError } from '../../files/models/file-error'-import { dirListFileNames, dirSize, getProps, isPathExists, moveFiles, removeFiles } from '../../files/utils/files'+import { dirListFileNames, getProps, isPathExists, moveFiles, removeFiles } from '../../files/utils/files'import { LINK_TYPE } from '../../links/constants/links'import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/constants/notifications'import { NotificationContent } from '../../notifications/interfaces/notification-properties.interface'@@ -20,12 +20,8 @@ import type { ShareProps } from '../../shares/interfaces/share-props.interface'import type { ShareChild } from '../../shares/models/share-child.model'import { SharesManager } from '../../shares/services/shares-manager.service'import { MEMBER_TYPE } from '../../users/constants/member'-import { USER_ROLE } from '../../users/constants/user'import { UserModel } from '../../users/models/user.model'-import type { User } from '../../users/schemas/user.interface'-import { users } from '../../users/schemas/users.schema'import { UsersQueries } from '../../users/services/users-queries.service'-import { CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_TTL, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'import {SPACE_ALIAS,SPACE_ALL_OPERATIONS,@@ -47,9 +43,10 @@ import { SpaceModel } from '../models/space.model'import type { SpaceRoot } from '../schemas/space-root.interface'import type { Space } from '../schemas/space.interface'import { spaces } from '../schemas/spaces.schema'-import { quotaKeyFromSpace } from '../utils/paths'import { haveSpacePermission } from '../utils/permissions'import { SpacesQueries } from './spaces-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'+import { CACHE_QUOTA_SPACE_PREFIX } from '../../files/constants/cache'@Injectable()export class SpacesManager {@@ -60,6 +57,7 @@ export class SpacesManager {private readonly spacesQueries: SpacesQueries,private readonly usersQueries: UsersQueries,private readonly sharesManager: SharesManager,+ private readonly filesQuotaManager: FilesQuotaManager,private readonly notificationsManager: NotificationsManager) {}@@ -138,7 +136,7 @@ export class SpacesManager {if (!space) return nulltry {space.setup(user, repository as SPACE_REPOSITORY, rootAlias, paths, urlSegments, skipEndpointProtection)- await this.setQuotaExceeded(user, space)+ await this.filesQuotaManager.setQuotaExceeded(user, space)return space} catch (e) {this.logger.warn({ tag: this.spaceEnv.name, msg: `*${space.alias}* : ${e}` })@@ -166,7 +164,7 @@ export class SpacesManager {return Promise.all((await this.spacesQueries.spaces(user.id, true)).map(async (s) => {const space = new SpaceEnv(s)- await this.setQuotaExceeded(user, space)+ await this.filesQuotaManager.setQuotaExceeded(user, space)return space}))@@ -363,80 +361,6 @@ export class SpacesManager {return null}- async updatePersonalSpacesQuota(): Promise<void>- async updatePersonalSpacesQuota(forUser: UserModel): Promise<StorageQuota>- async updatePersonalSpacesQuota(forUser?: UserModel): Promise<void | StorageQuota> {- const props: (keyof User)[] = ['id', 'login', 'storageUsage', 'storageQuota']- for (const user of await this.usersQueries.selectUsers(props, [- lte(users.role, USER_ROLE.USER),- ...(forUser ? [eq(users.id, forUser.id)] : [])- ])) {- const userPath = UserModel.getHomePath(user.login)- if (!(await isPathExists(userPath))) {- this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `*${user.login}* home path does not exist` })- continue- }- const [size, errors] = await dirSize(userPath)- for (const [path, error] of Object.entries(errors)) {- this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `unable to get size for *${user.login}* on ${path} : ${error}` })- }- const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }- this.spacesQueries.cache- .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)- .catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))- if (user.storageUsage !== spaceQuota.storageUsage) {- this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(- (updated: boolean) =>- updated &&- this.logger.log({- tag: this.updatePersonalSpacesQuota.name,- msg: `user *${user.login}* (${user.id}) - storage usage updated: ${spaceQuota.storageUsage}`- })- )- }- if (forUser) {- return spaceQuota- }- }- }-- async updateSpacesQuota(): Promise<void>- async updateSpacesQuota(spaceId: number): Promise<StorageQuota>- async updateSpacesQuota(spaceId?: number): Promise<void | StorageQuota> {- for (const space of await this.spacesQueries.spacesQuotaPaths(spaceId)) {- const spacePath = SpaceModel.getHomePath(space.alias)- if (!(await isPathExists(spacePath))) {- this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `*${space.alias}* home path does not exist` })- continue- }- let size = 0- for (const rPath of [spacePath, ...space.externalPaths.filter(Boolean)]) {- const [rPathSize, errors] = await dirSize(rPath)- size += rPathSize- for (const [path, error] of Object.entries(errors)) {- this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `unable to get size for *${space.alias}* on ${path} : ${error}` })- }- }- const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }- this.spacesQueries.cache- .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)- .catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))- if (space.storageUsage !== spaceQuota.storageUsage) {- this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(- (updated: boolean) =>- updated &&- this.logger.log({- tag: this.updateSpacesQuota.name,- msg: `space *${space.alias}* (${space.id}) - storage usage updated : ${spaceQuota.storageUsage}`- })- )- }- if (spaceId) {- return spaceQuota- }- }- }-async deleteExpiredSpaces() {/* Removes spaces that have been disabled for more than 30 days */const props: (keyof Space)[] = ['id', 'name', 'alias', 'disabledAt']@@ -829,37 +753,6 @@ export class SpacesManager {throw new HttpException('Not authorized', HttpStatus.FORBIDDEN)}- private async setQuotaExceeded(user: UserModel, space: SpaceEnv) {- /* extract quota from spaces|shares|roots */- if (space.inSharesList) {- return- }- const cacheQuotaKey = quotaKeyFromSpace(user.id, space)- if (!cacheQuotaKey) {- this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota was ignored for space : *${space.alias}* (${space.id})` })- return- }- let quota: StorageQuota = await this.spacesQueries.cache.get(cacheQuotaKey)- if (!quota) {- // the quota scheduler has not started yet or the cache has been cleared- if (space.inPersonalSpace) {- quota = await this.updatePersonalSpacesQuota(user)- } else if (space.inSharesRepository) {- // Shares with external paths- quota = await this.sharesManager.updateSharesExternalPathQuota(space.id)- } else {- quota = await this.updateSpacesQuota(space.id)- }- }- if (quota) {- space.storageUsage = quota.storageUsage- space.storageQuota = quota.storageQuota- space.quotaIsExceeded = quota.storageQuota !== null && quota.storageUsage >= quota.storageQuota- } else {- this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota not found for space : *${space.alias}* (${space.id})` })- }- }-private async onSpaceActionForMembers(space: Partial<SpaceProps>,action: ACTION,diff --git a/backend/src/applications/spaces/services/spaces-scheduler.service.ts b/backend/src/applications/spaces/services/spaces-scheduler.service.tsindex f4abaf37..81c97b2e 100644--- a/backend/src/applications/spaces/services/spaces-scheduler.service.ts+++ b/backend/src/applications/spaces/services/spaces-scheduler.service.ts@@ -1,46 +1,12 @@import { Injectable, Logger } from '@nestjs/common'-import { Cron, CronExpression, Timeout } from '@nestjs/schedule'-import { SharesManager } from '../../shares/services/shares-manager.service'+import { Cron, CronExpression } from '@nestjs/schedule'import { SpacesManager } from './spaces-manager.service'@Injectable()export class SpacesScheduler {private readonly logger = new Logger(SpacesScheduler.name)- constructor(- private readonly spacesManager: SpacesManager,- private readonly sharesManager: SharesManager- ) {}-- @Timeout(60_000)- async onStartup() {- await this.updateQuotas()- }-- @Cron(CronExpression.EVERY_HOUR)- async updateQuotas() {- this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - START' })- try {- await this.spacesManager.updatePersonalSpacesQuota()- } catch (e) {- this.logger.error({ tag: this.updateQuotas.name, msg: `Personals} - ${e}` })- }- this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - END' })- this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - START' })- try {- await this.spacesManager.updateSpacesQuota()- } catch (e) {- this.logger.error({ tag: this.updateQuotas.name, msg: `Spaces - ${e}` })- }- this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - END' })- this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - START' })- try {- await this.sharesManager.updateSharesExternalPathQuota()- } catch (e) {- this.logger.error({ tag: this.updateQuotas.name, msg: `Share External Paths - ${e}` })- }- this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })- }+ constructor(private readonly spacesManager: SpacesManager) {}@Cron(CronExpression.EVERY_DAY_AT_2AM)async deleteExpiredSpaces() {diff --git a/backend/src/applications/spaces/spaces.controller.spec.ts b/backend/src/applications/spaces/spaces.controller.spec.tsindex 89f88126..2253a271 100644--- a/backend/src/applications/spaces/spaces.controller.spec.ts+++ b/backend/src/applications/spaces/spaces.controller.spec.ts@@ -14,6 +14,7 @@ import { SpacesBrowser } from './services/spaces-browser.service'import { SpacesManager } from './services/spaces-manager.service'import { SpacesQueries } from './services/spaces-queries.service'import { SpacesController } from './spaces.controller'+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'describe(SpacesController.name, () => {let spacesController: SpacesController@@ -31,6 +32,10 @@ describe(SpacesController.name, () => {provide: Cache,useValue: {}},+ {+ provide: FilesQuotaManager,+ useValue: {}+ },ContextManager,SpacesManager,SpacesQueries,diff --git a/backend/src/applications/spaces/utils/paths.ts b/backend/src/applications/spaces/utils/paths.tsindex fb59c18a..e437bd83 100644--- a/backend/src/applications/spaces/utils/paths.ts+++ b/backend/src/applications/spaces/utils/paths.ts@@ -5,7 +5,6 @@ import { FileDBProps } from '../../files/interfaces/file-db-props.interface'import { FileProps } from '../../files/interfaces/file-props.interface'import { FileError } from '../../files/models/file-error'import { UserModel } from '../../users/models/user.model'-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'import { SPACE_REPOSITORY } from '../constants/spaces'import { SpaceEnv } from '../models/space-env.model'import { SpaceModel } from '../models/space.model'@@ -176,25 +175,3 @@ export function dbFileFromSpace(userId: number, space: SpaceEnv): FileDBProps {}return dbFile}--export function quotaKeyFromSpace(userId: number, space: SpaceEnv) {- if (space.inPersonalSpace) {- // Personal user space- return `${CACHE_QUOTA_USER_PREFIX}-${userId}`- } else if (space.root?.externalPath) {- // External paths used as shares or as space roots share the same quota as their origin- if (space.inSharesRepository) {- return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`- }- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`- } else if (space.root.file?.path && space.root.owner?.login) {- // Space root is linked on a user file- return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`- } else if (space.root.file?.space?.id) {- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`- } else if (space.id) {- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`- } else {- throw new FileError(HttpStatus.NOT_FOUND, 'Space root not found')- }-}