Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): move quota functions to the new quota manager

johaven53e00272df04
diff --git a/backend/src/applications/comments/comments.controller.spec.ts b/backend/src/applications/comments/comments.controller.spec.ts
index fc9c98d3..37b4cd54 100644
--- a/backend/src/applications/comments/comments.controller.spec.ts
+++ b/backend/src/applications/comments/comments.controller.spec.ts
@@ -13,6 +13,7 @@ import { UsersQueries } from '../users/services/users-queries.service'
import { CommentsController } from './comments.controller'
import { CommentsManager } from './services/comments-manager.service'
import { CommentsQueries } from './services/comments-queries.service'
+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'
describe(CommentsController.name, () => {
let commentsController: CommentsController
@@ -41,6 +42,7 @@ describe(CommentsController.name, () => {
{ provide: Cache, useValue: {} },
ContextManager,
{ provide: CommentsManager, useValue: commentsManagerMock },
+ { provide: FilesQuotaManager, useValue: {} },
CommentsQueries,
SpacesManager,
SpacesQueries,
diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.ts
index cd06483a..d5e2c0d1 100644
--- a/backend/src/applications/files/constants/cache.ts
+++ b/backend/src/applications/files/constants/cache.ts
@@ -5,4 +5,9 @@ export const CACHE_TASK_TTL = 86400 as const // one day
export const CACHE_LOCK_PREFIX = 'flock' as const
export const CACHE_LOCK_DEFAULT_TTL = 28800 as const // 8 hours in seconds
export const CACHE_LOCK_FILE_TTL = 3600 as const
-// cache only office = `office|${fileId}` => docKey
+// cache quota key = `(quota-user|quota-space)-${id}` => number
+export const CACHE_QUOTA_USER_PREFIX = 'quota-user'
+export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'
+export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'
+export const CACHE_QUOTA_MUST_UPDATE = 'must-update'
+export const CACHE_QUOTA_TTL = 86400 // 1 day
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index e10f9b7a..d1b8200a 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -16,6 +16,8 @@ import { FilesRecents } from './services/files-recents.service'
import { FilesScheduler } from './services/files-scheduler.service'
import { FilesSearchManager } from './services/files-search-manager.service'
import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesEventManager } from './services/files-event-manager.service'
+import { FilesQuotaManager } from './services/files-quota-manager.service'
@Module({
imports: [
@@ -35,7 +37,8 @@ import { FilesTasksManager } from './services/files-tasks-manager.service'
FilesContentManager,
{ provide: FilesIndexer, useClass: FilesIndexerMySQL },
FilesSearchManager
+ FilesQuotaManager
],
- exports: [FilesManager, FilesQueries, FilesLockManager, FilesMethods, FilesRecents]
+ exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]
})
export class FilesModule {}
diff --git a/backend/src/applications/files/modules/only-office/only-office.constants.ts b/backend/src/applications/files/modules/only-office/only-office.constants.ts
index 0b970a82..0bde9848 100644
--- a/backend/src/applications/files/modules/only-office/only-office.constants.ts
+++ b/backend/src/applications/files/modules/only-office/only-office.constants.ts
@@ -2,8 +2,8 @@ export const ONLY_OFFICE_INTERNAL_URI = '/onlyoffice' // used by nginx as a prox
export const ONLY_OFFICE_CONTEXT = 'OnlyOfficeEnvironment' as const
export const ONLY_OFFICE_TOKEN_QUERY_PARAM_NAME = 'token' as const
export const ONLY_OFFICE_APP_LOCK = 'OnlyOffice' as const
+// cache only office = `office|${fileId}` => docKey
export const ONLY_OFFICE_CACHE_KEY = 'foffice' as const
-
export const ONLY_OFFICE_EXTENSIONS = new Map<string, 'word' | 'cell' | 'slide' | 'pdf' | 'diagram'>([
// ─────────────
// WORD
diff --git a/backend/src/applications/files/services/files-methods.service.spec.ts b/backend/src/applications/files/services/files-methods.service.spec.ts
index 5c32debb..3aae7a7d 100644
--- a/backend/src/applications/files/services/files-methods.service.spec.ts
+++ b/backend/src/applications/files/services/files-methods.service.spec.ts
@@ -18,10 +18,12 @@ import { tarExtension } from '../constants/compress'
import { CompressFileDto, CopyMoveFileDto } from '../dto/file-operations.dto'
import { FilesManager } from './files-manager.service'
import { FilesMethods } from './files-methods.service'
+import { FilesQuotaManager } from './files-quota-manager.service'
describe(FilesMethods.name, () => {
let filesMethods: FilesMethods
let spacesManager: SpacesManager
+ let filesQuotaManager: FilesQuotaManager
let userTest: UserModel
const spaceEnv = {
id: 1,
@@ -53,6 +55,7 @@ describe(FilesMethods.name, () => {
},
{ provide: UsersQueries, useValue: {} },
{ provide: SharesManager, useValue: {} },
+ { provide: FilesQuotaManager, useValue: {} },
{
provide: SpacesQueries,
useValue: {
@@ -71,9 +74,10 @@ describe(FilesMethods.name, () => {
module.useLogger(['fatal'])
filesMethods = module.get<FilesMethods>(FilesMethods)
spacesManager = module.get<SpacesManager>(SpacesManager)
+ filesQuotaManager = module.get<FilesQuotaManager>(FilesQuotaManager)
userTest = new UserModel(generateUserTest())
// mock
- spacesManager.updateSpacesQuota = jest.fn().mockReturnValue(undefined)
+ filesQuotaManager.updateSpacesQuota = jest.fn().mockReturnValue(undefined)
})
it('should be defined', () => {
diff --git a/backend/src/applications/files/services/files-quota-manager.service.spec.ts b/backend/src/applications/files/services/files-quota-manager.service.spec.ts
new file mode 100644
index 00000000..9850130f
--- /dev/null
+++ b/backend/src/applications/files/services/files-quota-manager.service.spec.ts
@@ -0,0 +1,26 @@
+import { Test, TestingModule } from '@nestjs/testing'
+import { FilesQuotaManager } from './files-quota-manager.service'
+import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
+import { UsersQueries } from '../../users/services/users-queries.service'
+import { SharesQueries } from '../../shares/services/shares-queries.service'
+
+describe(FilesQuotaManager.name, () => {
+ let service: FilesQuotaManager
+
+ beforeAll(async () => {
+ const module: TestingModule = await Test.createTestingModule({
+ providers: [
+ FilesQuotaManager,
+ { provide: SpacesQueries, useValue: {} },
+ { provide: UsersQueries, useValue: {} },
+ { provide: SharesQueries, useValue: {} }
+ ]
+ }).compile()
+
+ service = module.get<FilesQuotaManager>(FilesQuotaManager)
+ })
+
+ it('should be defined', () => {
+ expect(service).toBeDefined()
+ })
+})
diff --git a/backend/src/applications/files/services/files-quota-manager.service.ts b/backend/src/applications/files/services/files-quota-manager.service.ts
new file mode 100644
index 00000000..13c2b19f
--- /dev/null
+++ b/backend/src/applications/files/services/files-quota-manager.service.ts
@@ -0,0 +1,189 @@
+import { Injectable, Logger } from '@nestjs/common'
+import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
+import { UsersQueries } from '../../users/services/users-queries.service'
+import type { StorageQuota } from '../../../common/interfaces'
+import type { User } from '../../users/schemas/user.interface'
+import { eq, lte } from 'drizzle-orm'
+import { users } from '../../users/schemas/users.schema'
+import { USER_ROLE } from '../../users/constants/user'
+import { UserModel } from '../../users/models/user.model'
+import { dirSize, isPathExists } from '../utils/files'
+import { SpaceModel } from '../../spaces/models/space.model'
+import { SharesQueries } from '../../shares/services/shares-queries.service'
+import { SpaceEnv } from '../../spaces/models/space-env.model'
+import {
+ CACHE_QUOTA_MUST_UPDATE,
+ CACHE_QUOTA_SHARE_PREFIX,
+ CACHE_QUOTA_SPACE_PREFIX,
+ CACHE_QUOTA_TTL,
+ CACHE_QUOTA_USER_PREFIX
+} from '../constants/cache'
+import { quotaCacheKeyFromSpace } from '../utils/quota'
+
+@Injectable()
+export class FilesQuotaManager {
+ private readonly logger = new Logger(FilesQuotaManager.name)
+
+ constructor(
+ private readonly spacesQueries: SpacesQueries,
+ private readonly usersQueries: UsersQueries,
+ private readonly sharesQueries: SharesQueries
+ ) {}
+
+ async setQuotaExceeded(user: UserModel, space: SpaceEnv) {
+ /* extract quota from spaces|shares|roots */
+ if (space.inSharesList) {
+ return
+ }
+ const cacheQuotaKey = quotaCacheKeyFromSpace(user.id, space)
+ if (!cacheQuotaKey) {
+ this.logger.warn({ tag: this.setQuotaExceeded.name, msg: `quota was ignored for space : *${space.alias}* (${space.id})` })
+ return
+ }
+ let quota: StorageQuota = await this.spacesQueries.cache.get(cacheQuotaKey)
+ if (!quota) {
+ // the quota scheduler has not started yet or the cache has been cleared
+ if (space.inPersonalSpace) {
+ quota = await this.updatePersonalSpacesQuota(user.id)
+ } else if (space.inSharesRepository) {
+ // Shares with external paths
+ quota = await this.updateSharesExternalPathQuota(space.id)
+ } else {
+ quota = await this.updateSpacesQuota(space.id)
+ }
+ }
+ if (quota) {
+ space.storageUsage = quota.storageUsage
+ space.storageQuota = quota.storageQuota
+ space.quotaIsExceeded = quota.storageQuota !== null && quota.storageUsage >= quota.storageQuota
+ } else {
+ this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota not found for space : *${space.alias}* (${space.id})` })
+ }
+ }
+
+ async updateQuotaEntries() {
+ for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_MUST_UPDATE}-quota-*`)) {
+ try {
+ const keySegments = k.split('-')
+ const [repository, idPart] = keySegments.slice(-2)
+ const id = Number.parseInt(idPart ?? '', 10)
+ if (repository === 'user') {
+ await this.updatePersonalSpacesQuota(id)
+ } else if (repository === 'space') {
+ await this.updateSpacesQuota(id)
+ } else if (repository === 'share') {
+ await this.updateSharesExternalPathQuota(id)
+ } else {
+ this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unknown type: ${repository}` })
+ }
+ } catch (e) {
+ this.logger.error({ tag: this.updateQuotaEntries.name, msg: `${e}` })
+ } finally {
+ this.spacesQueries.cache.del(k).catch((e) => this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))
+ }
+ }
+ }
+
+ async updatePersonalSpacesQuota(): Promise<void>
+ async updatePersonalSpacesQuota(userId: number): Promise<StorageQuota>
+ async updatePersonalSpacesQuota(userId?: number): Promise<void | StorageQuota> {
+ const props: (keyof User)[] = ['id', 'login', 'storageUsage', 'storageQuota']
+ for (const user of await this.usersQueries.selectUsers(props, [lte(users.role, USER_ROLE.USER), ...(userId ? [eq(users.id, userId)] : [])])) {
+ const userPath = UserModel.getHomePath(user.login)
+ if (!(await isPathExists(userPath))) {
+ this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `*${user.login}* home path does not exist` })
+ continue
+ }
+ const [size, errors] = await dirSize(userPath)
+ for (const [path, error] of Object.entries(errors)) {
+ this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `unable to get size for *${user.login}* on ${path} : ${error}` })
+ }
+ const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }
+ this.spacesQueries.cache
+ .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)
+ .catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))
+ if (user.storageUsage !== spaceQuota.storageUsage) {
+ this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(
+ (updated: boolean) =>
+ updated &&
+ this.logger.log({
+ tag: this.updatePersonalSpacesQuota.name,
+ msg: `user *${user.login}* (${user.id}) - storage usage updated: ${spaceQuota.storageUsage}`
+ })
+ )
+ }
+ if (userId) {
+ return spaceQuota
+ }
+ }
+ }
+
+ async updateSpacesQuota(): Promise<void>
+ async updateSpacesQuota(spaceId: number): Promise<StorageQuota>
+ async updateSpacesQuota(spaceId?: number): Promise<void | StorageQuota> {
+ for (const space of await this.spacesQueries.spacesQuotaPaths(spaceId)) {
+ const spacePath = SpaceModel.getHomePath(space.alias)
+ if (!(await isPathExists(spacePath))) {
+ this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `*${space.alias}* home path does not exist` })
+ continue
+ }
+ let size = 0
+ for (const rPath of [spacePath, ...space.externalPaths.filter(Boolean)]) {
+ const [rPathSize, errors] = await dirSize(rPath)
+ size += rPathSize
+ for (const [path, error] of Object.entries(errors)) {
+ this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `unable to get size for *${space.alias}* on ${path} : ${error}` })
+ }
+ }
+ const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }
+ this.spacesQueries.cache
+ .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)
+ .catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))
+ if (space.storageUsage !== spaceQuota.storageUsage) {
+ this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(
+ (updated: boolean) =>
+ updated &&
+ this.logger.log({
+ tag: this.updateSpacesQuota.name,
+ msg: `space *${space.alias}* (${space.id}) - storage usage updated : ${spaceQuota.storageUsage}`
+ })
+ )
+ }
+ if (spaceId) {
+ return spaceQuota
+ }
+ }
+ }
+
+ async updateSharesExternalPathQuota(): Promise<void>
+ async updateSharesExternalPathQuota(shareId: number): Promise<StorageQuota>
+ async updateSharesExternalPathQuota(shareId?: number): Promise<void | StorageQuota> {
+ for (const share of await this.sharesQueries.sharesQuotaExternalPaths(shareId)) {
+ if (!(await isPathExists(share.externalPath))) {
+ this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `*${share.alias}* home path does not exist` })
+ continue
+ }
+ const [size, errors] = await dirSize(share.externalPath)
+ for (const [path, error] of Object.entries(errors)) {
+ this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `unable to get size for *${share.alias}* on ${path} : ${error}` })
+ }
+ const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }
+ this.sharesQueries.cache
+ .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)
+ .catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))
+ if (share.storageUsage !== shareQuota.storageUsage) {
+ this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(
+ (updated: boolean) =>
+ updated &&
+ this.logger.log({
+ tag: this.updateSharesExternalPathQuota.name,
+ msg: `share *${share.alias}* (${share.id}) - storage usage updated : ${shareQuota.storageUsage}`
+ })
+ )
+ }
+ if (shareId) {
+ return shareQuota
+ }
+ }
+ }
+}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 80f46428..13f65e7f 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -1,5 +1,5 @@
import { Inject, Injectable, Logger } from '@nestjs/common'
-import { Cron, CronExpression, Timeout } from '@nestjs/schedule'
+import { Cron, CronExpression, Interval, Timeout } from '@nestjs/schedule'
import { isNotNull, sql } from 'drizzle-orm'
import { unionAll } from 'drizzle-orm/mysql-core'
import fs from 'node:fs/promises'
@@ -19,16 +19,20 @@ import { files } from '../schemas/files.schema'
import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'
import { FilesContentManager } from './files-content-manager.service'
import { FilesTasksManager } from './files-tasks-manager.service'
+import { FilesQuotaManager } from './files-quota-manager.service'
@Injectable()
export class FilesScheduler {
private readonly logger = new Logger(FilesScheduler.name)
private isIndexContentFilesRunning = false
+ private isQuotaUpdateIsRunning = false
+ private isQuotaUpdateEntriesIsRunning = false
constructor(
@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
private readonly cache: Cache,
- private readonly filesContentManager: FilesContentManager
+ private readonly filesContentManager: FilesContentManager,
+ private readonly filesQuotaManager: FilesQuotaManager
) {}
@Timeout(10_000)
@@ -36,6 +40,7 @@ export class FilesScheduler {
try {
await this.cleanupInterruptedTasks()
await this.clearRecentFiles()
+ await this.updateQuotas()
} catch (e) {
this.logger.error(e)
}
@@ -50,6 +55,19 @@ export class FilesScheduler {
}
}
+ @Interval(60_000)
+ async updateQuotaEntries() {
+ if (this.isQuotaUpdateIsRunning || this.isQuotaUpdateEntriesIsRunning) return
+ this.isQuotaUpdateEntriesIsRunning = true
+ try {
+ await this.filesQuotaManager.updateQuotaEntries()
+ } catch (e) {
+ this.logger.error({ tag: this.updateQuotas.name, msg: `${e}` })
+ } finally {
+ this.isQuotaUpdateEntriesIsRunning = false
+ }
+ }
+
async cleanupInterruptedTasks(): Promise<void> {
this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `START` })
try {
@@ -186,4 +204,32 @@ export class FilesScheduler {
}
this.logger.log({ tag: this.deleteOrphanFiles.name, msg: `END` })
}
+
+ @Cron(CronExpression.EVERY_HOUR)
+ async updateQuotas() {
+ if (this.isQuotaUpdateIsRunning) return
+ this.isQuotaUpdateIsRunning = true
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - START' })
+ try {
+ await this.filesQuotaManager.updatePersonalSpacesQuota()
+ } catch (e) {
+ this.logger.error({ tag: this.updateQuotas.name, msg: `Personals - ${e}` })
+ }
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - END' })
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - START' })
+ try {
+ await this.filesQuotaManager.updateSpacesQuota()
+ } catch (e) {
+ this.logger.error({ tag: this.updateQuotas.name, msg: `Spaces - ${e}` })
+ }
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - END' })
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - START' })
+ try {
+ await this.filesQuotaManager.updateSharesExternalPathQuota()
+ } catch (e) {
+ this.logger.error({ tag: this.updateQuotas.name, msg: `Share External Paths - ${e}` })
+ }
+ this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })
+ this.isQuotaUpdateIsRunning = false
+ }
}
diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.ts
new file mode 100644
index 00000000..1f085d3c
--- /dev/null
+++ b/backend/src/applications/files/utils/quota.ts
@@ -0,0 +1,24 @@
+import { SpaceEnv } from '../../spaces/models/space-env.model'
+import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'
+
+export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv) {
+ if (space.inPersonalSpace) {
+ // Personal user space
+ return `${CACHE_QUOTA_USER_PREFIX}-${userId}`
+ } else if (space.root?.externalPath) {
+ // External paths used as shares or as space roots share the same quota as their origin
+ if (space.inSharesRepository) {
+ return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`
+ }
+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
+ } else if (space.root.file?.path && space.root.owner?.login) {
+ // Space root is linked to a user file
+ return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`
+ } else if (space.root.file?.space?.id) {
+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`
+ } else if (space.id) {
+ return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
+ } else {
+ return null
+ }
+}
diff --git a/backend/src/applications/shares/services/shares-manager.service.ts b/backend/src/applications/shares/services/shares-manager.service.ts
index 13bb7823..533713ab 100644
--- a/backend/src/applications/shares/services/shares-manager.service.ts
+++ b/backend/src/applications/shares/services/shares-manager.service.ts
@@ -9,12 +9,12 @@ import {
generateShortUUID,
hashPassword
} from '../../../common/functions'
-import type { Entries, StorageQuota } from '../../../common/interfaces'
+import type { Entries } from '../../../common/interfaces'
import { intersectPermissions } from '../../../common/shared'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
import type { FileProps } from '../../files/interfaces/file-props.interface'
import { FileError } from '../../files/models/file-error'
-import { checkExternalPath, dirSize, getProps, isPathExists } from '../../files/utils/files'
+import { checkExternalPath, getProps, isPathExists } from '../../files/utils/files'
import { LINK_TYPE } from '../../links/constants/links'
import type { CreateOrUpdateLinkDto } from '../../links/dto/create-or-update-link.dto'
import type { LinkGuest } from '../../links/interfaces/link-guest.interface'
@@ -24,7 +24,6 @@ import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/co
import type { NotificationContent, NotificationOptions } from '../../notifications/interfaces/notification-properties.interface'
import type { UserMailNotification } from '../../notifications/interfaces/user-mail-notification.interface'
import { NotificationsManager } from '../../notifications/services/notifications-manager.service'
-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_TTL } from '../../spaces/constants/cache'
import { SPACE_OPERATION, SPACE_PERMS_SEP, SPACE_REPOSITORY, SPACE_ROLE } from '../../spaces/constants/spaces'
import type { SpaceMemberDto } from '../../spaces/dto/create-or-update-space.dto'
import { SpaceEnv } from '../../spaces/models/space-env.model'
@@ -736,38 +735,6 @@ export class SharesManager {
await this.deleteGuestLinks(members)
}
- async updateSharesExternalPathQuota(): Promise<void>
- async updateSharesExternalPathQuota(shareId: number): Promise<StorageQuota>
- async updateSharesExternalPathQuota(shareId?: number): Promise<void | StorageQuota> {
- for (const share of await this.sharesQueries.sharesQuotaExternalPaths(shareId)) {
- if (!(await isPathExists(share.externalPath))) {
- this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `*${share.alias}* home path does not exist` })
- continue
- }
- const [size, errors] = await dirSize(share.externalPath)
- for (const [path, error] of Object.entries(errors)) {
- this.logger.warn({ tag: this.updateSharesExternalPathQuota.name, msg: `unable to get size for *${share.alias}* on ${path} : ${error}` })
- }
- const shareQuota: StorageQuota = { storageUsage: size, storageQuota: share.storageQuota }
- this.sharesQueries.cache
- .set(`${CACHE_QUOTA_SHARE_PREFIX}-${share.id}`, shareQuota, CACHE_QUOTA_TTL)
- .catch((e: Error) => this.logger.error({ tag: this.updateSharesExternalPathQuota.name, msg: `share *${share.alias}* (${share.id}) : ${e}` }))
- if (share.storageUsage !== shareQuota.storageUsage) {
- this.sharesQueries.updateShare(share.id, { storageUsage: shareQuota.storageUsage }).then(
- (updated: boolean) =>
- updated &&
- this.logger.log({
- tag: this.updateSharesExternalPathQuota.name,
- msg: `share *${share.alias}* (${share.id}) - storage usage updated : ${shareQuota.storageUsage}`
- })
- )
- }
- if (shareId) {
- return shareQuota
- }
- }
- }
-
private async updateMembers(user: UserModel, share: Partial<Share>, oldMembers: ShareMemberDto[], currentMembers: ShareMemberDto[]) {
if (oldMembers.length === 0 && currentMembers.length === 0) {
return
diff --git a/backend/src/applications/spaces/constants/cache.ts b/backend/src/applications/spaces/constants/cache.ts
deleted file mode 100644
index bf57bff2..00000000
--- a/backend/src/applications/spaces/constants/cache.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-// cache quota key = `(quota-user|quota-space)-${id}` => number
-export const CACHE_QUOTA_USER_PREFIX = 'quota-user'
-export const CACHE_QUOTA_SPACE_PREFIX = 'quota-space'
-export const CACHE_QUOTA_SHARE_PREFIX = 'quota-share'
-export const CACHE_QUOTA_TTL = 86400 // 1 day
diff --git a/backend/src/applications/spaces/guards/space.guard.spec.ts b/backend/src/applications/spaces/guards/space.guard.spec.ts
index ec39d9e7..7a50c309 100644
--- a/backend/src/applications/spaces/guards/space.guard.spec.ts
+++ b/backend/src/applications/spaces/guards/space.guard.spec.ts
@@ -25,6 +25,7 @@ import { SpaceEnv } from '../models/space-env.model'
import { SpacesManager } from '../services/spaces-manager.service'
import { SpacesQueries } from '../services/spaces-queries.service'
import { SpaceGuard } from './space.guard'
+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'
describe(SpaceGuard.name, () => {
let spacesGuard: SpaceGuard
@@ -49,6 +50,10 @@ describe(SpaceGuard.name, () => {
provide: NotificationsManager,
useValue: {}
},
+ {
+ provide: FilesQuotaManager,
+ useValue: { setQuotaExceeded: () => jest.fn() }
+ },
SpaceGuard,
SpacesManager,
SpacesQueries,
@@ -64,7 +69,6 @@ describe(SpaceGuard.name, () => {
spacesQueries = module.get<SpacesQueries>(SpacesQueries)
spacesGuard = module.get<SpaceGuard>(SpaceGuard)
// mocks
- spacesManager['setQuotaExceeded'] = jest.fn()
userTest = new UserModel(generateUserTest())
})
diff --git a/backend/src/applications/spaces/services/spaces-browser.service.spec.ts b/backend/src/applications/spaces/services/spaces-browser.service.spec.ts
index 726fdb32..4017f60a 100644
--- a/backend/src/applications/spaces/services/spaces-browser.service.spec.ts
+++ b/backend/src/applications/spaces/services/spaces-browser.service.spec.ts
@@ -15,6 +15,7 @@ import { UsersQueries } from '../../users/services/users-queries.service'
import { SpacesBrowser } from './spaces-browser.service'
import { SpacesManager } from './spaces-manager.service'
import { SpacesQueries } from './spaces-queries.service'
+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'
describe(SpacesBrowser.name, () => {
let spacesBrowserService: SpacesBrowser
@@ -33,6 +34,7 @@ describe(SpacesBrowser.name, () => {
provide: NotificationsManager,
useValue: {}
},
+ { provide: FilesQuotaManager, useValue: {} },
SpacesManager,
SpacesBrowser,
SpacesQueries,
diff --git a/backend/src/applications/spaces/services/spaces-manager.service.spec.ts b/backend/src/applications/spaces/services/spaces-manager.service.spec.ts
index ab5c5598..a31a9f1c 100644
--- a/backend/src/applications/spaces/services/spaces-manager.service.spec.ts
+++ b/backend/src/applications/spaces/services/spaces-manager.service.spec.ts
@@ -29,6 +29,7 @@ import { SpaceModel } from '../models/space.model'
import { IsRealPathIsDirAndExists } from '../utils/paths'
import { SpacesManager } from './spaces-manager.service'
import { SpacesQueries } from './spaces-queries.service'
+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'
describe(SpacesManager.name, () => {
let filesConfig: FilesConfig
@@ -52,6 +53,10 @@ describe(SpacesManager.name, () => {
provide: NotificationsManager,
useValue: {}
},
+ {
+ provide: FilesQuotaManager,
+ useValue: { setQuotaExceeded: () => jest.fn() }
+ },
SpacesManager,
SpacesQueries,
UsersQueries,
@@ -66,9 +71,7 @@ describe(SpacesManager.name, () => {
filesConfig = module.get<ConfigService>(ConfigService).get('applications.files')
spacesManager = module.get<SpacesManager>(SpacesManager)
spacesQueries = module.get<SpacesQueries>(SpacesQueries)
- spacesManager['setQuotaExceeded'] = jest.fn()
userTest = new UserModel(generateUserTest())
- // todo: validate shares, permissions
})
afterAll(async () => {
diff --git a/backend/src/applications/spaces/services/spaces-manager.service.ts b/backend/src/applications/spaces/services/spaces-manager.service.ts
index a8e59b81..9e71059f 100644
--- a/backend/src/applications/spaces/services/spaces-manager.service.ts
+++ b/backend/src/applications/spaces/services/spaces-manager.service.ts
@@ -1,15 +1,15 @@
import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'
-import { eq, isNotNull, lte } from 'drizzle-orm'
+import { eq, isNotNull } from 'drizzle-orm'
import fs from 'node:fs/promises'
import path from 'node:path'
import { ACTION } from '../../../common/constants'
import { convertDiffUpdate, diffCollection, differencePermissions } from '../../../common/functions'
-import type { Entries, StorageQuota } from '../../../common/interfaces'
+import type { Entries } from '../../../common/interfaces'
import { createSlug, regExpNumberSuffix } from '../../../common/shared'
import { configuration } from '../../../configuration/config.environment'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
import { FileError } from '../../files/models/file-error'
-import { dirListFileNames, dirSize, getProps, isPathExists, moveFiles, removeFiles } from '../../files/utils/files'
+import { dirListFileNames, getProps, isPathExists, moveFiles, removeFiles } from '../../files/utils/files'
import { LINK_TYPE } from '../../links/constants/links'
import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/constants/notifications'
import { NotificationContent } from '../../notifications/interfaces/notification-properties.interface'
@@ -20,12 +20,8 @@ import type { ShareProps } from '../../shares/interfaces/share-props.interface'
import type { ShareChild } from '../../shares/models/share-child.model'
import { SharesManager } from '../../shares/services/shares-manager.service'
import { MEMBER_TYPE } from '../../users/constants/member'
-import { USER_ROLE } from '../../users/constants/user'
import { UserModel } from '../../users/models/user.model'
-import type { User } from '../../users/schemas/user.interface'
-import { users } from '../../users/schemas/users.schema'
import { UsersQueries } from '../../users/services/users-queries.service'
-import { CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_TTL, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'
import {
SPACE_ALIAS,
SPACE_ALL_OPERATIONS,
@@ -47,9 +43,10 @@ import { SpaceModel } from '../models/space.model'
import type { SpaceRoot } from '../schemas/space-root.interface'
import type { Space } from '../schemas/space.interface'
import { spaces } from '../schemas/spaces.schema'
-import { quotaKeyFromSpace } from '../utils/paths'
import { haveSpacePermission } from '../utils/permissions'
import { SpacesQueries } from './spaces-queries.service'
+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'
+import { CACHE_QUOTA_SPACE_PREFIX } from '../../files/constants/cache'
@Injectable()
export class SpacesManager {
@@ -60,6 +57,7 @@ export class SpacesManager {
private readonly spacesQueries: SpacesQueries,
private readonly usersQueries: UsersQueries,
private readonly sharesManager: SharesManager,
+ private readonly filesQuotaManager: FilesQuotaManager,
private readonly notificationsManager: NotificationsManager
) {}
@@ -138,7 +136,7 @@ export class SpacesManager {
if (!space) return null
try {
space.setup(user, repository as SPACE_REPOSITORY, rootAlias, paths, urlSegments, skipEndpointProtection)
- await this.setQuotaExceeded(user, space)
+ await this.filesQuotaManager.setQuotaExceeded(user, space)
return space
} catch (e) {
this.logger.warn({ tag: this.spaceEnv.name, msg: `*${space.alias}* : ${e}` })
@@ -166,7 +164,7 @@ export class SpacesManager {
return Promise.all(
(await this.spacesQueries.spaces(user.id, true)).map(async (s) => {
const space = new SpaceEnv(s)
- await this.setQuotaExceeded(user, space)
+ await this.filesQuotaManager.setQuotaExceeded(user, space)
return space
})
)
@@ -363,80 +361,6 @@ export class SpacesManager {
return null
}
- async updatePersonalSpacesQuota(): Promise<void>
- async updatePersonalSpacesQuota(forUser: UserModel): Promise<StorageQuota>
- async updatePersonalSpacesQuota(forUser?: UserModel): Promise<void | StorageQuota> {
- const props: (keyof User)[] = ['id', 'login', 'storageUsage', 'storageQuota']
- for (const user of await this.usersQueries.selectUsers(props, [
- lte(users.role, USER_ROLE.USER),
- ...(forUser ? [eq(users.id, forUser.id)] : [])
- ])) {
- const userPath = UserModel.getHomePath(user.login)
- if (!(await isPathExists(userPath))) {
- this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `*${user.login}* home path does not exist` })
- continue
- }
- const [size, errors] = await dirSize(userPath)
- for (const [path, error] of Object.entries(errors)) {
- this.logger.warn({ tag: this.updatePersonalSpacesQuota.name, msg: `unable to get size for *${user.login}* on ${path} : ${error}` })
- }
- const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: user.storageQuota }
- this.spacesQueries.cache
- .set(`${CACHE_QUOTA_USER_PREFIX}-${user.id}`, spaceQuota, CACHE_QUOTA_TTL)
- .catch((e: Error) => this.logger.error({ tag: this.updatePersonalSpacesQuota.name, msg: `user *${user.login}* (${user.id}) : ${e}` }))
- if (user.storageUsage !== spaceQuota.storageUsage) {
- this.usersQueries.updateUserOrGuest(user.id, { storageUsage: spaceQuota.storageUsage }).then(
- (updated: boolean) =>
- updated &&
- this.logger.log({
- tag: this.updatePersonalSpacesQuota.name,
- msg: `user *${user.login}* (${user.id}) - storage usage updated: ${spaceQuota.storageUsage}`
- })
- )
- }
- if (forUser) {
- return spaceQuota
- }
- }
- }
-
- async updateSpacesQuota(): Promise<void>
- async updateSpacesQuota(spaceId: number): Promise<StorageQuota>
- async updateSpacesQuota(spaceId?: number): Promise<void | StorageQuota> {
- for (const space of await this.spacesQueries.spacesQuotaPaths(spaceId)) {
- const spacePath = SpaceModel.getHomePath(space.alias)
- if (!(await isPathExists(spacePath))) {
- this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `*${space.alias}* home path does not exist` })
- continue
- }
- let size = 0
- for (const rPath of [spacePath, ...space.externalPaths.filter(Boolean)]) {
- const [rPathSize, errors] = await dirSize(rPath)
- size += rPathSize
- for (const [path, error] of Object.entries(errors)) {
- this.logger.warn({ tag: this.updateSpacesQuota.name, msg: `unable to get size for *${space.alias}* on ${path} : ${error}` })
- }
- }
- const spaceQuota: StorageQuota = { storageUsage: size, storageQuota: space.storageQuota }
- this.spacesQueries.cache
- .set(`${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`, spaceQuota, CACHE_QUOTA_TTL)
- .catch((e: Error) => this.logger.error({ tag: this.updateSpacesQuota.name, msg: `space *${space.alias}* (${space.id}) : ${e}` }))
- if (space.storageUsage !== spaceQuota.storageUsage) {
- this.spacesQueries.updateSpace(space.id, { storageUsage: spaceQuota.storageUsage }).then(
- (updated: boolean) =>
- updated &&
- this.logger.log({
- tag: this.updateSpacesQuota.name,
- msg: `space *${space.alias}* (${space.id}) - storage usage updated : ${spaceQuota.storageUsage}`
- })
- )
- }
- if (spaceId) {
- return spaceQuota
- }
- }
- }
-
async deleteExpiredSpaces() {
/* Removes spaces that have been disabled for more than 30 days */
const props: (keyof Space)[] = ['id', 'name', 'alias', 'disabledAt']
@@ -829,37 +753,6 @@ export class SpacesManager {
throw new HttpException('Not authorized', HttpStatus.FORBIDDEN)
}
- private async setQuotaExceeded(user: UserModel, space: SpaceEnv) {
- /* extract quota from spaces|shares|roots */
- if (space.inSharesList) {
- return
- }
- const cacheQuotaKey = quotaKeyFromSpace(user.id, space)
- if (!cacheQuotaKey) {
- this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota was ignored for space : *${space.alias}* (${space.id})` })
- return
- }
- let quota: StorageQuota = await this.spacesQueries.cache.get(cacheQuotaKey)
- if (!quota) {
- // the quota scheduler has not started yet or the cache has been cleared
- if (space.inPersonalSpace) {
- quota = await this.updatePersonalSpacesQuota(user)
- } else if (space.inSharesRepository) {
- // Shares with external paths
- quota = await this.sharesManager.updateSharesExternalPathQuota(space.id)
- } else {
- quota = await this.updateSpacesQuota(space.id)
- }
- }
- if (quota) {
- space.storageUsage = quota.storageUsage
- space.storageQuota = quota.storageQuota
- space.quotaIsExceeded = quota.storageQuota !== null && quota.storageUsage >= quota.storageQuota
- } else {
- this.logger.verbose({ tag: this.setQuotaExceeded.name, msg: `quota not found for space : *${space.alias}* (${space.id})` })
- }
- }
-
private async onSpaceActionForMembers(
space: Partial<SpaceProps>,
action: ACTION,
diff --git a/backend/src/applications/spaces/services/spaces-scheduler.service.ts b/backend/src/applications/spaces/services/spaces-scheduler.service.ts
index f4abaf37..81c97b2e 100644
--- a/backend/src/applications/spaces/services/spaces-scheduler.service.ts
+++ b/backend/src/applications/spaces/services/spaces-scheduler.service.ts
@@ -1,46 +1,12 @@
import { Injectable, Logger } from '@nestjs/common'
-import { Cron, CronExpression, Timeout } from '@nestjs/schedule'
-import { SharesManager } from '../../shares/services/shares-manager.service'
+import { Cron, CronExpression } from '@nestjs/schedule'
import { SpacesManager } from './spaces-manager.service'
@Injectable()
export class SpacesScheduler {
private readonly logger = new Logger(SpacesScheduler.name)
- constructor(
- private readonly spacesManager: SpacesManager,
- private readonly sharesManager: SharesManager
- ) {}
-
- @Timeout(60_000)
- async onStartup() {
- await this.updateQuotas()
- }
-
- @Cron(CronExpression.EVERY_HOUR)
- async updateQuotas() {
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - START' })
- try {
- await this.spacesManager.updatePersonalSpacesQuota()
- } catch (e) {
- this.logger.error({ tag: this.updateQuotas.name, msg: `Personals} - ${e}` })
- }
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Personals - END' })
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - START' })
- try {
- await this.spacesManager.updateSpacesQuota()
- } catch (e) {
- this.logger.error({ tag: this.updateQuotas.name, msg: `Spaces - ${e}` })
- }
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Spaces - END' })
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - START' })
- try {
- await this.sharesManager.updateSharesExternalPathQuota()
- } catch (e) {
- this.logger.error({ tag: this.updateQuotas.name, msg: `Share External Paths - ${e}` })
- }
- this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })
- }
+ constructor(private readonly spacesManager: SpacesManager) {}
@Cron(CronExpression.EVERY_DAY_AT_2AM)
async deleteExpiredSpaces() {
diff --git a/backend/src/applications/spaces/spaces.controller.spec.ts b/backend/src/applications/spaces/spaces.controller.spec.ts
index 89f88126..2253a271 100644
--- a/backend/src/applications/spaces/spaces.controller.spec.ts
+++ b/backend/src/applications/spaces/spaces.controller.spec.ts
@@ -14,6 +14,7 @@ import { SpacesBrowser } from './services/spaces-browser.service'
import { SpacesManager } from './services/spaces-manager.service'
import { SpacesQueries } from './services/spaces-queries.service'
import { SpacesController } from './spaces.controller'
+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'
describe(SpacesController.name, () => {
let spacesController: SpacesController
@@ -31,6 +32,10 @@ describe(SpacesController.name, () => {
provide: Cache,
useValue: {}
},
+ {
+ provide: FilesQuotaManager,
+ useValue: {}
+ },
ContextManager,
SpacesManager,
SpacesQueries,
diff --git a/backend/src/applications/spaces/utils/paths.ts b/backend/src/applications/spaces/utils/paths.ts
index fb59c18a..e437bd83 100644
--- a/backend/src/applications/spaces/utils/paths.ts
+++ b/backend/src/applications/spaces/utils/paths.ts
@@ -5,7 +5,6 @@ import { FileDBProps } from '../../files/interfaces/file-db-props.interface'
import { FileProps } from '../../files/interfaces/file-props.interface'
import { FileError } from '../../files/models/file-error'
import { UserModel } from '../../users/models/user.model'
-import { CACHE_QUOTA_SHARE_PREFIX, CACHE_QUOTA_SPACE_PREFIX, CACHE_QUOTA_USER_PREFIX } from '../constants/cache'
import { SPACE_REPOSITORY } from '../constants/spaces'
import { SpaceEnv } from '../models/space-env.model'
import { SpaceModel } from '../models/space.model'
@@ -176,25 +175,3 @@ export function dbFileFromSpace(userId: number, space: SpaceEnv): FileDBProps {
}
return dbFile
}
-
-export function quotaKeyFromSpace(userId: number, space: SpaceEnv) {
- if (space.inPersonalSpace) {
- // Personal user space
- return `${CACHE_QUOTA_USER_PREFIX}-${userId}`
- } else if (space.root?.externalPath) {
- // External paths used as shares or as space roots share the same quota as their origin
- if (space.inSharesRepository) {
- return `${CACHE_QUOTA_SHARE_PREFIX}-${space.root?.externalParentShareId || space.id}`
- }
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
- } else if (space.root.file?.path && space.root.owner?.login) {
- // Space root is linked on a user file
- return `${CACHE_QUOTA_USER_PREFIX}-${space.root.owner.id}`
- } else if (space.root.file?.space?.id) {
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.root.file.space.id}`
- } else if (space.id) {
- return `${CACHE_QUOTA_SPACE_PREFIX}-${space.id}`
- } else {
- throw new FileError(HttpStatus.NOT_FOUND, 'Space root not found')
- }
-}