Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): ensure storage quota is updated in cache
johaven030b87ebeec6
diff --git a/backend/src/applications/files/services/files-quota-manager.service.ts b/backend/src/applications/files/services/files-quota-manager.service.tsindex 0a79957d..5a826e38 100644--- a/backend/src/applications/files/services/files-quota-manager.service.ts+++ b/backend/src/applications/files/services/files-quota-manager.service.ts@@ -56,7 +56,7 @@ export class FilesQuotaManager {}}- async updateQuotaEntries() {+ async updateStorageUsageEntries() {for (const k of await this.spacesQueries.cache.keys(`${CACHE_QUOTA_EVENT_UPDATE_PREFIX}-*`)) {try {const keySegments = k.split('-')@@ -69,16 +69,34 @@ export class FilesQuotaManager {} else if (repository === FILE_REPOSITORY.SHARE) {await this.updateSharesExternalPathQuota(id)} else {- this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unknown type: ${repository}` })+ this.logger.warn({ tag: this.updateStorageUsageEntries.name, msg: `Unknown type: ${repository}` })}} catch (e) {- this.logger.error({ tag: this.updateQuotaEntries.name, msg: `${e}` })+ this.logger.error({ tag: this.updateStorageUsageEntries.name, msg: `${e}` })} finally {- this.spacesQueries.cache.del(k).catch((e) => this.logger.warn({ tag: this.updateQuotaEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))+ this.spacesQueries.cache+ .del(k)+ .catch((e) => this.logger.warn({ tag: this.updateStorageUsageEntries.name, msg: `Unable to clean key: ${k} - ${e}` }))}}}+ async updateStorageQuota(id: number, type: FILE_REPOSITORY, quota: number) {+ const cacheKey = genQuotaCacheKey(id, type)+ const entryQuota: StorageQuota = await this.spacesQueries.cache.get(cacheKey)+ if (entryQuota && entryQuota.storageUsage !== undefined) {+ this.spacesQueries.cache+ .set(cacheKey, { storageUsage: entryQuota.storageUsage, storageQuota: quota }, CACHE_QUOTA_TTL)+ .catch((e: Error) => this.logger.error({ tag: this.updateStorageQuota.name, msg: `type *${type}* (${id}) - ${e}` }))+ } else {+ // store event update for later propagation+ const updateCacheKey = genQuotaCacheKey(id, type, true)+ this.spacesQueries.cache+ .set(updateCacheKey, true, CACHE_QUOTA_TTL)+ .catch((e: Error) => this.logger.error({ tag: this.updateStorageQuota.name, msg: `type *${type}* (${id}) event - ${e}` }))+ }+ }+async updatePersonalSpacesQuota(): Promise<void>async updatePersonalSpacesQuota(userId: number): Promise<StorageQuota>async updatePersonalSpacesQuota(userId?: number): Promise<void | StorageQuota> {diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 13f65e7f..2777d449 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -56,11 +56,11 @@ export class FilesScheduler {}@Interval(60_000)- async updateQuotaEntries() {+ async updateStorageUsageEntries() {if (this.isQuotaUpdateIsRunning || this.isQuotaUpdateEntriesIsRunning) returnthis.isQuotaUpdateEntriesIsRunning = truetry {- await this.filesQuotaManager.updateQuotaEntries()+ await this.filesQuotaManager.updateStorageUsageEntries()} catch (e) {this.logger.error({ tag: this.updateQuotas.name, msg: `${e}` })} finally {diff --git a/backend/src/applications/shares/services/shares-manager.service.spec.ts b/backend/src/applications/shares/services/shares-manager.service.spec.tsindex 49fde082..91cf22d4 100644--- a/backend/src/applications/shares/services/shares-manager.service.spec.ts+++ b/backend/src/applications/shares/services/shares-manager.service.spec.ts@@ -14,6 +14,7 @@ import { UsersQueries } from '../../users/services/users-queries.service'import { SHARE_ALL_OPERATIONS } from '../constants/shares'import { SharesManager } from './shares-manager.service'import { SharesQueries } from './shares-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'// Mock classes and utility modules used by SharesManagerjest.mock('../../spaces/models/space-env.model', () => ({@@ -102,6 +103,10 @@ describe(SharesManager.name, () => {const module: TestingModule = await Test.createTestingModule({providers: [{ provide: DB_TOKEN_PROVIDER, useValue: {} },+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },{ provide: ContextManager, useValue: contextManagerMock },{ provide: NotificationsManager, useValue: notificationsManagerMock },{ provide: SpacesQueries, useValue: spacesQueriesMock },diff --git a/backend/src/applications/shares/services/shares-manager.service.ts b/backend/src/applications/shares/services/shares-manager.service.tsindex 533713ab..11b2e79d 100644--- a/backend/src/applications/shares/services/shares-manager.service.ts+++ b/backend/src/applications/shares/services/shares-manager.service.ts@@ -47,6 +47,8 @@ import type { ShareChild } from '../models/share-child.model'import type { ShareMembers } from '../schemas/share-members.interface'import type { Share } from '../schemas/share.interface'import { SharesQueries } from './shares-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'+import { FILE_REPOSITORY } from '../../files/constants/operations'@Injectable()export class SharesManager {@@ -55,6 +57,7 @@ export class SharesManager {constructor(private readonly contextManager: ContextManager,private readonly notificationsManager: NotificationsManager,+ private readonly filesQuotaManager: FilesQuotaManager,private readonly sharesQueries: SharesQueries,private readonly spacesQueries: SpacesQueries,private readonly usersQueries: UsersQueries,@@ -247,6 +250,10 @@ export class SharesManager {if (!(await this.sharesQueries.updateShare(shareId, shareDiffProps))) {throw new HttpException('Unable to update share', HttpStatus.INTERNAL_SERVER_ERROR)}+ // update quota in cache+ if ('storageQuota' in shareDiffProps) {+ void this.filesQuotaManager.updateStorageQuota(shareId, FILE_REPOSITORY.SHARE, shareDiffProps.storageQuota)+ }// check & update membersconst linkMembers: ShareMemberDto[] = await this.createOrUpdateLinksAsMembers(user, share, LINK_TYPE.SHARE, createOrUpdateShareDto.links)// intersect share permissions for membersdiff --git a/backend/src/applications/shares/shares.controller.spec.ts b/backend/src/applications/shares/shares.controller.spec.tsindex 45aa0011..edfcdaec 100644--- a/backend/src/applications/shares/shares.controller.spec.ts+++ b/backend/src/applications/shares/shares.controller.spec.ts@@ -10,6 +10,7 @@ import { UsersQueries } from '../users/services/users-queries.service'import { SharesManager } from './services/shares-manager.service'import { SharesQueries } from './services/shares-queries.service'import { SharesController } from './shares.controller'+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'describe(SharesController.name, () => {let controller: SharesController@@ -28,6 +29,10 @@ describe(SharesController.name, () => {provide: NotificationsManager,useValue: {}},+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },SpacesQueries,FilesQueries,UsersQueries,diff --git a/backend/src/applications/spaces/services/spaces-manager.service.ts b/backend/src/applications/spaces/services/spaces-manager.service.tsindex 3b9b6371..8c776b41 100644--- a/backend/src/applications/spaces/services/spaces-manager.service.ts+++ b/backend/src/applications/spaces/services/spaces-manager.service.ts@@ -256,8 +256,14 @@ export class SpacesManager {}}}- // updates in db- this.spacesQueries.updateSpace(spaceId, spaceDiffProps).catch((e: Error) => this.logger.error({ tag: this.updateSpace.name, msg: `${e}` }))+ // update in db+ if (!(await this.spacesQueries.updateSpace(spaceId, spaceDiffProps))) {+ throw new HttpException('Unable to update space', HttpStatus.INTERNAL_SERVER_ERROR)+ }+ // update quota in cache+ if ('storageQuota' in spaceDiffProps) {+ void this.filesQuotaManager.updateStorageQuota(spaceId, FILE_REPOSITORY.SPACE, spaceDiffProps.storageQuota)+ }// checks & updates membersconst linkMembers: SpaceMemberDto[] = await this.sharesManager.createOrUpdateLinksAsMembers(user,diff --git a/backend/src/applications/users/admin-users.controller.spec.ts b/backend/src/applications/users/admin-users.controller.spec.tsindex 35e8776a..742c795c 100644--- a/backend/src/applications/users/admin-users.controller.spec.ts+++ b/backend/src/applications/users/admin-users.controller.spec.ts@@ -13,6 +13,7 @@ import { AdminUsersManager } from './services/admin-users-manager.service'import { AdminUsersQueries } from './services/admin-users-queries.service'import { UsersManager } from './services/users-manager.service'import { UsersQueries } from './services/users-queries.service'+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'describe(AdminUsersController.name, () => {let controller: AdminUsersController@@ -30,6 +31,10 @@ describe(AdminUsersController.name, () => {{ provide: AuthProvider2FA, useValue: {} },{ provide: AuthTwoFaGuard, useValue: {} },{ provide: NotificationsManager, useValue: {} },+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },JwtService,AuthManager,AdminUsersManager,diff --git a/backend/src/applications/users/services/admin-users-manager.service.spec.ts b/backend/src/applications/users/services/admin-users-manager.service.spec.tsindex 9f11be8e..fc4ac31b 100644--- a/backend/src/applications/users/services/admin-users-manager.service.spec.ts+++ b/backend/src/applications/users/services/admin-users-manager.service.spec.ts@@ -10,6 +10,7 @@ import type { SearchMembersDto } from '../dto/search-members.dto'import { UserModel } from '../models/user.model'import { AdminUsersManager } from './admin-users-manager.service'import { AdminUsersQueries } from './admin-users-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'// mock file utils used by the service (delete/rename user space)jest.mock('../../files/utils/files', () => ({@@ -114,7 +115,15 @@ describe(AdminUsersManager.name, () => {}const module: TestingModule = await Test.createTestingModule({- providers: [AdminUsersManager, { provide: AuthManager, useValue: authManagerMock }, { provide: AdminUsersQueries, useValue: adminQueriesMock }]+ providers: [+ AdminUsersManager,+ { provide: AuthManager, useValue: authManagerMock },+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },+ { provide: AdminUsersQueries, useValue: adminQueriesMock }+ ]}).compile()module.useLogger(['fatal'])diff --git a/backend/src/applications/users/services/admin-users-manager.service.ts b/backend/src/applications/users/services/admin-users-manager.service.tsindex f044c560..cfb7d433 100644--- a/backend/src/applications/users/services/admin-users-manager.service.ts+++ b/backend/src/applications/users/services/admin-users-manager.service.ts@@ -19,6 +19,8 @@ import { UserModel } from '../models/user.model'import type { Group } from '../schemas/group.interface'import type { User } from '../schemas/user.interface'import { AdminUsersQueries } from './admin-users-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'+import { FILE_REPOSITORY } from '../../files/constants/operations'@Injectable()export class AdminUsersManager {@@ -26,6 +28,7 @@ export class AdminUsersManager {constructor(private readonly authManager: AuthManager,+ private readonly filesQuotaManager: FilesQuotaManager,private readonly adminQueries: AdminUsersQueries) {}@@ -143,9 +146,14 @@ export class AdminUsersManager {if (Object.keys(updateUser).length) {// force the type for security reasonconst forceRole = userRole === USER_ROLE.GUEST ? USER_ROLE.GUEST : undefined+ // updates in dbif (!(await this.adminQueries.usersQueries.updateUserOrGuest(user.id, updateUser, forceRole))) {throw new HttpException('Unable to update user', HttpStatus.INTERNAL_SERVER_ERROR)}+ // update quota in cache+ if ('storageQuota' in updateUser) {+ void this.filesQuotaManager.updateStorageQuota(user.id, FILE_REPOSITORY.USER, updateUser.storageQuota)+ }}if (updateUserGroups.add.length || updateUserGroups.delete.length) {diff --git a/backend/src/applications/users/services/users-manager.service.spec.ts b/backend/src/applications/users/services/users-manager.service.spec.tsindex d2f76d4a..122217ed 100755--- a/backend/src/applications/users/services/users-manager.service.spec.ts+++ b/backend/src/applications/users/services/users-manager.service.spec.ts@@ -22,6 +22,7 @@ import { AdminUsersManager } from './admin-users-manager.service'import { AdminUsersQueries } from './admin-users-queries.service'import { UsersManager } from './users-manager.service'import { UsersQueries } from './users-queries.service'+import { FilesQuotaManager } from '../../files/services/files-quota-manager.service'jest.mock('../../../common/functions', () => {const actual = jest.requireActual('../../../common/functions')@@ -76,6 +77,10 @@ describe(UsersManager.name, () => {AdminUsersQueries,UsersManager,UsersQueries,+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },{ provide: NotificationsManager, useValue: notificationsManager },{ provide: AuthManager, useValue: {} },{ provide: DB_TOKEN_PROVIDER, useValue: {} },diff --git a/backend/src/applications/users/users.controller.spec.ts b/backend/src/applications/users/users.controller.spec.tsindex 1147f510..581fb41d 100755--- a/backend/src/applications/users/users.controller.spec.ts+++ b/backend/src/applications/users/users.controller.spec.ts@@ -14,6 +14,7 @@ import { UsersManager } from './services/users-manager.service'import { UsersQueries } from './services/users-queries.service'import { UsersController } from './users.controller'import { generateUserTest } from './utils/test'+import { FilesQuotaManager } from '../files/services/files-quota-manager.service'describe(UsersController.name, () => {let module: TestingModule@@ -32,6 +33,10 @@ describe(UsersController.name, () => {provide: Cache,useValue: {}},+ {+ provide: FilesQuotaManager,+ useValue: { updateStorageQuota: () => jest.fn() }+ },UsersManager,UsersQueries,AdminUsersManager,diff --git a/backend/src/authentication/providers/mysql/auth-provider-mysql.service.spec.ts b/backend/src/authentication/providers/mysql/auth-provider-mysql.service.spec.tsindex 967873de..61e7b0b6 100644--- a/backend/src/authentication/providers/mysql/auth-provider-mysql.service.spec.ts+++ b/backend/src/authentication/providers/mysql/auth-provider-mysql.service.spec.ts@@ -24,8 +24,8 @@ describe(AuthProviderMySQL.name, () => {AuthProviderMySQL,UsersManager,UsersQueries,- AdminUsersManager,AdminUsersQueries,+ { provide: AdminUsersManager, useValue: {} },{ provide: AuthManager, useValue: {} },{ provide: DB_TOKEN_PROVIDER, useValue: {} },{ provide: Cache, useValue: {} },