Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): split trash retention by repository type
johaven1c490eee902b
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.tsindex ab128da5..0abf7a10 100644--- a/backend/src/applications/files/files.config.ts+++ b/backend/src/applications/files/files.config.ts@@ -48,6 +48,20 @@ export class FilesContentIndexingConfig {ocr: FilesContentIndexingOCRConfig = new FilesContentIndexingOCRConfig()}+export class FilesTrashRetentionConfig {+ @Transform(({ value }) => (value === 0 ? false : value))+ @ValidateIf((o: FilesTrashRetentionConfig) => o.users !== false)+ @IsInt()+ @Min(1)+ users: number | false = false++ @Transform(({ value }) => (value === 0 ? false : value))+ @ValidateIf((o: FilesTrashRetentionConfig) => o.spaces !== false)+ @IsInt()+ @Min(1)+ spaces: number | false = false+}+export class FilesConfig {@IsNotEmpty()@IsString()@@ -73,11 +87,10 @@ export class FilesConfig {@Type(() => FilesContentIndexingConfig)contentIndexing: FilesContentIndexingConfig = new FilesContentIndexingConfig()- @Transform(({ value }) => (value === 0 ? false : value))- @ValidateIf((o: FilesConfig) => o.trashRetentionDays !== false)- @IsInt()- @Min(1)- trashRetentionDays: number | false = false+ @IsNotEmptyObject()+ @ValidateNested()+ @Type(() => FilesTrashRetentionConfig)+ trashRetention: FilesTrashRetentionConfig = new FilesTrashRetentionConfig()@IsBoolean()showHiddenFiles: boolean = falsediff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.tsindex c2501e7e..7b5035bb 100644--- a/backend/src/applications/files/interfaces/file-parse-index.ts+++ b/backend/src/applications/files/interfaces/file-parse-index.ts@@ -22,6 +22,7 @@ export interface FileParseTrashRetentionPath {id: numbertype: FILE_REPOSITORYrealPath: string+ retentionDays: number}export interface FileTrashRetentionIndexContext {diff --git a/backend/src/applications/files/services/files-trash-retention.service.spec.ts b/backend/src/applications/files/services/files-trash-retention.service.spec.tsindex ec839628..cadeab6b 100644--- a/backend/src/applications/files/services/files-trash-retention.service.spec.ts+++ b/backend/src/applications/files/services/files-trash-retention.service.spec.ts@@ -49,8 +49,8 @@ describe(FilesTrashRetention.name, () => {return records}- const mockTrashPath = (files: FileTrash[] = []) => {- jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type: FILE_REPOSITORY.USER, realPath: '/trash' }])+ const mockTrashPath = (files: FileTrash[] = [], type: FILE_REPOSITORY.USER | FILE_REPOSITORY.SPACE = FILE_REPOSITORY.USER, retentionDays = 30) => {+ jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type, realPath: '/trash', retentionDays }])jest.spyOn(service as any, 'createTable').mockResolvedValueOnce(true)jest.spyOn(service as any, 'parseFiles').mockReturnValueOnce(asyncGen(files))}@@ -82,7 +82,6 @@ describe(FilesTrashRetention.name, () => {})it('should process trash files by batches of 1000 and delete unseen records', async () => {- ;(service as any).retentionDays = 30const files = Array.from({ length: 1001 }, (_, i) => fileTrash(i + 1))mockTrashPath(files)const indexTrashBatchSpy = jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValue({ indexedRecords: 0, errorRecords: 0 })@@ -95,11 +94,21 @@ describe(FilesTrashRetention.name, () => {expect(indexTrashBatchSpy.mock.calls[0][2]).toHaveLength(1000)expect(indexTrashBatchSpy.mock.calls[1][2]).toHaveLength(1)expect(deleteUnseenRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', expect.any(String))- expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash')+ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash', 30)+ })++ it('should use the trash retention attached to each repository path', async () => {+ mockTrashPath([fileTrash(1)], FILE_REPOSITORY.SPACE, 14)+ jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })+ jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(0)+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords').mockResolvedValueOnce({ deletedRecords: 0, errorRecords: 0 })++ await service.indexAndCleanTrash()++ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_space_9', '/trash', 14)})it('should skip expired cleanup when unseen records cannot be deleted', async () => {- ;(service as any).retentionDays = 30mockTrashPath([fileTrash(1)])jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(null)@@ -167,7 +176,6 @@ describe(FilesTrashRetention.name, () => {})it('should delete expired records from filesystem and database', async () => {- ;(service as any).retentionDays = 30const expiredRecords = [fileTrash(1), fileTrash(2, { path: 'nested' })]const getExpiredRecordsSpy = jest.spyOn(service as any, 'getExpiredRecords')@@ -180,23 +188,22 @@ describe(FilesTrashRetention.name, () => {.mockResolvedValueOnce(false)const deleteRecordsByIdsSpy = jest.spyOn(service as any, 'deleteRecordsByIds').mockResolvedValueOnce(1)- await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash')).resolves.toEqual({+ await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash', 30)).resolves.toEqual({deletedRecords: 1,errorRecords: 1})expect(getExpiredRecordsSpy).toHaveBeenCalledTimes(3)- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, [])- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, [2])- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, [])+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, 30, [])+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, 30, [2])+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, 30, [])expect(deleteRecordsByIdsSpy).toHaveBeenCalledWith('files_trash_user_1', [1])})it('should only query expired directories without indexed descendants', async () => {- ;(service as any).retentionDays = 30db.execute.mockResolvedValueOnce([[]])- await expect((service as any).getExpiredRecords('files_trash_user_1', true)).resolves.toEqual([])+ await expect((service as any).getExpiredRecords('files_trash_user_1', true, 30)).resolves.toEqual([])const query = sqlText(db.execute.mock.calls[0][0])expect(query).toContain('NOT EXISTS')diff --git a/backend/src/applications/files/services/files-trash-retention.service.ts b/backend/src/applications/files/services/files-trash-retention.service.tsindex 4ce3a31f..6cc7c5c1 100644--- a/backend/src/applications/files/services/files-trash-retention.service.ts+++ b/backend/src/applications/files/services/files-trash-retention.service.ts@@ -19,17 +19,17 @@ import { FileTrash, FileTrashCleanupResult, FileTrashRecordMetadata, FileTrashReimport fs from 'fs/promises'import path from 'node:path'import { Stats } from 'node:fs'+import type { FilesTrashRetentionConfig } from '../files.config'@Injectable()export class FilesTrashRetention {private readonly logger = new Logger(FilesTrashRetention.name)- private readonly retentionDays = configuration.applications.files.trashRetentionDays+ private readonly trashRetention: FilesTrashRetentionConfig = configuration.applications.files.trashRetentionprivate readonly fileBatchSize = 1000constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}async indexAndCleanTrash(userIds?: number[], spaceIds?: number[]): Promise<void> {- if (this.retentionDays === false) returnfor (const p of await this.allPaths(userIds, spaceIds)) {const tableName = this.getTableName(genIndexingKey(p.id, p.type))if (!(await this.createTable(tableName))) {@@ -69,7 +69,7 @@ export class FilesTrashRetention {this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - cleanup skipped because unseen records could not be deleted` })continue}- const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath)+ const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath, p.retentionDays)const totalErrorRecords = errorRecords + expiredRecords.errorRecordsif (scannedRecords === 0 && indexedRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {@@ -126,8 +126,8 @@ export class FilesTrashRetention {private async allPaths(userIds?: number[], spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {const hasNoFilters = userIds === undefined && spaceIds === undefined- const includeUsers = hasNoFilters || !!userIds?.length- const includeSpaces = hasNoFilters || !!spaceIds?.length+ const includeUsers = this.trashRetention.users !== false && (hasNoFilters || !!userIds?.length)+ const includeSpaces = this.trashRetention.spaces !== false && (hasNoFilters || !!spaceIds?.length)const [userPaths, spacePaths] = await Promise.all([includeUsers ? this.userPaths(userIds) : [], includeSpaces ? this.spacePaths(spaceIds) : []])@@ -135,6 +135,8 @@ export class FilesTrashRetention {}private async userPaths(userIds?: number[]): Promise<FileParseTrashRetentionPath[]> {+ if (this.trashRetention.users === false) return []+ const retentionDays = this.trashRetention.usersconst paths: FileParseTrashRetentionPath[] = []for (const user of await this.db.select({@@ -148,12 +150,14 @@ export class FilesTrashRetention {this.logger.warn({ tag: this.userPaths.name, msg: `user trash path does not exist : ${userTrashPath}` })continue}- paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath })+ paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath, retentionDays })}return paths}private async spacePaths(spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {+ if (this.trashRetention.spaces === false) return []+ const retentionDays = this.trashRetention.spacesconst paths: FileParseTrashRetentionPath[] = []for (const space of await this.db.select({@@ -167,7 +171,7 @@ export class FilesTrashRetention {this.logger.warn({ tag: this.spacePaths.name, msg: `space trash path does not exist : ${spaceTrashPath}` })continue}- paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath })+ paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath, retentionDays })}return paths}@@ -279,13 +283,9 @@ export class FilesTrashRetention {return null}- private async cleanupExpiredRecords(tableName: string, realBasePath: string): Promise<FileTrashCleanupResult> {- if (this.retentionDays === false) {- return { deletedRecords: 0, errorRecords: 0 }- }-- const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false)- const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true)+ private async cleanupExpiredRecords(tableName: string, realBasePath: string, retentionDays: number): Promise<FileTrashCleanupResult> {+ const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false, retentionDays)+ const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true, retentionDays)return {deletedRecords: fileResult.deletedRecords + dirResult.deletedRecords,@@ -293,14 +293,19 @@ export class FilesTrashRetention {}}- private async cleanupExpiredRecordsByType(tableName: string, realBasePath: string, isDir: boolean): Promise<FileTrashCleanupResult> {+ private async cleanupExpiredRecordsByType(+ tableName: string,+ realBasePath: string,+ isDir: boolean,+ retentionDays: number+ ): Promise<FileTrashCleanupResult> {// Process fixed-size pages. Filesystem failures are kept in DB and retried on a later run.// They are ignored only for the current run so later eligible records are not blocked.const result: FileTrashCleanupResult = { deletedRecords: 0, errorRecords: 0 }const ignoredIds: number[] = []while (true) {- const expiredRecords = await this.getExpiredRecords(tableName, isDir, [...ignoredIds])+ const expiredRecords = await this.getExpiredRecords(tableName, isDir, retentionDays, [...ignoredIds])if (!expiredRecords.length) {break}@@ -335,10 +340,7 @@ export class FilesTrashRetention {return result}- private async getExpiredRecords(tableName: string, isDir: boolean, ignoredIds: number[] = []): Promise<FileTrash[]> {- if (this.retentionDays === false) {- return []- }+ private async getExpiredRecords(tableName: string, isDir: boolean, retentionDays: number, ignoredIds: number[] = []): Promise<FileTrash[]> {const ignoredRecordsFilter = ignoredIds.length ? sql`AND trash_record.id NOT IN (${sql.raw(ignoredIds.join(','))})` : sql``const escapedRecordPath = sql`REPLACE(REPLACE(REPLACE(trash_record.path, '=', '=='), '%', '=%'), '_', '=_')`// Directories are eligible only after their indexed children are gone.@@ -355,7 +357,7 @@ export class FilesTrashRetention {const req = sql`SELECT trash_record.id, trash_record.path, trash_record.isDir, trash_record.deletedAtFROM ${sql.raw(tableName)} AS trash_record- WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${this.retentionDays} DAY)+ WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${retentionDays} DAY)AND trash_record.isDir = ${isDir}${ignoredRecordsFilter}${emptyDirectoryFilter}diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yamlindex 317b8fb0..16fc4a40 100755--- a/environment/environment.dist.yaml+++ b/environment/environment.dist.yaml@@ -337,10 +337,12 @@ applications:# Used when offline mode is enabled or to override default location# default: built-in OCR directorylanguagesPath:- # Trash retention in days. Set to a number of days to enable automatic trash cleanup.- # Set to `false` or `0` to disable it.- # default: false- trashRetentionDays: false+ # Trash retention in days. Set each value to a number of days to enable automatic trash cleanup.+ # Set to `false` or `0` to disable cleanup for that repository type.+ # default: users/spaces false+ trashRetention:+ users: false+ spaces: false# Show files starting with a dot in the file explorer# default: falseshowHiddenFiles: false