Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): split trash retention by repository type

johaven1c490eee902b
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.ts
index ab128da5..0abf7a10 100644
--- a/backend/src/applications/files/files.config.ts
+++ b/backend/src/applications/files/files.config.ts
@@ -48,6 +48,20 @@ export class FilesContentIndexingConfig {
ocr: FilesContentIndexingOCRConfig = new FilesContentIndexingOCRConfig()
}
+export class FilesTrashRetentionConfig {
+ @Transform(({ value }) => (value === 0 ? false : value))
+ @ValidateIf((o: FilesTrashRetentionConfig) => o.users !== false)
+ @IsInt()
+ @Min(1)
+ users: number | false = false
+
+ @Transform(({ value }) => (value === 0 ? false : value))
+ @ValidateIf((o: FilesTrashRetentionConfig) => o.spaces !== false)
+ @IsInt()
+ @Min(1)
+ spaces: number | false = false
+}
+
export class FilesConfig {
@IsNotEmpty()
@IsString()
@@ -73,11 +87,10 @@ export class FilesConfig {
@Type(() => FilesContentIndexingConfig)
contentIndexing: FilesContentIndexingConfig = new FilesContentIndexingConfig()
- @Transform(({ value }) => (value === 0 ? false : value))
- @ValidateIf((o: FilesConfig) => o.trashRetentionDays !== false)
- @IsInt()
- @Min(1)
- trashRetentionDays: number | false = false
+ @IsNotEmptyObject()
+ @ValidateNested()
+ @Type(() => FilesTrashRetentionConfig)
+ trashRetention: FilesTrashRetentionConfig = new FilesTrashRetentionConfig()
@IsBoolean()
showHiddenFiles: boolean = false
diff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.ts
index c2501e7e..7b5035bb 100644
--- a/backend/src/applications/files/interfaces/file-parse-index.ts
+++ b/backend/src/applications/files/interfaces/file-parse-index.ts
@@ -22,6 +22,7 @@ export interface FileParseTrashRetentionPath {
id: number
type: FILE_REPOSITORY
realPath: string
+ retentionDays: number
}
export interface FileTrashRetentionIndexContext {
diff --git a/backend/src/applications/files/services/files-trash-retention.service.spec.ts b/backend/src/applications/files/services/files-trash-retention.service.spec.ts
index ec839628..cadeab6b 100644
--- a/backend/src/applications/files/services/files-trash-retention.service.spec.ts
+++ b/backend/src/applications/files/services/files-trash-retention.service.spec.ts
@@ -49,8 +49,8 @@ describe(FilesTrashRetention.name, () => {
return records
}
- const mockTrashPath = (files: FileTrash[] = []) => {
- jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type: FILE_REPOSITORY.USER, realPath: '/trash' }])
+ const mockTrashPath = (files: FileTrash[] = [], type: FILE_REPOSITORY.USER | FILE_REPOSITORY.SPACE = FILE_REPOSITORY.USER, retentionDays = 30) => {
+ jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type, realPath: '/trash', retentionDays }])
jest.spyOn(service as any, 'createTable').mockResolvedValueOnce(true)
jest.spyOn(service as any, 'parseFiles').mockReturnValueOnce(asyncGen(files))
}
@@ -82,7 +82,6 @@ describe(FilesTrashRetention.name, () => {
})
it('should process trash files by batches of 1000 and delete unseen records', async () => {
- ;(service as any).retentionDays = 30
const files = Array.from({ length: 1001 }, (_, i) => fileTrash(i + 1))
mockTrashPath(files)
const indexTrashBatchSpy = jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValue({ indexedRecords: 0, errorRecords: 0 })
@@ -95,11 +94,21 @@ describe(FilesTrashRetention.name, () => {
expect(indexTrashBatchSpy.mock.calls[0][2]).toHaveLength(1000)
expect(indexTrashBatchSpy.mock.calls[1][2]).toHaveLength(1)
expect(deleteUnseenRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', expect.any(String))
- expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash')
+ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash', 30)
+ })
+
+ it('should use the trash retention attached to each repository path', async () => {
+ mockTrashPath([fileTrash(1)], FILE_REPOSITORY.SPACE, 14)
+ jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })
+ jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(0)
+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords').mockResolvedValueOnce({ deletedRecords: 0, errorRecords: 0 })
+
+ await service.indexAndCleanTrash()
+
+ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_space_9', '/trash', 14)
})
it('should skip expired cleanup when unseen records cannot be deleted', async () => {
- ;(service as any).retentionDays = 30
mockTrashPath([fileTrash(1)])
jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })
jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(null)
@@ -167,7 +176,6 @@ describe(FilesTrashRetention.name, () => {
})
it('should delete expired records from filesystem and database', async () => {
- ;(service as any).retentionDays = 30
const expiredRecords = [fileTrash(1), fileTrash(2, { path: 'nested' })]
const getExpiredRecordsSpy = jest
.spyOn(service as any, 'getExpiredRecords')
@@ -180,23 +188,22 @@ describe(FilesTrashRetention.name, () => {
.mockResolvedValueOnce(false)
const deleteRecordsByIdsSpy = jest.spyOn(service as any, 'deleteRecordsByIds').mockResolvedValueOnce(1)
- await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash')).resolves.toEqual({
+ await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash', 30)).resolves.toEqual({
deletedRecords: 1,
errorRecords: 1
})
expect(getExpiredRecordsSpy).toHaveBeenCalledTimes(3)
- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, [])
- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, [2])
- expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, [])
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, 30, [])
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, 30, [2])
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, 30, [])
expect(deleteRecordsByIdsSpy).toHaveBeenCalledWith('files_trash_user_1', [1])
})
it('should only query expired directories without indexed descendants', async () => {
- ;(service as any).retentionDays = 30
db.execute.mockResolvedValueOnce([[]])
- await expect((service as any).getExpiredRecords('files_trash_user_1', true)).resolves.toEqual([])
+ await expect((service as any).getExpiredRecords('files_trash_user_1', true, 30)).resolves.toEqual([])
const query = sqlText(db.execute.mock.calls[0][0])
expect(query).toContain('NOT EXISTS')
diff --git a/backend/src/applications/files/services/files-trash-retention.service.ts b/backend/src/applications/files/services/files-trash-retention.service.ts
index 4ce3a31f..6cc7c5c1 100644
--- a/backend/src/applications/files/services/files-trash-retention.service.ts
+++ b/backend/src/applications/files/services/files-trash-retention.service.ts
@@ -19,17 +19,17 @@ import { FileTrash, FileTrashCleanupResult, FileTrashRecordMetadata, FileTrashRe
import fs from 'fs/promises'
import path from 'node:path'
import { Stats } from 'node:fs'
+import type { FilesTrashRetentionConfig } from '../files.config'
@Injectable()
export class FilesTrashRetention {
private readonly logger = new Logger(FilesTrashRetention.name)
- private readonly retentionDays = configuration.applications.files.trashRetentionDays
+ private readonly trashRetention: FilesTrashRetentionConfig = configuration.applications.files.trashRetention
private readonly fileBatchSize = 1000
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
async indexAndCleanTrash(userIds?: number[], spaceIds?: number[]): Promise<void> {
- if (this.retentionDays === false) return
for (const p of await this.allPaths(userIds, spaceIds)) {
const tableName = this.getTableName(genIndexingKey(p.id, p.type))
if (!(await this.createTable(tableName))) {
@@ -69,7 +69,7 @@ export class FilesTrashRetention {
this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - cleanup skipped because unseen records could not be deleted` })
continue
}
- const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath)
+ const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath, p.retentionDays)
const totalErrorRecords = errorRecords + expiredRecords.errorRecords
if (scannedRecords === 0 && indexedRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {
@@ -126,8 +126,8 @@ export class FilesTrashRetention {
private async allPaths(userIds?: number[], spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
const hasNoFilters = userIds === undefined && spaceIds === undefined
- const includeUsers = hasNoFilters || !!userIds?.length
- const includeSpaces = hasNoFilters || !!spaceIds?.length
+ const includeUsers = this.trashRetention.users !== false && (hasNoFilters || !!userIds?.length)
+ const includeSpaces = this.trashRetention.spaces !== false && (hasNoFilters || !!spaceIds?.length)
const [userPaths, spacePaths] = await Promise.all([includeUsers ? this.userPaths(userIds) : [], includeSpaces ? this.spacePaths(spaceIds) : []])
@@ -135,6 +135,8 @@ export class FilesTrashRetention {
}
private async userPaths(userIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
+ if (this.trashRetention.users === false) return []
+ const retentionDays = this.trashRetention.users
const paths: FileParseTrashRetentionPath[] = []
for (const user of await this.db
.select({
@@ -148,12 +150,14 @@ export class FilesTrashRetention {
this.logger.warn({ tag: this.userPaths.name, msg: `user trash path does not exist : ${userTrashPath}` })
continue
}
- paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath })
+ paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath, retentionDays })
}
return paths
}
private async spacePaths(spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
+ if (this.trashRetention.spaces === false) return []
+ const retentionDays = this.trashRetention.spaces
const paths: FileParseTrashRetentionPath[] = []
for (const space of await this.db
.select({
@@ -167,7 +171,7 @@ export class FilesTrashRetention {
this.logger.warn({ tag: this.spacePaths.name, msg: `space trash path does not exist : ${spaceTrashPath}` })
continue
}
- paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath })
+ paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath, retentionDays })
}
return paths
}
@@ -279,13 +283,9 @@ export class FilesTrashRetention {
return null
}
- private async cleanupExpiredRecords(tableName: string, realBasePath: string): Promise<FileTrashCleanupResult> {
- if (this.retentionDays === false) {
- return { deletedRecords: 0, errorRecords: 0 }
- }
-
- const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false)
- const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true)
+ private async cleanupExpiredRecords(tableName: string, realBasePath: string, retentionDays: number): Promise<FileTrashCleanupResult> {
+ const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false, retentionDays)
+ const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true, retentionDays)
return {
deletedRecords: fileResult.deletedRecords + dirResult.deletedRecords,
@@ -293,14 +293,19 @@ export class FilesTrashRetention {
}
}
- private async cleanupExpiredRecordsByType(tableName: string, realBasePath: string, isDir: boolean): Promise<FileTrashCleanupResult> {
+ private async cleanupExpiredRecordsByType(
+ tableName: string,
+ realBasePath: string,
+ isDir: boolean,
+ retentionDays: number
+ ): Promise<FileTrashCleanupResult> {
// Process fixed-size pages. Filesystem failures are kept in DB and retried on a later run.
// They are ignored only for the current run so later eligible records are not blocked.
const result: FileTrashCleanupResult = { deletedRecords: 0, errorRecords: 0 }
const ignoredIds: number[] = []
while (true) {
- const expiredRecords = await this.getExpiredRecords(tableName, isDir, [...ignoredIds])
+ const expiredRecords = await this.getExpiredRecords(tableName, isDir, retentionDays, [...ignoredIds])
if (!expiredRecords.length) {
break
}
@@ -335,10 +340,7 @@ export class FilesTrashRetention {
return result
}
- private async getExpiredRecords(tableName: string, isDir: boolean, ignoredIds: number[] = []): Promise<FileTrash[]> {
- if (this.retentionDays === false) {
- return []
- }
+ private async getExpiredRecords(tableName: string, isDir: boolean, retentionDays: number, ignoredIds: number[] = []): Promise<FileTrash[]> {
const ignoredRecordsFilter = ignoredIds.length ? sql`AND trash_record.id NOT IN (${sql.raw(ignoredIds.join(','))})` : sql``
const escapedRecordPath = sql`REPLACE(REPLACE(REPLACE(trash_record.path, '=', '=='), '%', '=%'), '_', '=_')`
// Directories are eligible only after their indexed children are gone.
@@ -355,7 +357,7 @@ export class FilesTrashRetention {
const req = sql`
SELECT trash_record.id, trash_record.path, trash_record.isDir, trash_record.deletedAt
FROM ${sql.raw(tableName)} AS trash_record
- WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${this.retentionDays} DAY)
+ WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${retentionDays} DAY)
AND trash_record.isDir = ${isDir}
${ignoredRecordsFilter}
${emptyDirectoryFilter}
diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yaml
index 317b8fb0..16fc4a40 100755
--- a/environment/environment.dist.yaml
+++ b/environment/environment.dist.yaml
@@ -337,10 +337,12 @@ applications:
# Used when offline mode is enabled or to override default location
# default: built-in OCR directory
languagesPath:
- # Trash retention in days. Set to a number of days to enable automatic trash cleanup.
- # Set to `false` or `0` to disable it.
- # default: false
- trashRetentionDays: false
+ # Trash retention in days. Set each value to a number of days to enable automatic trash cleanup.
+ # Set to `false` or `0` to disable cleanup for that repository type.
+ # default: users/spaces false
+ trashRetention:
+ users: false
+ spaces: false
# Show files starting with a dot in the file explorer
# default: false
showHiddenFiles: false