Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): trash retention support with indexing and cleanup

johavenc99033573bac
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.ts
index c46df553..6d6869b2 100644
--- a/backend/src/applications/files/files.config.ts
+++ b/backend/src/applications/files/files.config.ts
@@ -1,4 +1,4 @@
-import { Type } from 'class-transformer'
+import { Transform, Type } from 'class-transformer'
import {
ArrayNotEmpty,
IsArray,
@@ -8,6 +8,7 @@ import {
IsNotEmptyObject,
IsOptional,
IsString,
+ Min,
ValidateIf,
ValidateNested
} from 'class-validator'
@@ -68,6 +69,12 @@ export class FilesConfig {
@Type(() => FilesContentIndexingConfig)
contentIndexing: FilesContentIndexingConfig = new FilesContentIndexingConfig()
+ @Transform(({ value }) => (value === 0 ? false : value))
+ @ValidateIf((o: FilesConfig) => o.trashRetentionDays !== false)
+ @IsInt()
+ @Min(1)
+ trashRetentionDays: number | false = false
+
@IsBoolean()
showHiddenFiles: boolean = false
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index b30dfb35..a83606dd 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -18,7 +18,7 @@ import { FilesSearchManager } from './services/files-search-manager.service'
import { FilesTasksManager } from './services/files-tasks-manager.service'
import { FilesEventManager } from './services/files-event-manager.service'
import { FilesQuotaManager } from './services/files-quota-manager.service'
-import { FilesTrashManagerService } from './services/files-trash-manager.service'
+import { FilesTrashRetention } from './services/files-trash-retention.service'
@Module({
imports: [
@@ -39,7 +39,8 @@ import { FilesTrashManagerService } from './services/files-trash-manager.service
{ provide: FilesContentStore, useClass: FilesContentStoreMySQL },
FilesSearchManager,
FilesEventManager,
- FilesQuotaManager
+ FilesQuotaManager,
+ FilesTrashRetention
],
exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]
})
diff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.ts
index ff765798..c2501e7e 100644
--- a/backend/src/applications/files/interfaces/file-parse-index.ts
+++ b/backend/src/applications/files/interfaces/file-parse-index.ts
@@ -1,4 +1,3 @@
-import type { FileTrash } from '../schemas/file-trash.interface'
import type { FILE_REPOSITORY } from '../constants/operations'
export interface FileParseContext {
@@ -28,6 +27,4 @@ export interface FileParseTrashRetentionPath {
export interface FileTrashRetentionIndexContext {
tableName: string
regexBasePath: RegExp
- db: Map<FileTrash['id'], { name: FileTrash['name']; path: FileTrash['path']; deletedAt: FileTrash['deletedAt'] }>
- fs: Set<number>
}
diff --git a/backend/src/applications/files/schemas/file-trash.interface.ts b/backend/src/applications/files/schemas/file-trash.interface.ts
new file mode 100644
index 00000000..202fda2d
--- /dev/null
+++ b/backend/src/applications/files/schemas/file-trash.interface.ts
@@ -0,0 +1,15 @@
+export interface FileTrash {
+ id: number
+ path: string
+ isDir: boolean
+ deletedAt?: Date
+}
+
+export interface FileTrashCleanupResult {
+ deletedRecords: number
+ errorRecords: number
+}
+
+export type FileTrashRecordMetadata = Pick<FileTrash, 'path' | 'isDir'>
+
+export type FileTrashRecordMetadataMap = Map<FileTrash['id'], FileTrashRecordMetadata>
diff --git a/backend/src/applications/files/schemas/files-trash.schema.ts b/backend/src/applications/files/schemas/files-trash.schema.ts
new file mode 100644
index 00000000..f3d20dbd
--- /dev/null
+++ b/backend/src/applications/files/schemas/files-trash.schema.ts
@@ -0,0 +1,19 @@
+export const FILES_TRASH_TABLE_PREFIX = 'files_trash_' as const
+
+// The utf8mb4_uca1400_ai_ci COLLATE is better for precision but slower
+export function createTableFilesTrash(tableName: string): string {
+ return `
+ CREATE TABLE IF NOT EXISTS ${tableName}
+ (
+ id bigint unsigned NOT NULL,
+ path varchar(4096) NOT NULL,
+ isDir boolean NOT NULL,
+ deletedAt date NOT NULL DEFAULT CURRENT_DATE,
+ seen_run_id varchar(64),
+ PRIMARY KEY (id),
+ INDEX is_dir_deleted_at (isDir, deletedAt),
+ INDEX path_prefix (path(768)),
+ INDEX seen_run_id (seen_run_id)
+ ) CHARACTER SET utf8mb4
+ COLLATE utf8mb4_general_ci;`
+}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 4448021c..9fae077e 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -4,7 +4,7 @@ import { isNotNull, sql } from 'drizzle-orm'
import { unionAll } from 'drizzle-orm/mysql-core'
import fs from 'node:fs/promises'
import path from 'node:path'
-import { Cache } from '../../../infrastructure/cache/services/cache.service'
+import { Cache } from '../../../infrastructure/cache/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
import { getTablesWithFileIdColumn } from '../../../infrastructure/database/utils'
@@ -19,18 +19,21 @@ import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'
import { FilesContentIndexer } from './files-content-indexer.service'
import { FilesTasksManager } from './files-tasks-manager.service'
import { FilesQuotaManager } from './files-quota-manager.service'
+import { FilesTrashRetention } from './files-trash-retention.service'
@Injectable()
export class FilesScheduler {
private readonly logger = new Logger(FilesScheduler.name)
private isQuotaUpdateIsRunning = false
private isQuotaUpdateEntriesIsRunning = false
+ private isTrashCleanupRunning = false
constructor(
@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
private readonly cache: Cache,
private readonly filesContentIndexer: FilesContentIndexer,
- private readonly filesQuotaManager: FilesQuotaManager
+ private readonly filesQuotaManager: FilesQuotaManager,
+ private readonly filesTrashRetention: FilesTrashRetention
) {}
@Timeout(10_000)
@@ -40,6 +43,7 @@ export class FilesScheduler {
await this.cleanupInterruptedTasks()
await this.clearRecentFiles()
await this.updateQuotas()
+ await this.cleanupTrashFiles()
} catch (e) {
this.logger.error({ tag: this.onStartup.name, msg: `${e}` })
}
@@ -73,6 +77,19 @@ export class FilesScheduler {
}
}
+ @Cron(CronExpression.EVERY_DAY_AT_2AM)
+ async cleanupTrashFiles() {
+ if (this.isTrashCleanupRunning) return
+ this.isTrashCleanupRunning = true
+ try {
+ await this.filesTrashRetention.indexAndCleanTrash()
+ } catch (e) {
+ this.logger.error({ tag: this.cleanupTrashFiles.name, msg: `${e}` })
+ } finally {
+ this.isTrashCleanupRunning = false
+ }
+ }
+
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async cleanupUserTaskFiles(): Promise<void> {
this.logger.log({ tag: this.cleanupUserTaskFiles.name, msg: `START` })
@@ -138,7 +155,7 @@ export class FilesScheduler {
async indexContentFiles(): Promise<void> {
// queue a full content indexing request, it will be consumed by the minute scheduler
if (await this.filesContentIndexer.requestFullIndexing()) {
- this.logger.log({ tag: this.indexContentFiles.name, msg: `PENDING` })
+ this.logger.log({ tag: this.indexContentFiles.name, msg: 'REQUESTED' })
}
}
diff --git a/backend/src/applications/files/services/files-trash-retention.service.spec.ts b/backend/src/applications/files/services/files-trash-retention.service.spec.ts
new file mode 100644
index 00000000..ec839628
--- /dev/null
+++ b/backend/src/applications/files/services/files-trash-retention.service.spec.ts
@@ -0,0 +1,214 @@
+import { Test, TestingModule } from '@nestjs/testing'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { escapePath } from '../../../common/functions'
+import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
+import { FILE_REPOSITORY } from '../constants/operations'
+import { FileTrash } from '../schemas/file-trash.interface'
+import { createTableFilesTrash } from '../schemas/files-trash.schema'
+import { FilesTrashRetention } from './files-trash-retention.service'
+
+describe(FilesTrashRetention.name, () => {
+ let module: TestingModule
+ let service: FilesTrashRetention
+ let db: { execute: jest.Mock }
+
+ const asyncGen = <T>(items: T[]) =>
+ (async function* () {
+ for (const item of items) {
+ yield item
+ }
+ })()
+
+ const sqlText = (query: any): string => {
+ if (typeof query === 'string') return query
+ if (Array.isArray(query)) return query.map(sqlText).join('')
+ if (Array.isArray(query?.value)) return query.value.join('')
+ if (Array.isArray(query?.queryChunks)) return query.queryChunks.map(sqlText).join('')
+ return ''
+ }
+
+ const fileTrash = (id: number, overrides: Partial<FileTrash> = {}): FileTrash => ({
+ id,
+ path: `file-${id}.txt`,
+ isDir: false,
+ ...overrides
+ })
+
+ const trashContext = (trashPath: string) => ({
+ tableName: 'files_trash_user_1',
+ regexBasePath: new RegExp(`^/?${escapePath(trashPath)}/?`)
+ })
+
+ const collectParsedRecords = async (trashPath: string): Promise<FileTrash[]> => {
+ const records: FileTrash[] = []
+ for await (const record of (service as any).parseFiles(trashPath, trashContext(trashPath))) {
+ records.push(record)
+ }
+ return records
+ }
+
+ const mockTrashPath = (files: FileTrash[] = []) => {
+ jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type: FILE_REPOSITORY.USER, realPath: '/trash' }])
+ jest.spyOn(service as any, 'createTable').mockResolvedValueOnce(true)
+ jest.spyOn(service as any, 'parseFiles').mockReturnValueOnce(asyncGen(files))
+ }
+
+ beforeEach(async () => {
+ db = { execute: jest.fn() }
+
+ module = await Test.createTestingModule({
+ providers: [FilesTrashRetention, { provide: DB_TOKEN_PROVIDER, useValue: db }]
+ }).compile()
+
+ module.useLogger(['fatal'])
+ service = module.get<FilesTrashRetention>(FilesTrashRetention)
+ })
+
+ afterEach(async () => {
+ jest.restoreAllMocks()
+ jest.clearAllMocks()
+ await module.close()
+ })
+
+ it('should create trash tables with a seen run id column', () => {
+ const schema = createTableFilesTrash('files_trash_user_1')
+
+ expect(schema).toContain('deletedAt date')
+ expect(schema).toContain('INDEX is_dir_deleted_at (isDir, deletedAt)')
+ expect(schema).toContain('seen_run_id varchar(64)')
+ expect(schema).toContain('INDEX seen_run_id (seen_run_id)')
+ })
+
+ it('should process trash files by batches of 1000 and delete unseen records', async () => {
+ ;(service as any).retentionDays = 30
+ const files = Array.from({ length: 1001 }, (_, i) => fileTrash(i + 1))
+ mockTrashPath(files)
+ const indexTrashBatchSpy = jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValue({ indexedRecords: 0, errorRecords: 0 })
+ const deleteUnseenRecordsSpy = jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(2)
+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords').mockResolvedValueOnce({ deletedRecords: 3, errorRecords: 0 })
+
+ await service.indexAndCleanTrash()
+
+ expect(indexTrashBatchSpy).toHaveBeenCalledTimes(2)
+ expect(indexTrashBatchSpy.mock.calls[0][2]).toHaveLength(1000)
+ expect(indexTrashBatchSpy.mock.calls[1][2]).toHaveLength(1)
+ expect(deleteUnseenRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', expect.any(String))
+ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash')
+ })
+
+ it('should skip expired cleanup when unseen records cannot be deleted', async () => {
+ ;(service as any).retentionDays = 30
+ mockTrashPath([fileTrash(1)])
+ jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })
+ jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(null)
+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords')
+
+ await service.indexAndCleanTrash()
+
+ expect(cleanupExpiredRecordsSpy).not.toHaveBeenCalled()
+ })
+
+ it('should parse directories, files and directory symlinks from trash paths', async () => {
+ const trashPath = await fs.mkdtemp(path.join(os.tmpdir(), 'trash-retention-'))
+ try {
+ await fs.mkdir(path.join(trashPath, 'empty-dir'))
+ await fs.mkdir(path.join(trashPath, 'nested-dir'))
+ await fs.writeFile(path.join(trashPath, 'nested-dir', 'file.txt'), 'content')
+ await fs.symlink(path.join(trashPath, 'nested-dir'), path.join(trashPath, 'directory-link'))
+ const records = await collectParsedRecords(trashPath)
+
+ expect(records).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ path: 'empty-dir', isDir: true }),
+ expect.objectContaining({ path: 'nested-dir', isDir: true }),
+ expect.objectContaining({ path: 'nested-dir/file.txt', isDir: false }),
+ expect.objectContaining({ path: 'directory-link', isDir: false })
+ ])
+ )
+ } finally {
+ await fs.rm(trashPath, { recursive: true, force: true })
+ }
+ })
+
+ it('should insert changed records and mark unchanged records seen inside a batch', async () => {
+ const runId = 'run-1'
+ jest.spyOn(service as any, 'getRecordMetadataByIds').mockResolvedValueOnce(
+ new Map([
+ [1, { path: 'file-1.txt', isDir: false }],
+ [2, { path: 'old.txt', isDir: false }]
+ ])
+ )
+ const insertRecordSpy = jest
+ .spyOn(service as any, 'insertRecord')
+ .mockResolvedValueOnce(false)
+ .mockResolvedValueOnce(true)
+ const markRecordsSeenSpy = jest.spyOn(service as any, 'markRecordsSeen').mockResolvedValueOnce(true)
+
+ await expect(
+ (service as any).indexTrashBatch('files_trash_user_1', runId, [fileTrash(1), fileTrash(2, { path: 'file-2.txt' }), fileTrash(3)])
+ ).resolves.toEqual({ indexedRecords: 2, errorRecords: 1 })
+
+ expect(insertRecordSpy).toHaveBeenCalledTimes(2)
+ expect(insertRecordSpy).toHaveBeenCalledWith('files_trash_user_1', expect.objectContaining({ id: 2 }), runId)
+ expect(insertRecordSpy).toHaveBeenCalledWith('files_trash_user_1', expect.objectContaining({ id: 3 }), runId)
+ expect(markRecordsSeenSpy).toHaveBeenCalledWith('files_trash_user_1', [1], runId)
+ })
+
+ it('should reset deletedAt when an existing record metadata changes', async () => {
+ db.execute.mockResolvedValueOnce([{ affectedRows: 1 }])
+
+ await expect((service as any).insertRecord('files_trash_user_1', fileTrash(2, { path: 'file-2.txt' }), 'run-1')).resolves.toBe(true)
+
+ const query = sqlText(db.execute.mock.calls[0][0])
+ expect(query).toContain('ON DUPLICATE KEY UPDATE')
+ expect(query).toContain('deletedAt = CURRENT_DATE')
+ })
+
+ it('should delete expired records from filesystem and database', async () => {
+ ;(service as any).retentionDays = 30
+ const expiredRecords = [fileTrash(1), fileTrash(2, { path: 'nested' })]
+ const getExpiredRecordsSpy = jest
+ .spyOn(service as any, 'getExpiredRecords')
+ .mockResolvedValueOnce(expiredRecords)
+ .mockResolvedValueOnce([])
+ .mockResolvedValueOnce([])
+ jest
+ .spyOn(service as any, 'deleteTrashRecordFile')
+ .mockResolvedValueOnce(true)
+ .mockResolvedValueOnce(false)
+ const deleteRecordsByIdsSpy = jest.spyOn(service as any, 'deleteRecordsByIds').mockResolvedValueOnce(1)
+
+ await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash')).resolves.toEqual({
+ deletedRecords: 1,
+ errorRecords: 1
+ })
+
+ expect(getExpiredRecordsSpy).toHaveBeenCalledTimes(3)
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, [])
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, [2])
+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, [])
+ expect(deleteRecordsByIdsSpy).toHaveBeenCalledWith('files_trash_user_1', [1])
+ })
+
+ it('should only query expired directories without indexed descendants', async () => {
+ ;(service as any).retentionDays = 30
+ db.execute.mockResolvedValueOnce([[]])
+
+ await expect((service as any).getExpiredRecords('files_trash_user_1', true)).resolves.toEqual([])
+
+ const query = sqlText(db.execute.mock.calls[0][0])
+ expect(query).toContain('NOT EXISTS')
+ expect(query).toContain('FROM files_trash_user_1 AS child')
+ expect(query).toContain('child.path LIKE CONCAT')
+ expect(query).toContain("ESCAPE '='")
+ expect(query).toContain('ORDER BY trash_record.deletedAt, trash_record.id')
+ })
+
+ it('should resolve trash record paths inside the trash root only', () => {
+ expect((service as any).resolveTrashRecordPath('/trash', fileTrash(1, { path: 'root.txt' }))).toBe('/trash/root.txt')
+ expect((service as any).resolveTrashRecordPath('/trash', fileTrash(1, { path: 'nested/old.txt' }))).toBe('/trash/nested/old.txt')
+ expect(() => (service as any).resolveTrashRecordPath('/trash', fileTrash(2, { path: '../outside.txt' }))).toThrow('outside trash path')
+ })
+})
diff --git a/backend/src/applications/files/services/files-trash-retention.service.ts b/backend/src/applications/files/services/files-trash-retention.service.ts
new file mode 100644
index 00000000..b22ed059
--- /dev/null
+++ b/backend/src/applications/files/services/files-trash-retention.service.ts
@@ -0,0 +1,408 @@
+import { Inject, Injectable, Logger } from '@nestjs/common'
+import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
+import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
+import { FILE_REPOSITORY } from '../constants/operations'
+import { users } from '../../users/schemas/users.schema'
+import { and, eq, inArray, lte, sql } from 'drizzle-orm'
+import { USER_ROLE } from '../../users/constants/user'
+import { UserModel } from '../../users/models/user.model'
+import { isPathExists, removeFiles } from '../utils/files'
+import { spaces } from '../../spaces/schemas/spaces.schema'
+import { SpaceModel } from '../../spaces/models/space.model'
+import { configuration } from '../../../configuration/config.environment'
+import { genIndexingKey, genRunId } from '../utils/indexing'
+import { createTableFilesTrash, FILES_TRASH_TABLE_PREFIX } from '../schemas/files-trash.schema'
+import { FileParseTrashRetentionPath, FileTrashRetentionIndexContext } from '../interfaces/file-parse-index'
+import { escapePath } from '../../../common/functions'
+import { MySqlQueryResult } from 'drizzle-orm/mysql2'
+import { FileTrash, FileTrashCleanupResult, FileTrashRecordMetadata, FileTrashRecordMetadataMap } from '../schemas/file-trash.interface'
+import fs from 'fs/promises'
+import path from 'node:path'
+import { Stats } from 'node:fs'
+
+@Injectable()
+export class FilesTrashRetention {
+ private readonly logger = new Logger(FilesTrashRetention.name)
+ private readonly retentionDays = configuration.applications.files.trashRetentionDays
+ private readonly fileBatchSize = 1000
+
+ constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
+
+ async indexAndCleanTrash(userIds?: number[], spaceIds?: number[]): Promise<void> {
+ if (this.retentionDays === false) return
+ for (const p of await this.allPaths(userIds, spaceIds)) {
+ const tableName = this.getTableName(genIndexingKey(p.id, p.type))
+ if (!(await this.createTable(tableName))) {
+ continue
+ }
+ // this run id marks records still present on the filesystem; old run ids become stale DB rows after the scan.
+ const runId = genRunId()
+ const context: FileTrashRetentionIndexContext = {
+ tableName: tableName,
+ regexBasePath: new RegExp(`^/?${escapePath(p.realPath)}/?`)
+ }
+ let indexedRecords = 0
+ let errorRecords = 0
+ let scannedRecords = 0
+
+ const processBatch = async (batch: FileTrash[]): Promise<void> => {
+ const result = await this.indexTrashBatch(tableName, runId, batch)
+ indexedRecords += result.indexedRecords
+ errorRecords += result.errorRecords
+ scannedRecords += batch.length
+ }
+
+ let batch: FileTrash[] = []
+ for await (const fileTrash of this.parseFiles(p.realPath, context)) {
+ batch.push(fileTrash)
+ if (batch.length >= this.fileBatchSize) {
+ await processBatch(batch)
+ batch = []
+ }
+ }
+ await processBatch(batch)
+
+ // The scan is the source of truth for this disposable retention table:
+ // records not seen in this run are removed before applying retention.
+ const deletedRecords = await this.deleteUnseenRecords(tableName, runId)
+ if (deletedRecords === null) {
+ this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - cleanup skipped because unseen records could not be deleted` })
+ continue
+ }
+ const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath)
+ const totalErrorRecords = errorRecords + expiredRecords.errorRecords
+
+ if (scannedRecords === 0 && indexedRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {
+ this.dropTable(tableName).catch((e: Error) =>
+ this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - unable to drop table : ${e}` })
+ )
+ this.logger.verbose({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - no data, index not stored` })
+ } else if (indexedRecords === 0 && totalErrorRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {
+ this.logger.verbose({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - no new data` })
+ } else {
+ this.logger.log({
+ tag: this.indexAndCleanTrash.name,
+ msg: `${tableName} - indexed: ${indexedRecords - errorRecords}, deleted: ${deletedRecords}, expired: ${expiredRecords.deletedRecords}, errors: ${totalErrorRecords}`
+ })
+ }
+ }
+ }
+
+ private async indexTrashBatch(tableName: string, runId: string, batch: FileTrash[]): Promise<{ indexedRecords: number; errorRecords: number }> {
+ if (!batch.length) {
+ return { indexedRecords: 0, errorRecords: 0 }
+ }
+
+ const dbRecords = await this.getRecordMetadataByIds(
+ tableName,
+ batch.map((f) => f.id)
+ )
+ const seenRecordIds: number[] = []
+ let indexedRecords = 0
+ let errorRecords = 0
+
+ for (const fileTrash of batch) {
+ const dbRecord = dbRecords.get(fileTrash.id)
+ if (dbRecord && dbRecord.path === fileTrash.path && dbRecord.isDir === fileTrash.isDir) {
+ // unchanged records are not upserted, so they are marked as seen in bulk after the batch.
+ seenRecordIds.push(fileTrash.id)
+ continue
+ }
+
+ if ((await this.insertRecord(tableName, fileTrash, runId)) === false) {
+ // A failed refresh may leave an old path/isDir in DB; do not mark it seen.
+ // `deleteUnseenRecords` will discard the stale row instead of retaining unsafe metadata.
+ errorRecords++
+ }
+ indexedRecords++
+ }
+
+ if (!(await this.markRecordsSeen(tableName, seenRecordIds, runId))) {
+ throw new Error(`${tableName} - unable to mark records as seen`)
+ }
+
+ return { indexedRecords, errorRecords }
+ }
+
+ private async allPaths(userIds?: number[], spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
+ const hasNoFilters = userIds === undefined && spaceIds === undefined
+ const includeUsers = hasNoFilters || !!userIds?.length
+ const includeSpaces = hasNoFilters || !!spaceIds?.length
+
+ const [userPaths, spacePaths] = await Promise.all([includeUsers ? this.userPaths(userIds) : [], includeSpaces ? this.spacePaths(spaceIds) : []])
+
+ return [...userPaths, ...spacePaths]
+ }
+
+ private async userPaths(userIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
+ const paths: FileParseTrashRetentionPath[] = []
+ for (const user of await this.db
+ .select({
+ id: users.id,
+ login: users.login
+ })
+ .from(users)
+ .where(and(...[lte(users.role, USER_ROLE.USER), ...(userIds ? [inArray(users.id, userIds)] : [])]))) {
+ const userTrashPath = UserModel.getTrashPath(user.login)
+ if (!(await isPathExists(userTrashPath))) {
+ this.logger.warn({ tag: this.userPaths.name, msg: `user trash path does not exist : ${userTrashPath}` })
+ continue
+ }
+ paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath })
+ }
+ return paths
+ }
+
+ private async spacePaths(spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {
+ const paths: FileParseTrashRetentionPath[] = []
+ for (const space of await this.db
+ .select({
+ id: spaces.id,
+ alias: spaces.alias
+ })
+ .from(spaces)
+ .where(and(eq(spaces.enabled, true), ...(spaceIds ? [inArray(spaces.id, spaceIds)] : [])))) {
+ const spaceTrashPath = SpaceModel.getTrashPath(space.alias)
+ if (!(await isPathExists(spaceTrashPath))) {
+ this.logger.warn({ tag: this.spacePaths.name, msg: `space trash path does not exist : ${spaceTrashPath}` })
+ continue
+ }
+ paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath })
+ }
+ return paths
+ }
+
+ private async *parseFiles(dir: string, context: FileTrashRetentionIndexContext): AsyncGenerator<FileTrash> {
+ try {
+ for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
+ const realPath = path.join(entry.parentPath, entry.name)
+ const fileContent = await this.analyzeFile(realPath, context)
+ if (fileContent !== null) {
+ yield fileContent
+ }
+ if (entry.isDirectory()) {
+ yield* this.parseFiles(realPath, context)
+ }
+ }
+ } catch (e) {
+ this.logger.warn({ tag: this.parseFiles.name, msg: `${context.tableName} - unable to parse: ${dir} - ${e}` })
+ }
+ }
+
+ private async analyzeFile(realPath: string, context: FileTrashRetentionIndexContext): Promise<FileTrash | null> {
+ let stats: Stats
+ try {
+ stats = await fs.lstat(realPath)
+ } catch (e) {
+ this.logger.warn({ tag: this.analyzeFile.name, msg: `unable to stats: ${realPath} - ${e}` })
+ return null
+ }
+ const filePath = realPath.replace(context.regexBasePath, '')
+
+ return {
+ id: stats.ino,
+ path: filePath,
+ isDir: stats.isDirectory()
+ } satisfies FileTrash
+ }
+
+ private getTableName(indexSuffix: string): string {
+ return `${FILES_TRASH_TABLE_PREFIX}${indexSuffix}`
+ }
+
+ private async createTable(tableName: string): Promise<boolean> {
+ try {
+ await this.db.execute(createTableFilesTrash(tableName))
+ return true
+ } catch (e) {
+ this.logger.error({ tag: this.createTable.name, msg: `${tableName} : ${e}` })
+ return false
+ }
+ }
+
+ private async dropTable(tableName: string): Promise<boolean> {
+ try {
+ await this.db.execute(sql`DROP TABLE IF EXISTS ${sql.raw(tableName)} `)
+ return true
+ } catch (e) {
+ this.logger.error({ tag: this.dropTable.name, msg: `${tableName} : ${e}` })
+ return false
+ }
+ }
+
+ private async getRecordMetadataByIds(tableName: string, ids: number[]): Promise<FileTrashRecordMetadataMap> {
+ if (!ids.length) {
+ return new Map()
+ }
+ const [r]: { id: number; path: string; isDir: boolean }[][] = (await this.db.execute(
+ sql`SELECT id, path, isDir FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`
+ )) as MySqlQueryResult
+ return new Map(r.map((row) => [row.id, { path: row.path, isDir: Boolean(row.isDir) }] satisfies [FileTrash['id'], FileTrashRecordMetadata]))
+ }
+
+ private async markRecordsSeen(tableName: string, ids: number[], runId: string): Promise<boolean> {
+ if (!ids.length) return true
+ try {
+ await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${sql.raw(ids.join(','))})`)
+ return true
+ } catch (e) {
+ this.logger.error({ tag: this.markRecordsSeen.name, msg: `${tableName} : ${e}` })
+ }
+ return false
+ }
+
+ private async insertRecord(tableName: string, ft: FileTrash, runId: string): Promise<boolean> {
+ try {
+ await this.db.execute(sql`
+ INSERT INTO ${sql.raw(tableName)} (id, path, isDir, seen_run_id)
+ VALUES ${sql`(${ft.id}, ${ft.path}, ${ft.isDir}, ${runId})`}
+ -- If an inode is reused or moved inside trash, restart retention for the new metadata.
+ ON DUPLICATE KEY UPDATE path = VALUES(path),
+ isDir = VALUES(isDir),
+ deletedAt = CURRENT_DATE,
+ seen_run_id = VALUES(seen_run_id)
+ `)
+ return true
+ } catch (e) {
+ this.logger.error({ tag: this.insertRecord.name, msg: `${tableName} : ${e}` })
+ }
+ return false
+ }
+
+ private async deleteUnseenRecords(tableName: string, runId: string): Promise<number | null> {
+ try {
+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE seen_run_id IS NULL OR seen_run_id <> ${runId}`)
+ return r.affectedRows ?? 0
+ } catch (e) {
+ this.logger.error({ tag: this.deleteUnseenRecords.name, msg: `${tableName} : ${e}` })
+ }
+ return null
+ }
+
+ private async cleanupExpiredRecords(tableName: string, realBasePath: string): Promise<FileTrashCleanupResult> {
+ if (this.retentionDays === false) {
+ return { deletedRecords: 0, errorRecords: 0 }
+ }
+
+ const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false)
+ const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true)
+
+ return {
+ deletedRecords: fileResult.deletedRecords + dirResult.deletedRecords,
+ errorRecords: fileResult.errorRecords + dirResult.errorRecords
+ }
+ }
+
+ private async cleanupExpiredRecordsByType(tableName: string, realBasePath: string, isDir: boolean): Promise<FileTrashCleanupResult> {
+ // Process fixed-size pages. Filesystem failures are kept in DB and retried on a later run.
+ // They are ignored only for the current run so later eligible records are not blocked.
+ const result: FileTrashCleanupResult = { deletedRecords: 0, errorRecords: 0 }
+ const ignoredIds: number[] = []
+
+ while (true) {
+ const expiredRecords = await this.getExpiredRecords(tableName, isDir, [...ignoredIds])
+ if (!expiredRecords.length) {
+ break
+ }
+
+ const deletedIds: number[] = []
+ for (const record of expiredRecords) {
+ if (await this.deleteTrashRecordFile(realBasePath, record)) {
+ deletedIds.push(record.id)
+ } else {
+ result.errorRecords++
+ ignoredIds.push(record.id)
+ }
+ }
+
+ if (!deletedIds.length) {
+ continue
+ }
+
+ const deletedRecords = await this.deleteRecordsByIds(tableName, deletedIds)
+ if (deletedRecords === null) {
+ result.errorRecords += deletedIds.length
+ break
+ }
+ result.deletedRecords += deletedRecords
+
+ if (deletedRecords < deletedIds.length) {
+ result.errorRecords += deletedIds.length - deletedRecords
+ break
+ }
+ }
+
+ return result
+ }
+
+ private async getExpiredRecords(tableName: string, isDir: boolean, ignoredIds: number[] = []): Promise<FileTrash[]> {
+ if (this.retentionDays === false) {
+ return []
+ }
+ const ignoredRecordsFilter = ignoredIds.length ? sql`AND trash_record.id NOT IN (${sql.raw(ignoredIds.join(','))})` : sql``
+ const escapedRecordPath = sql`REPLACE(REPLACE(REPLACE(trash_record.path, '=', '=='), '%', '=%'), '_', '=_')`
+ // Directories are eligible only after their indexed children are gone.
+ // Once selected, removeFiles can delete any remaining filesystem content recursively.
+ const emptyDirectoryFilter = isDir
+ ? sql`
+ AND NOT EXISTS (SELECT 1
+ FROM ${sql.raw(tableName)} AS child
+ WHERE child.id <> trash_record.id
+ AND child.path LIKE CONCAT(${escapedRecordPath}, '/%') ESCAPE '=')
+ `
+ : sql``
+ // Keep the predicate and ordering aligned with the (isDir, deletedAt) index.
+ const req = sql`
+ SELECT trash_record.id, trash_record.path, trash_record.isDir, trash_record.deletedAt
+ FROM ${sql.raw(tableName)} AS trash_record
+ WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${this.retentionDays} DAY)
+ AND trash_record.isDir = ${isDir}
+ ${ignoredRecordsFilter}
+ ${emptyDirectoryFilter}
+ ORDER BY trash_record.deletedAt, trash_record.id
+ LIMIT ${this.fileBatchSize}
+ `
+ const [r]: { id: number; path: string; isDir: boolean; deletedAt: Date }[][] = (await this.db.execute(req)) as MySqlQueryResult
+ return r.map((row) => ({ id: row.id, path: row.path, isDir: Boolean(row.isDir), deletedAt: row.deletedAt }) satisfies FileTrash)
+ }
+
+ private async deleteRecordsByIds(tableName: string, ids: number[]): Promise<number | null> {
+ if (!ids.length) {
+ return 0
+ }
+ try {
+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`)
+ if (r.affectedRows !== ids.length) {
+ this.logger.warn({ tag: this.deleteRecordsByIds.name, msg: `${tableName} - deleted : ${r.affectedRows}/${ids.length}` })
+ }
+ return r.affectedRows ?? 0
+ } catch (e) {
+ this.logger.error({ tag: this.deleteRecordsByIds.name, msg: `${tableName} : ${e}` })
+ }
+ return null
+ }
+
+ private async deleteTrashRecordFile(realBasePath: string, ft: FileTrash): Promise<boolean> {
+ // `removeFiles` is recursive and tolerant of missing paths; success means the DB row can go.
+ try {
+ const realPath = this.resolveTrashRecordPath(realBasePath, ft)
+ await removeFiles(realPath)
+ return true
+ } catch (e) {
+ this.logger.error({ tag: this.deleteTrashRecordFile.name, msg: `${realBasePath}/${ft.path} : ${e}` })
+ }
+ return false
+ }
+
+ private resolveTrashRecordPath(realBasePath: string, ft: Pick<FileTrash, 'path'>): string {
+ const basePath = path.resolve(realBasePath)
+ const realPath = path.resolve(basePath, ft.path)
+
+ // DB paths are resolved against the trash root and must stay inside it before any filesystem deletion.
+ if (realPath !== basePath && realPath.startsWith(`${basePath}${path.sep}`)) {
+ return realPath
+ }
+
+ throw new Error(`${realPath} is outside trash path ${basePath}`)
+ }
+}
diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yaml
index 242c06ac..2cc920a4 100755
--- a/environment/environment.dist.yaml
+++ b/environment/environment.dist.yaml
@@ -336,6 +336,10 @@ applications:
# Used when offline mode is enabled or to override default location
# default: built-in OCR directory
languagesPath:
+ # Trash retention in days. Set to a number of days to enable automatic trash cleanup.
+ # Set to `false` or `0` to disable it.
+ # default: false
+ trashRetentionDays: false
# Show files starting with a dot in the file explorer
# default: false
showHiddenFiles: false