Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): trash retention support with indexing and cleanup
johavenc99033573bac
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.tsindex c46df553..6d6869b2 100644--- a/backend/src/applications/files/files.config.ts+++ b/backend/src/applications/files/files.config.ts@@ -1,4 +1,4 @@-import { Type } from 'class-transformer'+import { Transform, Type } from 'class-transformer'import {ArrayNotEmpty,IsArray,@@ -8,6 +8,7 @@ import {IsNotEmptyObject,IsOptional,IsString,+ Min,ValidateIf,ValidateNested} from 'class-validator'@@ -68,6 +69,12 @@ export class FilesConfig {@Type(() => FilesContentIndexingConfig)contentIndexing: FilesContentIndexingConfig = new FilesContentIndexingConfig()+ @Transform(({ value }) => (value === 0 ? false : value))+ @ValidateIf((o: FilesConfig) => o.trashRetentionDays !== false)+ @IsInt()+ @Min(1)+ trashRetentionDays: number | false = false+@IsBoolean()showHiddenFiles: boolean = falsediff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex b30dfb35..a83606dd 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -18,7 +18,7 @@ import { FilesSearchManager } from './services/files-search-manager.service'import { FilesTasksManager } from './services/files-tasks-manager.service'import { FilesEventManager } from './services/files-event-manager.service'import { FilesQuotaManager } from './services/files-quota-manager.service'-import { FilesTrashManagerService } from './services/files-trash-manager.service'+import { FilesTrashRetention } from './services/files-trash-retention.service'@Module({imports: [@@ -39,7 +39,8 @@ import { FilesTrashManagerService } from './services/files-trash-manager.service{ provide: FilesContentStore, useClass: FilesContentStoreMySQL },FilesSearchManager,FilesEventManager,- FilesQuotaManager+ FilesQuotaManager,+ FilesTrashRetention],exports: [FilesManager, FilesQueries, FilesLockManager, FilesQuotaManager, FilesMethods, FilesRecents]})diff --git a/backend/src/applications/files/interfaces/file-parse-index.ts b/backend/src/applications/files/interfaces/file-parse-index.tsindex ff765798..c2501e7e 100644--- a/backend/src/applications/files/interfaces/file-parse-index.ts+++ b/backend/src/applications/files/interfaces/file-parse-index.ts@@ -1,4 +1,3 @@-import type { FileTrash } from '../schemas/file-trash.interface'import type { FILE_REPOSITORY } from '../constants/operations'export interface FileParseContext {@@ -28,6 +27,4 @@ export interface FileParseTrashRetentionPath {export interface FileTrashRetentionIndexContext {tableName: stringregexBasePath: RegExp- db: Map<FileTrash['id'], { name: FileTrash['name']; path: FileTrash['path']; deletedAt: FileTrash['deletedAt'] }>- fs: Set<number>}diff --git a/backend/src/applications/files/schemas/file-trash.interface.ts b/backend/src/applications/files/schemas/file-trash.interface.tsnew file mode 100644index 00000000..202fda2d--- /dev/null+++ b/backend/src/applications/files/schemas/file-trash.interface.ts@@ -0,0 +1,15 @@+export interface FileTrash {+ id: number+ path: string+ isDir: boolean+ deletedAt?: Date+}++export interface FileTrashCleanupResult {+ deletedRecords: number+ errorRecords: number+}++export type FileTrashRecordMetadata = Pick<FileTrash, 'path' | 'isDir'>++export type FileTrashRecordMetadataMap = Map<FileTrash['id'], FileTrashRecordMetadata>diff --git a/backend/src/applications/files/schemas/files-trash.schema.ts b/backend/src/applications/files/schemas/files-trash.schema.tsnew file mode 100644index 00000000..f3d20dbd--- /dev/null+++ b/backend/src/applications/files/schemas/files-trash.schema.ts@@ -0,0 +1,19 @@+export const FILES_TRASH_TABLE_PREFIX = 'files_trash_' as const++// The utf8mb4_uca1400_ai_ci COLLATE is better for precision but slower+export function createTableFilesTrash(tableName: string): string {+ return `+ CREATE TABLE IF NOT EXISTS ${tableName}+ (+ id bigint unsigned NOT NULL,+ path varchar(4096) NOT NULL,+ isDir boolean NOT NULL,+ deletedAt date NOT NULL DEFAULT CURRENT_DATE,+ seen_run_id varchar(64),+ PRIMARY KEY (id),+ INDEX is_dir_deleted_at (isDir, deletedAt),+ INDEX path_prefix (path(768)),+ INDEX seen_run_id (seen_run_id)+ ) CHARACTER SET utf8mb4+ COLLATE utf8mb4_general_ci;`+}diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 4448021c..9fae077e 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -4,7 +4,7 @@ import { isNotNull, sql } from 'drizzle-orm'import { unionAll } from 'drizzle-orm/mysql-core'import fs from 'node:fs/promises'import path from 'node:path'-import { Cache } from '../../../infrastructure/cache/services/cache.service'+import { Cache } from '../../../infrastructure/cache/cache.service'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'import { getTablesWithFileIdColumn } from '../../../infrastructure/database/utils'@@ -19,18 +19,21 @@ import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'import { FilesContentIndexer } from './files-content-indexer.service'import { FilesTasksManager } from './files-tasks-manager.service'import { FilesQuotaManager } from './files-quota-manager.service'+import { FilesTrashRetention } from './files-trash-retention.service'@Injectable()export class FilesScheduler {private readonly logger = new Logger(FilesScheduler.name)private isQuotaUpdateIsRunning = falseprivate isQuotaUpdateEntriesIsRunning = false+ private isTrashCleanupRunning = falseconstructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,private readonly cache: Cache,private readonly filesContentIndexer: FilesContentIndexer,- private readonly filesQuotaManager: FilesQuotaManager+ private readonly filesQuotaManager: FilesQuotaManager,+ private readonly filesTrashRetention: FilesTrashRetention) {}@Timeout(10_000)@@ -40,6 +43,7 @@ export class FilesScheduler {await this.cleanupInterruptedTasks()await this.clearRecentFiles()await this.updateQuotas()+ await this.cleanupTrashFiles()} catch (e) {this.logger.error({ tag: this.onStartup.name, msg: `${e}` })}@@ -73,6 +77,19 @@ export class FilesScheduler {}}+ @Cron(CronExpression.EVERY_DAY_AT_2AM)+ async cleanupTrashFiles() {+ if (this.isTrashCleanupRunning) return+ this.isTrashCleanupRunning = true+ try {+ await this.filesTrashRetention.indexAndCleanTrash()+ } catch (e) {+ this.logger.error({ tag: this.cleanupTrashFiles.name, msg: `${e}` })+ } finally {+ this.isTrashCleanupRunning = false+ }+ }+@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)async cleanupUserTaskFiles(): Promise<void> {this.logger.log({ tag: this.cleanupUserTaskFiles.name, msg: `START` })@@ -138,7 +155,7 @@ export class FilesScheduler {async indexContentFiles(): Promise<void> {// queue a full content indexing request, it will be consumed by the minute schedulerif (await this.filesContentIndexer.requestFullIndexing()) {- this.logger.log({ tag: this.indexContentFiles.name, msg: `PENDING` })+ this.logger.log({ tag: this.indexContentFiles.name, msg: 'REQUESTED' })}}diff --git a/backend/src/applications/files/services/files-trash-retention.service.spec.ts b/backend/src/applications/files/services/files-trash-retention.service.spec.tsnew file mode 100644index 00000000..ec839628--- /dev/null+++ b/backend/src/applications/files/services/files-trash-retention.service.spec.ts@@ -0,0 +1,214 @@+import { Test, TestingModule } from '@nestjs/testing'+import fs from 'node:fs/promises'+import os from 'node:os'+import path from 'node:path'+import { escapePath } from '../../../common/functions'+import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'+import { FILE_REPOSITORY } from '../constants/operations'+import { FileTrash } from '../schemas/file-trash.interface'+import { createTableFilesTrash } from '../schemas/files-trash.schema'+import { FilesTrashRetention } from './files-trash-retention.service'++describe(FilesTrashRetention.name, () => {+ let module: TestingModule+ let service: FilesTrashRetention+ let db: { execute: jest.Mock }++ const asyncGen = <T>(items: T[]) =>+ (async function* () {+ for (const item of items) {+ yield item+ }+ })()++ const sqlText = (query: any): string => {+ if (typeof query === 'string') return query+ if (Array.isArray(query)) return query.map(sqlText).join('')+ if (Array.isArray(query?.value)) return query.value.join('')+ if (Array.isArray(query?.queryChunks)) return query.queryChunks.map(sqlText).join('')+ return ''+ }++ const fileTrash = (id: number, overrides: Partial<FileTrash> = {}): FileTrash => ({+ id,+ path: `file-${id}.txt`,+ isDir: false,+ ...overrides+ })++ const trashContext = (trashPath: string) => ({+ tableName: 'files_trash_user_1',+ regexBasePath: new RegExp(`^/?${escapePath(trashPath)}/?`)+ })++ const collectParsedRecords = async (trashPath: string): Promise<FileTrash[]> => {+ const records: FileTrash[] = []+ for await (const record of (service as any).parseFiles(trashPath, trashContext(trashPath))) {+ records.push(record)+ }+ return records+ }++ const mockTrashPath = (files: FileTrash[] = []) => {+ jest.spyOn(service as any, 'allPaths').mockResolvedValueOnce([{ id: 9, type: FILE_REPOSITORY.USER, realPath: '/trash' }])+ jest.spyOn(service as any, 'createTable').mockResolvedValueOnce(true)+ jest.spyOn(service as any, 'parseFiles').mockReturnValueOnce(asyncGen(files))+ }++ beforeEach(async () => {+ db = { execute: jest.fn() }++ module = await Test.createTestingModule({+ providers: [FilesTrashRetention, { provide: DB_TOKEN_PROVIDER, useValue: db }]+ }).compile()++ module.useLogger(['fatal'])+ service = module.get<FilesTrashRetention>(FilesTrashRetention)+ })++ afterEach(async () => {+ jest.restoreAllMocks()+ jest.clearAllMocks()+ await module.close()+ })++ it('should create trash tables with a seen run id column', () => {+ const schema = createTableFilesTrash('files_trash_user_1')++ expect(schema).toContain('deletedAt date')+ expect(schema).toContain('INDEX is_dir_deleted_at (isDir, deletedAt)')+ expect(schema).toContain('seen_run_id varchar(64)')+ expect(schema).toContain('INDEX seen_run_id (seen_run_id)')+ })++ it('should process trash files by batches of 1000 and delete unseen records', async () => {+ ;(service as any).retentionDays = 30+ const files = Array.from({ length: 1001 }, (_, i) => fileTrash(i + 1))+ mockTrashPath(files)+ const indexTrashBatchSpy = jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValue({ indexedRecords: 0, errorRecords: 0 })+ const deleteUnseenRecordsSpy = jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(2)+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords').mockResolvedValueOnce({ deletedRecords: 3, errorRecords: 0 })++ await service.indexAndCleanTrash()++ expect(indexTrashBatchSpy).toHaveBeenCalledTimes(2)+ expect(indexTrashBatchSpy.mock.calls[0][2]).toHaveLength(1000)+ expect(indexTrashBatchSpy.mock.calls[1][2]).toHaveLength(1)+ expect(deleteUnseenRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', expect.any(String))+ expect(cleanupExpiredRecordsSpy).toHaveBeenCalledWith('files_trash_user_9', '/trash')+ })++ it('should skip expired cleanup when unseen records cannot be deleted', async () => {+ ;(service as any).retentionDays = 30+ mockTrashPath([fileTrash(1)])+ jest.spyOn(service as any, 'indexTrashBatch').mockResolvedValueOnce({ indexedRecords: 1, errorRecords: 0 })+ jest.spyOn(service as any, 'deleteUnseenRecords').mockResolvedValueOnce(null)+ const cleanupExpiredRecordsSpy = jest.spyOn(service as any, 'cleanupExpiredRecords')++ await service.indexAndCleanTrash()++ expect(cleanupExpiredRecordsSpy).not.toHaveBeenCalled()+ })++ it('should parse directories, files and directory symlinks from trash paths', async () => {+ const trashPath = await fs.mkdtemp(path.join(os.tmpdir(), 'trash-retention-'))+ try {+ await fs.mkdir(path.join(trashPath, 'empty-dir'))+ await fs.mkdir(path.join(trashPath, 'nested-dir'))+ await fs.writeFile(path.join(trashPath, 'nested-dir', 'file.txt'), 'content')+ await fs.symlink(path.join(trashPath, 'nested-dir'), path.join(trashPath, 'directory-link'))+ const records = await collectParsedRecords(trashPath)++ expect(records).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ path: 'empty-dir', isDir: true }),+ expect.objectContaining({ path: 'nested-dir', isDir: true }),+ expect.objectContaining({ path: 'nested-dir/file.txt', isDir: false }),+ expect.objectContaining({ path: 'directory-link', isDir: false })+ ])+ )+ } finally {+ await fs.rm(trashPath, { recursive: true, force: true })+ }+ })++ it('should insert changed records and mark unchanged records seen inside a batch', async () => {+ const runId = 'run-1'+ jest.spyOn(service as any, 'getRecordMetadataByIds').mockResolvedValueOnce(+ new Map([+ [1, { path: 'file-1.txt', isDir: false }],+ [2, { path: 'old.txt', isDir: false }]+ ])+ )+ const insertRecordSpy = jest+ .spyOn(service as any, 'insertRecord')+ .mockResolvedValueOnce(false)+ .mockResolvedValueOnce(true)+ const markRecordsSeenSpy = jest.spyOn(service as any, 'markRecordsSeen').mockResolvedValueOnce(true)++ await expect(+ (service as any).indexTrashBatch('files_trash_user_1', runId, [fileTrash(1), fileTrash(2, { path: 'file-2.txt' }), fileTrash(3)])+ ).resolves.toEqual({ indexedRecords: 2, errorRecords: 1 })++ expect(insertRecordSpy).toHaveBeenCalledTimes(2)+ expect(insertRecordSpy).toHaveBeenCalledWith('files_trash_user_1', expect.objectContaining({ id: 2 }), runId)+ expect(insertRecordSpy).toHaveBeenCalledWith('files_trash_user_1', expect.objectContaining({ id: 3 }), runId)+ expect(markRecordsSeenSpy).toHaveBeenCalledWith('files_trash_user_1', [1], runId)+ })++ it('should reset deletedAt when an existing record metadata changes', async () => {+ db.execute.mockResolvedValueOnce([{ affectedRows: 1 }])++ await expect((service as any).insertRecord('files_trash_user_1', fileTrash(2, { path: 'file-2.txt' }), 'run-1')).resolves.toBe(true)++ const query = sqlText(db.execute.mock.calls[0][0])+ expect(query).toContain('ON DUPLICATE KEY UPDATE')+ expect(query).toContain('deletedAt = CURRENT_DATE')+ })++ it('should delete expired records from filesystem and database', async () => {+ ;(service as any).retentionDays = 30+ const expiredRecords = [fileTrash(1), fileTrash(2, { path: 'nested' })]+ const getExpiredRecordsSpy = jest+ .spyOn(service as any, 'getExpiredRecords')+ .mockResolvedValueOnce(expiredRecords)+ .mockResolvedValueOnce([])+ .mockResolvedValueOnce([])+ jest+ .spyOn(service as any, 'deleteTrashRecordFile')+ .mockResolvedValueOnce(true)+ .mockResolvedValueOnce(false)+ const deleteRecordsByIdsSpy = jest.spyOn(service as any, 'deleteRecordsByIds').mockResolvedValueOnce(1)++ await expect((service as any).cleanupExpiredRecords('files_trash_user_1', '/trash')).resolves.toEqual({+ deletedRecords: 1,+ errorRecords: 1+ })++ expect(getExpiredRecordsSpy).toHaveBeenCalledTimes(3)+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(1, 'files_trash_user_1', false, [])+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(2, 'files_trash_user_1', false, [2])+ expect(getExpiredRecordsSpy).toHaveBeenNthCalledWith(3, 'files_trash_user_1', true, [])+ expect(deleteRecordsByIdsSpy).toHaveBeenCalledWith('files_trash_user_1', [1])+ })++ it('should only query expired directories without indexed descendants', async () => {+ ;(service as any).retentionDays = 30+ db.execute.mockResolvedValueOnce([[]])++ await expect((service as any).getExpiredRecords('files_trash_user_1', true)).resolves.toEqual([])++ const query = sqlText(db.execute.mock.calls[0][0])+ expect(query).toContain('NOT EXISTS')+ expect(query).toContain('FROM files_trash_user_1 AS child')+ expect(query).toContain('child.path LIKE CONCAT')+ expect(query).toContain("ESCAPE '='")+ expect(query).toContain('ORDER BY trash_record.deletedAt, trash_record.id')+ })++ it('should resolve trash record paths inside the trash root only', () => {+ expect((service as any).resolveTrashRecordPath('/trash', fileTrash(1, { path: 'root.txt' }))).toBe('/trash/root.txt')+ expect((service as any).resolveTrashRecordPath('/trash', fileTrash(1, { path: 'nested/old.txt' }))).toBe('/trash/nested/old.txt')+ expect(() => (service as any).resolveTrashRecordPath('/trash', fileTrash(2, { path: '../outside.txt' }))).toThrow('outside trash path')+ })+})diff --git a/backend/src/applications/files/services/files-trash-retention.service.ts b/backend/src/applications/files/services/files-trash-retention.service.tsnew file mode 100644index 00000000..b22ed059--- /dev/null+++ b/backend/src/applications/files/services/files-trash-retention.service.ts@@ -0,0 +1,408 @@+import { Inject, Injectable, Logger } from '@nestjs/common'+import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'+import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'+import { FILE_REPOSITORY } from '../constants/operations'+import { users } from '../../users/schemas/users.schema'+import { and, eq, inArray, lte, sql } from 'drizzle-orm'+import { USER_ROLE } from '../../users/constants/user'+import { UserModel } from '../../users/models/user.model'+import { isPathExists, removeFiles } from '../utils/files'+import { spaces } from '../../spaces/schemas/spaces.schema'+import { SpaceModel } from '../../spaces/models/space.model'+import { configuration } from '../../../configuration/config.environment'+import { genIndexingKey, genRunId } from '../utils/indexing'+import { createTableFilesTrash, FILES_TRASH_TABLE_PREFIX } from '../schemas/files-trash.schema'+import { FileParseTrashRetentionPath, FileTrashRetentionIndexContext } from '../interfaces/file-parse-index'+import { escapePath } from '../../../common/functions'+import { MySqlQueryResult } from 'drizzle-orm/mysql2'+import { FileTrash, FileTrashCleanupResult, FileTrashRecordMetadata, FileTrashRecordMetadataMap } from '../schemas/file-trash.interface'+import fs from 'fs/promises'+import path from 'node:path'+import { Stats } from 'node:fs'++@Injectable()+export class FilesTrashRetention {+ private readonly logger = new Logger(FilesTrashRetention.name)+ private readonly retentionDays = configuration.applications.files.trashRetentionDays+ private readonly fileBatchSize = 1000++ constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}++ async indexAndCleanTrash(userIds?: number[], spaceIds?: number[]): Promise<void> {+ if (this.retentionDays === false) return+ for (const p of await this.allPaths(userIds, spaceIds)) {+ const tableName = this.getTableName(genIndexingKey(p.id, p.type))+ if (!(await this.createTable(tableName))) {+ continue+ }+ // this run id marks records still present on the filesystem; old run ids become stale DB rows after the scan.+ const runId = genRunId()+ const context: FileTrashRetentionIndexContext = {+ tableName: tableName,+ regexBasePath: new RegExp(`^/?${escapePath(p.realPath)}/?`)+ }+ let indexedRecords = 0+ let errorRecords = 0+ let scannedRecords = 0++ const processBatch = async (batch: FileTrash[]): Promise<void> => {+ const result = await this.indexTrashBatch(tableName, runId, batch)+ indexedRecords += result.indexedRecords+ errorRecords += result.errorRecords+ scannedRecords += batch.length+ }++ let batch: FileTrash[] = []+ for await (const fileTrash of this.parseFiles(p.realPath, context)) {+ batch.push(fileTrash)+ if (batch.length >= this.fileBatchSize) {+ await processBatch(batch)+ batch = []+ }+ }+ await processBatch(batch)++ // The scan is the source of truth for this disposable retention table:+ // records not seen in this run are removed before applying retention.+ const deletedRecords = await this.deleteUnseenRecords(tableName, runId)+ if (deletedRecords === null) {+ this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - cleanup skipped because unseen records could not be deleted` })+ continue+ }+ const expiredRecords = await this.cleanupExpiredRecords(tableName, p.realPath)+ const totalErrorRecords = errorRecords + expiredRecords.errorRecords++ if (scannedRecords === 0 && indexedRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {+ this.dropTable(tableName).catch((e: Error) =>+ this.logger.error({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - unable to drop table : ${e}` })+ )+ this.logger.verbose({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - no data, index not stored` })+ } else if (indexedRecords === 0 && totalErrorRecords === 0 && deletedRecords === 0 && expiredRecords.deletedRecords === 0) {+ this.logger.verbose({ tag: this.indexAndCleanTrash.name, msg: `${tableName} - no new data` })+ } else {+ this.logger.log({+ tag: this.indexAndCleanTrash.name,+ msg: `${tableName} - indexed: ${indexedRecords - errorRecords}, deleted: ${deletedRecords}, expired: ${expiredRecords.deletedRecords}, errors: ${totalErrorRecords}`+ })+ }+ }+ }++ private async indexTrashBatch(tableName: string, runId: string, batch: FileTrash[]): Promise<{ indexedRecords: number; errorRecords: number }> {+ if (!batch.length) {+ return { indexedRecords: 0, errorRecords: 0 }+ }++ const dbRecords = await this.getRecordMetadataByIds(+ tableName,+ batch.map((f) => f.id)+ )+ const seenRecordIds: number[] = []+ let indexedRecords = 0+ let errorRecords = 0++ for (const fileTrash of batch) {+ const dbRecord = dbRecords.get(fileTrash.id)+ if (dbRecord && dbRecord.path === fileTrash.path && dbRecord.isDir === fileTrash.isDir) {+ // unchanged records are not upserted, so they are marked as seen in bulk after the batch.+ seenRecordIds.push(fileTrash.id)+ continue+ }++ if ((await this.insertRecord(tableName, fileTrash, runId)) === false) {+ // A failed refresh may leave an old path/isDir in DB; do not mark it seen.+ // `deleteUnseenRecords` will discard the stale row instead of retaining unsafe metadata.+ errorRecords+++ }+ indexedRecords+++ }++ if (!(await this.markRecordsSeen(tableName, seenRecordIds, runId))) {+ throw new Error(`${tableName} - unable to mark records as seen`)+ }++ return { indexedRecords, errorRecords }+ }++ private async allPaths(userIds?: number[], spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {+ const hasNoFilters = userIds === undefined && spaceIds === undefined+ const includeUsers = hasNoFilters || !!userIds?.length+ const includeSpaces = hasNoFilters || !!spaceIds?.length++ const [userPaths, spacePaths] = await Promise.all([includeUsers ? this.userPaths(userIds) : [], includeSpaces ? this.spacePaths(spaceIds) : []])++ return [...userPaths, ...spacePaths]+ }++ private async userPaths(userIds?: number[]): Promise<FileParseTrashRetentionPath[]> {+ const paths: FileParseTrashRetentionPath[] = []+ for (const user of await this.db+ .select({+ id: users.id,+ login: users.login+ })+ .from(users)+ .where(and(...[lte(users.role, USER_ROLE.USER), ...(userIds ? [inArray(users.id, userIds)] : [])]))) {+ const userTrashPath = UserModel.getTrashPath(user.login)+ if (!(await isPathExists(userTrashPath))) {+ this.logger.warn({ tag: this.userPaths.name, msg: `user trash path does not exist : ${userTrashPath}` })+ continue+ }+ paths.push({ id: user.id, type: FILE_REPOSITORY.USER, realPath: userTrashPath })+ }+ return paths+ }++ private async spacePaths(spaceIds?: number[]): Promise<FileParseTrashRetentionPath[]> {+ const paths: FileParseTrashRetentionPath[] = []+ for (const space of await this.db+ .select({+ id: spaces.id,+ alias: spaces.alias+ })+ .from(spaces)+ .where(and(eq(spaces.enabled, true), ...(spaceIds ? [inArray(spaces.id, spaceIds)] : [])))) {+ const spaceTrashPath = SpaceModel.getTrashPath(space.alias)+ if (!(await isPathExists(spaceTrashPath))) {+ this.logger.warn({ tag: this.spacePaths.name, msg: `space trash path does not exist : ${spaceTrashPath}` })+ continue+ }+ paths.push({ id: space.id, type: FILE_REPOSITORY.SPACE, realPath: spaceTrashPath })+ }+ return paths+ }++ private async *parseFiles(dir: string, context: FileTrashRetentionIndexContext): AsyncGenerator<FileTrash> {+ try {+ for (const entry of await fs.readdir(dir, { withFileTypes: true })) {+ const realPath = path.join(entry.parentPath, entry.name)+ const fileContent = await this.analyzeFile(realPath, context)+ if (fileContent !== null) {+ yield fileContent+ }+ if (entry.isDirectory()) {+ yield* this.parseFiles(realPath, context)+ }+ }+ } catch (e) {+ this.logger.warn({ tag: this.parseFiles.name, msg: `${context.tableName} - unable to parse: ${dir} - ${e}` })+ }+ }++ private async analyzeFile(realPath: string, context: FileTrashRetentionIndexContext): Promise<FileTrash | null> {+ let stats: Stats+ try {+ stats = await fs.lstat(realPath)+ } catch (e) {+ this.logger.warn({ tag: this.analyzeFile.name, msg: `unable to stats: ${realPath} - ${e}` })+ return null+ }+ const filePath = realPath.replace(context.regexBasePath, '')++ return {+ id: stats.ino,+ path: filePath,+ isDir: stats.isDirectory()+ } satisfies FileTrash+ }++ private getTableName(indexSuffix: string): string {+ return `${FILES_TRASH_TABLE_PREFIX}${indexSuffix}`+ }++ private async createTable(tableName: string): Promise<boolean> {+ try {+ await this.db.execute(createTableFilesTrash(tableName))+ return true+ } catch (e) {+ this.logger.error({ tag: this.createTable.name, msg: `${tableName} : ${e}` })+ return false+ }+ }++ private async dropTable(tableName: string): Promise<boolean> {+ try {+ await this.db.execute(sql`DROP TABLE IF EXISTS ${sql.raw(tableName)} `)+ return true+ } catch (e) {+ this.logger.error({ tag: this.dropTable.name, msg: `${tableName} : ${e}` })+ return false+ }+ }++ private async getRecordMetadataByIds(tableName: string, ids: number[]): Promise<FileTrashRecordMetadataMap> {+ if (!ids.length) {+ return new Map()+ }+ const [r]: { id: number; path: string; isDir: boolean }[][] = (await this.db.execute(+ sql`SELECT id, path, isDir FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`+ )) as MySqlQueryResult+ return new Map(r.map((row) => [row.id, { path: row.path, isDir: Boolean(row.isDir) }] satisfies [FileTrash['id'], FileTrashRecordMetadata]))+ }++ private async markRecordsSeen(tableName: string, ids: number[], runId: string): Promise<boolean> {+ if (!ids.length) return true+ try {+ await this.db.execute(sql`UPDATE ${sql.raw(tableName)} SET seen_run_id = ${runId} WHERE id IN (${sql.raw(ids.join(','))})`)+ return true+ } catch (e) {+ this.logger.error({ tag: this.markRecordsSeen.name, msg: `${tableName} : ${e}` })+ }+ return false+ }++ private async insertRecord(tableName: string, ft: FileTrash, runId: string): Promise<boolean> {+ try {+ await this.db.execute(sql`+ INSERT INTO ${sql.raw(tableName)} (id, path, isDir, seen_run_id)+ VALUES ${sql`(${ft.id}, ${ft.path}, ${ft.isDir}, ${runId})`}+ -- If an inode is reused or moved inside trash, restart retention for the new metadata.+ ON DUPLICATE KEY UPDATE path = VALUES(path),+ isDir = VALUES(isDir),+ deletedAt = CURRENT_DATE,+ seen_run_id = VALUES(seen_run_id)+ `)+ return true+ } catch (e) {+ this.logger.error({ tag: this.insertRecord.name, msg: `${tableName} : ${e}` })+ }+ return false+ }++ private async deleteUnseenRecords(tableName: string, runId: string): Promise<number | null> {+ try {+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE seen_run_id IS NULL OR seen_run_id <> ${runId}`)+ return r.affectedRows ?? 0+ } catch (e) {+ this.logger.error({ tag: this.deleteUnseenRecords.name, msg: `${tableName} : ${e}` })+ }+ return null+ }++ private async cleanupExpiredRecords(tableName: string, realBasePath: string): Promise<FileTrashCleanupResult> {+ if (this.retentionDays === false) {+ return { deletedRecords: 0, errorRecords: 0 }+ }++ const fileResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, false)+ const dirResult = await this.cleanupExpiredRecordsByType(tableName, realBasePath, true)++ return {+ deletedRecords: fileResult.deletedRecords + dirResult.deletedRecords,+ errorRecords: fileResult.errorRecords + dirResult.errorRecords+ }+ }++ private async cleanupExpiredRecordsByType(tableName: string, realBasePath: string, isDir: boolean): Promise<FileTrashCleanupResult> {+ // Process fixed-size pages. Filesystem failures are kept in DB and retried on a later run.+ // They are ignored only for the current run so later eligible records are not blocked.+ const result: FileTrashCleanupResult = { deletedRecords: 0, errorRecords: 0 }+ const ignoredIds: number[] = []++ while (true) {+ const expiredRecords = await this.getExpiredRecords(tableName, isDir, [...ignoredIds])+ if (!expiredRecords.length) {+ break+ }++ const deletedIds: number[] = []+ for (const record of expiredRecords) {+ if (await this.deleteTrashRecordFile(realBasePath, record)) {+ deletedIds.push(record.id)+ } else {+ result.errorRecords+++ ignoredIds.push(record.id)+ }+ }++ if (!deletedIds.length) {+ continue+ }++ const deletedRecords = await this.deleteRecordsByIds(tableName, deletedIds)+ if (deletedRecords === null) {+ result.errorRecords += deletedIds.length+ break+ }+ result.deletedRecords += deletedRecords++ if (deletedRecords < deletedIds.length) {+ result.errorRecords += deletedIds.length - deletedRecords+ break+ }+ }++ return result+ }++ private async getExpiredRecords(tableName: string, isDir: boolean, ignoredIds: number[] = []): Promise<FileTrash[]> {+ if (this.retentionDays === false) {+ return []+ }+ const ignoredRecordsFilter = ignoredIds.length ? sql`AND trash_record.id NOT IN (${sql.raw(ignoredIds.join(','))})` : sql``+ const escapedRecordPath = sql`REPLACE(REPLACE(REPLACE(trash_record.path, '=', '=='), '%', '=%'), '_', '=_')`+ // Directories are eligible only after their indexed children are gone.+ // Once selected, removeFiles can delete any remaining filesystem content recursively.+ const emptyDirectoryFilter = isDir+ ? sql`+ AND NOT EXISTS (SELECT 1+ FROM ${sql.raw(tableName)} AS child+ WHERE child.id <> trash_record.id+ AND child.path LIKE CONCAT(${escapedRecordPath}, '/%') ESCAPE '=')+ `+ : sql``+ // Keep the predicate and ordering aligned with the (isDir, deletedAt) index.+ const req = sql`+ SELECT trash_record.id, trash_record.path, trash_record.isDir, trash_record.deletedAt+ FROM ${sql.raw(tableName)} AS trash_record+ WHERE trash_record.deletedAt < DATE_SUB(CURRENT_DATE, INTERVAL ${this.retentionDays} DAY)+ AND trash_record.isDir = ${isDir}+ ${ignoredRecordsFilter}+ ${emptyDirectoryFilter}+ ORDER BY trash_record.deletedAt, trash_record.id+ LIMIT ${this.fileBatchSize}+ `+ const [r]: { id: number; path: string; isDir: boolean; deletedAt: Date }[][] = (await this.db.execute(req)) as MySqlQueryResult+ return r.map((row) => ({ id: row.id, path: row.path, isDir: Boolean(row.isDir), deletedAt: row.deletedAt }) satisfies FileTrash)+ }++ private async deleteRecordsByIds(tableName: string, ids: number[]): Promise<number | null> {+ if (!ids.length) {+ return 0+ }+ try {+ const [r] = await this.db.execute(sql`DELETE FROM ${sql.raw(tableName)} WHERE id IN (${sql.raw(ids.join(','))})`)+ if (r.affectedRows !== ids.length) {+ this.logger.warn({ tag: this.deleteRecordsByIds.name, msg: `${tableName} - deleted : ${r.affectedRows}/${ids.length}` })+ }+ return r.affectedRows ?? 0+ } catch (e) {+ this.logger.error({ tag: this.deleteRecordsByIds.name, msg: `${tableName} : ${e}` })+ }+ return null+ }++ private async deleteTrashRecordFile(realBasePath: string, ft: FileTrash): Promise<boolean> {+ // `removeFiles` is recursive and tolerant of missing paths; success means the DB row can go.+ try {+ const realPath = this.resolveTrashRecordPath(realBasePath, ft)+ await removeFiles(realPath)+ return true+ } catch (e) {+ this.logger.error({ tag: this.deleteTrashRecordFile.name, msg: `${realBasePath}/${ft.path} : ${e}` })+ }+ return false+ }++ private resolveTrashRecordPath(realBasePath: string, ft: Pick<FileTrash, 'path'>): string {+ const basePath = path.resolve(realBasePath)+ const realPath = path.resolve(basePath, ft.path)++ // DB paths are resolved against the trash root and must stay inside it before any filesystem deletion.+ if (realPath !== basePath && realPath.startsWith(`${basePath}${path.sep}`)) {+ return realPath+ }++ throw new Error(`${realPath} is outside trash path ${basePath}`)+ }+}diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yamlindex 242c06ac..2cc920a4 100755--- a/environment/environment.dist.yaml+++ b/environment/environment.dist.yaml@@ -336,6 +336,10 @@ applications:# Used when offline mode is enabled or to override default location# default: built-in OCR directorylanguagesPath:+ # Trash retention in days. Set to a number of days to enable automatic trash cleanup.+ # Set to `false` or `0` to disable it.+ # default: false+ trashRetentionDays: false# Show files starting with a dot in the file explorer# default: falseshowHiddenFiles: false