Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): add daily cron job to clean up orphaned files and optimize table

johaven19b9e6ebee9f
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 41f509f2..0a0d1ce6 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -6,19 +6,22 @@
import { Inject, Injectable, Logger } from '@nestjs/common'
import { Cron, CronExpression, Timeout } from '@nestjs/schedule'
-import { sql } from 'drizzle-orm'
+import { isNotNull, sql } from 'drizzle-orm'
+import { unionAll } from 'drizzle-orm/mysql-core'
import fs from 'node:fs/promises'
import path from 'node:path'
import { configuration } from '../../../configuration/config.environment'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
+import { getTablesWithFileIdColumn } from '../../../infrastructure/database/utils'
import { USER_ROLE } from '../../users/constants/user'
import { UserModel } from '../../users/models/user.model'
import { users } from '../../users/schemas/users.schema'
import { CACHE_TASK_PREFIX } from '../constants/cache'
import { FileTask, FileTaskStatus } from '../models/file-task'
import { filesRecents } from '../schemas/files-recents.schema'
+import { files } from '../schemas/files.schema'
import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'
import { FilesContentManager } from './files-content-manager.service'
import { FilesTasksManager } from './files-tasks-manager.service'
@@ -126,4 +129,41 @@ export class FilesScheduler {
await this.filesContentManager.parseAndIndexAllFiles()
this.logger.log(`${this.indexContentFiles.name} - END`)
}
+
+ @Cron(CronExpression.EVERY_DAY_AT_4AM)
+ async deleteOrphanFiles() {
+ this.logger.log(`${this.deleteOrphanFiles.name} - START`)
+ const selects: any[] = []
+ for (const table of getTablesWithFileIdColumn()) {
+ selects.push(this.db.selectDistinct({ id: table.fileId }).from(table).where(isNotNull(table.fileId)))
+ }
+ if (selects.length === 0) {
+ this.logger.warn(`${this.deleteOrphanFiles.name} - no tables with fileId column`)
+ return
+ }
+ const unionSub = (selects.length === 1 ? selects[0] : unionAll(...(selects as [any, any, ...any[]]))).as('u')
+ // Debug
+ // const [preview] = (await this.db.execute(sql`
+ // SELECT f.id
+ // FROM ${files} AS f
+ // LEFT JOIN ${unionSub} ON ${unionSub.id} = f.id
+ // WHERE ${unionSub.id} IS NULL
+ // `)) as any[]
+ // console.log(preview.length, preview)
+ const deleteQuery = sql`
+ DELETE f
+ FROM ${files} AS f
+ LEFT JOIN ${unionSub} ON ${unionSub.id} = f.id
+ WHERE ${unionSub.id} IS NULL
+ `
+ try {
+ await this.db.transaction(async (tx) => {
+ const [r] = await tx.execute(deleteQuery)
+ this.logger.log(`${this.deleteOrphanFiles.name} - files: ${r.affectedRows}`)
+ })
+ } catch (e) {
+ this.logger.log(`${this.deleteOrphanFiles.name} - ${e}`)
+ }
+ this.logger.log(`${this.deleteOrphanFiles.name} - END`)
+ }
}
diff --git a/backend/src/applications/spaces/services/spaces-browser.service.ts b/backend/src/applications/spaces/services/spaces-browser.service.ts
index 0ed0b1a8..5e2e9cc0 100644
--- a/backend/src/applications/spaces/services/spaces-browser.service.ts
+++ b/backend/src/applications/spaces/services/spaces-browser.service.ts
@@ -20,7 +20,7 @@ import { UserModel } from '../../users/models/user.model'
import { LOCK_SCOPE } from '../../webdav/constants/webdav'
import { SpaceFiles } from '../interfaces/space-files.interface'
import { SpaceEnv } from '../models/space-env.model'
-import { realPathFromRootFile, IsRealPathIsDirAndExists } from '../utils/paths'
+import { IsRealPathIsDirAndExists, realPathFromRootFile } from '../utils/paths'
import { SpacesManager } from './spaces-manager.service'
import { SpacesQueries } from './spaces-queries.service'
diff --git a/backend/src/applications/spaces/services/spaces-scheduler.service.ts b/backend/src/applications/spaces/services/spaces-scheduler.service.ts
index ef27ef35..c4a975b5 100644
--- a/backend/src/applications/spaces/services/spaces-scheduler.service.ts
+++ b/backend/src/applications/spaces/services/spaces-scheduler.service.ts
@@ -44,7 +44,7 @@ export class SpacesScheduler {
this.logger.log('Update Share External Path Quotas - END')
}
- @Cron(CronExpression.EVERY_DAY_AT_4AM)
+ @Cron(CronExpression.EVERY_DAY_AT_2AM)
async deleteExpiredSpaces() {
/* Removes spaces that have been disabled for more than 30 days */
this.logger.log(`${this.deleteExpiredSpaces.name} - START`)
diff --git a/backend/src/infrastructure/database/utils.ts b/backend/src/infrastructure/database/utils.ts
index 6ded146b..ff20ad15 100644
--- a/backend/src/infrastructure/database/utils.ts
+++ b/backend/src/infrastructure/database/utils.ts
@@ -7,9 +7,10 @@
import { NestFastifyApplication } from '@nestjs/platform-fastify'
import { TestingModule } from '@nestjs/testing'
import { Column, eq, inArray, isNull, SQL, sql } from 'drizzle-orm'
-import { MySqlDatabase } from 'drizzle-orm/mysql-core'
+import { MySqlDatabase, MySqlTableWithColumns } from 'drizzle-orm/mysql-core'
import { MySqlQueryResult } from 'drizzle-orm/mysql2'
import { DB_TOKEN_PROVIDER } from './constants'
+import * as schema from './schema'
async function dbGetConnection(app: NestFastifyApplication | TestingModule, mode: 'pool' | 'client' = 'pool') {
const db: MySqlDatabase<any, any> = await app.resolve(DB_TOKEN_PROVIDER)
@@ -90,3 +91,9 @@ export function concatDistinctObjectsInArray(mustBeNotNull: Column, object: Reco
expr.mapWith(JSON.parse)
return expr
}
+
+export function getTablesWithFileIdColumn(): MySqlTableWithColumns<any>[] {
+ return Object.entries(schema)
+ .filter(([_table, columns]) => 'fileId' in columns)
+ .map(([_table, columns]) => columns as MySqlTableWithColumns<any>)
+}