Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): add daily cron job to clean up orphaned files and optimize table
johaven19b9e6ebee9f
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 41f509f2..0a0d1ce6 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -6,19 +6,22 @@import { Inject, Injectable, Logger } from '@nestjs/common'import { Cron, CronExpression, Timeout } from '@nestjs/schedule'-import { sql } from 'drizzle-orm'+import { isNotNull, sql } from 'drizzle-orm'+import { unionAll } from 'drizzle-orm/mysql-core'import fs from 'node:fs/promises'import path from 'node:path'import { configuration } from '../../../configuration/config.environment'import { Cache } from '../../../infrastructure/cache/services/cache.service'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'+import { getTablesWithFileIdColumn } from '../../../infrastructure/database/utils'import { USER_ROLE } from '../../users/constants/user'import { UserModel } from '../../users/models/user.model'import { users } from '../../users/schemas/users.schema'import { CACHE_TASK_PREFIX } from '../constants/cache'import { FileTask, FileTaskStatus } from '../models/file-task'import { filesRecents } from '../schemas/files-recents.schema'+import { files } from '../schemas/files.schema'import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'import { FilesContentManager } from './files-content-manager.service'import { FilesTasksManager } from './files-tasks-manager.service'@@ -126,4 +129,41 @@ export class FilesScheduler {await this.filesContentManager.parseAndIndexAllFiles()this.logger.log(`${this.indexContentFiles.name} - END`)}++ @Cron(CronExpression.EVERY_DAY_AT_4AM)+ async deleteOrphanFiles() {+ this.logger.log(`${this.deleteOrphanFiles.name} - START`)+ const selects: any[] = []+ for (const table of getTablesWithFileIdColumn()) {+ selects.push(this.db.selectDistinct({ id: table.fileId }).from(table).where(isNotNull(table.fileId)))+ }+ if (selects.length === 0) {+ this.logger.warn(`${this.deleteOrphanFiles.name} - no tables with fileId column`)+ return+ }+ const unionSub = (selects.length === 1 ? selects[0] : unionAll(...(selects as [any, any, ...any[]]))).as('u')+ // Debug+ // const [preview] = (await this.db.execute(sql`+ // SELECT f.id+ // FROM ${files} AS f+ // LEFT JOIN ${unionSub} ON ${unionSub.id} = f.id+ // WHERE ${unionSub.id} IS NULL+ // `)) as any[]+ // console.log(preview.length, preview)+ const deleteQuery = sql`+ DELETE f+ FROM ${files} AS f+ LEFT JOIN ${unionSub} ON ${unionSub.id} = f.id+ WHERE ${unionSub.id} IS NULL+ `+ try {+ await this.db.transaction(async (tx) => {+ const [r] = await tx.execute(deleteQuery)+ this.logger.log(`${this.deleteOrphanFiles.name} - files: ${r.affectedRows}`)+ })+ } catch (e) {+ this.logger.log(`${this.deleteOrphanFiles.name} - ${e}`)+ }+ this.logger.log(`${this.deleteOrphanFiles.name} - END`)+ }}diff --git a/backend/src/applications/spaces/services/spaces-browser.service.ts b/backend/src/applications/spaces/services/spaces-browser.service.tsindex 0ed0b1a8..5e2e9cc0 100644--- a/backend/src/applications/spaces/services/spaces-browser.service.ts+++ b/backend/src/applications/spaces/services/spaces-browser.service.ts@@ -20,7 +20,7 @@ import { UserModel } from '../../users/models/user.model'import { LOCK_SCOPE } from '../../webdav/constants/webdav'import { SpaceFiles } from '../interfaces/space-files.interface'import { SpaceEnv } from '../models/space-env.model'-import { realPathFromRootFile, IsRealPathIsDirAndExists } from '../utils/paths'+import { IsRealPathIsDirAndExists, realPathFromRootFile } from '../utils/paths'import { SpacesManager } from './spaces-manager.service'import { SpacesQueries } from './spaces-queries.service'diff --git a/backend/src/applications/spaces/services/spaces-scheduler.service.ts b/backend/src/applications/spaces/services/spaces-scheduler.service.tsindex ef27ef35..c4a975b5 100644--- a/backend/src/applications/spaces/services/spaces-scheduler.service.ts+++ b/backend/src/applications/spaces/services/spaces-scheduler.service.ts@@ -44,7 +44,7 @@ export class SpacesScheduler {this.logger.log('Update Share External Path Quotas - END')}- @Cron(CronExpression.EVERY_DAY_AT_4AM)+ @Cron(CronExpression.EVERY_DAY_AT_2AM)async deleteExpiredSpaces() {/* Removes spaces that have been disabled for more than 30 days */this.logger.log(`${this.deleteExpiredSpaces.name} - START`)diff --git a/backend/src/infrastructure/database/utils.ts b/backend/src/infrastructure/database/utils.tsindex 6ded146b..ff20ad15 100644--- a/backend/src/infrastructure/database/utils.ts+++ b/backend/src/infrastructure/database/utils.ts@@ -7,9 +7,10 @@import { NestFastifyApplication } from '@nestjs/platform-fastify'import { TestingModule } from '@nestjs/testing'import { Column, eq, inArray, isNull, SQL, sql } from 'drizzle-orm'-import { MySqlDatabase } from 'drizzle-orm/mysql-core'+import { MySqlDatabase, MySqlTableWithColumns } from 'drizzle-orm/mysql-core'import { MySqlQueryResult } from 'drizzle-orm/mysql2'import { DB_TOKEN_PROVIDER } from './constants'+import * as schema from './schema'async function dbGetConnection(app: NestFastifyApplication | TestingModule, mode: 'pool' | 'client' = 'pool') {const db: MySqlDatabase<any, any> = await app.resolve(DB_TOKEN_PROVIDER)@@ -90,3 +91,9 @@ export function concatDistinctObjectsInArray(mustBeNotNull: Column, object: Recoexpr.mapWith(JSON.parse)return expr}++export function getTablesWithFileIdColumn(): MySqlTableWithColumns<any>[] {+ return Object.entries(schema)+ .filter(([_table, columns]) => 'fileId' in columns)+ .map(([_table, columns]) => columns as MySqlTableWithColumns<any>)+}