🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better clean up job
Edward Cui639a432efd6a
diff --git a/server/src/index.ts b/server/src/index.tsindex f75705cb6..2b10073dc 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -71,7 +71,6 @@ import { initPostgres } from "./db/postgres/initPostgres.js";import { getSessionFromReq, getUserHasAccessToSitePublic, mapHeaders } from "./lib/auth-utils.js";import { auth } from "./lib/auth.js";import { IS_CLOUD } from "./lib/const.js";-import { siteConfig } from "./lib/siteConfig.js";import { trackEvent } from "./services/tracker/trackEvent.js";// need to import telemetry service here to start itimport { telemetryService } from "./services/telemetryService.js";@@ -463,7 +462,9 @@ const start = async () => {await registerDataInsertWorker();// Initialize import cleanup service (runs daily to clean up old import files)- importCleanupService.initializeCleanupCron();+ if (IS_CLOUD) {+ importCleanupService.initializeCleanupCron();+ }// Start the server firstawait server.listen({ port: 3001, host: "0.0.0.0" });diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.tsindex 03ed5df8b..b3118fcf3 100644--- a/server/src/services/import/importCleanupService.ts+++ b/server/src/services/import/importCleanupService.ts@@ -1,9 +1,5 @@import * as cron from "node-cron";-import { DateTime } from "luxon";-import { db } from "../../db/postgres/postgres.js";-import { importStatus } from "../../db/postgres/schema.js";-import { sql } from "drizzle-orm";-import { deleteImportFile, getImportStorageLocation } from "./utils.js";+import { r2Storage } from "../storage/r2StorageService.js";class ImportCleanupService {private cleanupTask: cron.ScheduledTask | null = null;@@ -34,56 +30,17 @@ class ImportCleanupService {}/**- * Clean up orphaned import files that are more than 1 day old- * and belong to completed or failed imports.+ * Clean up orphaned R2 import files that are more than 1 day old.*/private async cleanupOrphanedFiles() {- console.info("[ImportCleanup] Starting cleanup of orphaned import files");-- const oneDayAgo = DateTime.utc().minus({ days: 1 }).toISO();-- try {- // Find completed/failed imports older than 1 day that still have files- const oldImports = await db- .select({- importId: importStatus.importId,- fileName: importStatus.fileName,- })- .from(importStatus)- .where(- sql`${importStatus.status} IN ('completed', 'failed')- AND ${importStatus.startedAt} < ${oneDayAgo}`- );-- console.info(`[ImportCleanup] Found ${oldImports.length} old imports to check`);-- let deletedCount = 0;- let failedCount = 0;-- for (const importRecord of oldImports) {- const storage = getImportStorageLocation(importRecord.importId, importRecord.fileName);-- // Attempt to delete the file- const result = await deleteImportFile(storage.location, storage.isR2);-- if (result.success) {- deletedCount++;- } else {- // File might already be deleted or doesn't exist - this is not an error- if (result.error?.includes("ENOENT") || result.error?.includes("NoSuchKey")) {- // File doesn't exist, which is fine- continue;- }- failedCount++;- console.warn(`[ImportCleanup] Failed to delete file for import ${importRecord.importId}: ${result.error}`);- }+ if (r2Storage.isEnabled()) {+ console.info("[ImportCleanup] Starting cleanup of old R2 import files");+ try {+ const r2DeletedCount = await r2Storage.deleteOldImportFiles(1);+ console.info(`[ImportCleanup] Deleted ${r2DeletedCount} old files from R2`);+ } catch (error) {+ console.error("[ImportCleanup] Error cleaning up R2 files:", error);}-- console.info(- `[ImportCleanup] Cleanup complete: ${deletedCount} files deleted, ${failedCount} failures, ${oldImports.length - deletedCount - failedCount} already gone`- );- } catch (error) {- console.error("[ImportCleanup] Error during cleanup:", error);}}diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.tsindex 9f268bb4f..3ba724c0e 100644--- a/server/src/services/storage/r2StorageService.ts+++ b/server/src/services/storage/r2StorageService.ts@@ -1,4 +1,10 @@-import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";+import {+ S3Client,+ PutObjectCommand,+ GetObjectCommand,+ DeleteObjectCommand,+ ListObjectsV2Command,+} from "@aws-sdk/client-s3";import { Upload } from "@aws-sdk/lib-storage";import { NodeHttpHandler } from "@smithy/node-http-handler";import { Readable } from "stream";@@ -276,6 +282,89 @@ class R2StorageService {// Non-critical error, log but don't throw}}++ /**+ * List all import files in R2+ * Returns array of objects with key and lastModified date+ */+ async listImportFiles(): Promise<Array<{ key: string; lastModified: Date }>> {+ if (!this.enabled || !this.client) {+ return [];+ }++ try {+ const allFiles: Array<{ key: string; lastModified: Date }> = [];+ let continuationToken: string | undefined;++ do {+ const response = await this.client.send(+ new ListObjectsV2Command({+ Bucket: this.bucketName,+ Prefix: "imports/",+ ContinuationToken: continuationToken,+ })+ );++ if (response.Contents) {+ for (const obj of response.Contents) {+ if (obj.Key && obj.LastModified) {+ allFiles.push({+ key: obj.Key,+ lastModified: obj.LastModified,+ });+ }+ }+ }++ continuationToken = response.NextContinuationToken;+ } while (continuationToken);++ this.logger.info({ count: allFiles.length }, "Listed import files");+ return allFiles;+ } catch (error) {+ this.logger.error({ error }, "Failed to list import files");+ throw error;+ }+ }++ /**+ * Delete all import files older than the specified number of days+ * Returns count of deleted files+ */+ async deleteOldImportFiles(daysOld: number = 1): Promise<number> {+ if (!this.enabled || !this.client) {+ return 0;+ }++ try {+ const files = await this.listImportFiles();+ const cutoffDate = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);++ const filesToDelete = files.filter(file => file.lastModified < cutoffDate);++ this.logger.info(+ { total: files.length, toDelete: filesToDelete.length, cutoffDate },+ "Deleting old import files"+ );++ let deletedCount = 0;+ for (const file of filesToDelete) {+ try {+ await this.deleteImportFile(file.key);+ deletedCount++;+ } catch (error) {+ this.logger.error({ error, key: file.key }, "Failed to delete old import file");+ // Continue with other files even if one fails+ }+ }++ this.logger.info({ deletedCount }, "Completed deleting old import files");+ return deletedCount;+ } catch (error) {+ this.logger.error({ error }, "Failed to delete old import files");+ throw error;+ }+ }}// Singleton instance