Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better clean up job

Edward Cui639a432efd6a
diff --git a/server/src/index.ts b/server/src/index.ts
index f75705cb6..2b10073dc 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -71,7 +71,6 @@ import { initPostgres } from "./db/postgres/initPostgres.js";
import { getSessionFromReq, getUserHasAccessToSitePublic, mapHeaders } from "./lib/auth-utils.js";
import { auth } from "./lib/auth.js";
import { IS_CLOUD } from "./lib/const.js";
-import { siteConfig } from "./lib/siteConfig.js";
import { trackEvent } from "./services/tracker/trackEvent.js";
// need to import telemetry service here to start it
import { telemetryService } from "./services/telemetryService.js";
@@ -463,7 +462,9 @@ const start = async () => {
await registerDataInsertWorker();
// Initialize import cleanup service (runs daily to clean up old import files)
- importCleanupService.initializeCleanupCron();
+ if (IS_CLOUD) {
+ importCleanupService.initializeCleanupCron();
+ }
// Start the server first
await server.listen({ port: 3001, host: "0.0.0.0" });
diff --git a/server/src/services/import/importCleanupService.ts b/server/src/services/import/importCleanupService.ts
index 03ed5df8b..b3118fcf3 100644
--- a/server/src/services/import/importCleanupService.ts
+++ b/server/src/services/import/importCleanupService.ts
@@ -1,9 +1,5 @@
import * as cron from "node-cron";
-import { DateTime } from "luxon";
-import { db } from "../../db/postgres/postgres.js";
-import { importStatus } from "../../db/postgres/schema.js";
-import { sql } from "drizzle-orm";
-import { deleteImportFile, getImportStorageLocation } from "./utils.js";
+import { r2Storage } from "../storage/r2StorageService.js";
class ImportCleanupService {
private cleanupTask: cron.ScheduledTask | null = null;
@@ -34,56 +30,17 @@ class ImportCleanupService {
}
/**
- * Clean up orphaned import files that are more than 1 day old
- * and belong to completed or failed imports.
+ * Clean up orphaned R2 import files that are more than 1 day old.
*/
private async cleanupOrphanedFiles() {
- console.info("[ImportCleanup] Starting cleanup of orphaned import files");
-
- const oneDayAgo = DateTime.utc().minus({ days: 1 }).toISO();
-
- try {
- // Find completed/failed imports older than 1 day that still have files
- const oldImports = await db
- .select({
- importId: importStatus.importId,
- fileName: importStatus.fileName,
- })
- .from(importStatus)
- .where(
- sql`${importStatus.status} IN ('completed', 'failed')
- AND ${importStatus.startedAt} < ${oneDayAgo}`
- );
-
- console.info(`[ImportCleanup] Found ${oldImports.length} old imports to check`);
-
- let deletedCount = 0;
- let failedCount = 0;
-
- for (const importRecord of oldImports) {
- const storage = getImportStorageLocation(importRecord.importId, importRecord.fileName);
-
- // Attempt to delete the file
- const result = await deleteImportFile(storage.location, storage.isR2);
-
- if (result.success) {
- deletedCount++;
- } else {
- // File might already be deleted or doesn't exist - this is not an error
- if (result.error?.includes("ENOENT") || result.error?.includes("NoSuchKey")) {
- // File doesn't exist, which is fine
- continue;
- }
- failedCount++;
- console.warn(`[ImportCleanup] Failed to delete file for import ${importRecord.importId}: ${result.error}`);
- }
+ if (r2Storage.isEnabled()) {
+ console.info("[ImportCleanup] Starting cleanup of old R2 import files");
+ try {
+ const r2DeletedCount = await r2Storage.deleteOldImportFiles(1);
+ console.info(`[ImportCleanup] Deleted ${r2DeletedCount} old files from R2`);
+ } catch (error) {
+ console.error("[ImportCleanup] Error cleaning up R2 files:", error);
}
-
- console.info(
- `[ImportCleanup] Cleanup complete: ${deletedCount} files deleted, ${failedCount} failures, ${oldImports.length - deletedCount - failedCount} already gone`
- );
- } catch (error) {
- console.error("[ImportCleanup] Error during cleanup:", error);
}
}
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 9f268bb4f..3ba724c0e 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -1,4 +1,10 @@
-import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";
+import {
+ S3Client,
+ PutObjectCommand,
+ GetObjectCommand,
+ DeleteObjectCommand,
+ ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { NodeHttpHandler } from "@smithy/node-http-handler";
import { Readable } from "stream";
@@ -276,6 +282,89 @@ class R2StorageService {
// Non-critical error, log but don't throw
}
}
+
+ /**
+ * List all import files in R2
+ * Returns array of objects with key and lastModified date
+ */
+ async listImportFiles(): Promise<Array<{ key: string; lastModified: Date }>> {
+ if (!this.enabled || !this.client) {
+ return [];
+ }
+
+ try {
+ const allFiles: Array<{ key: string; lastModified: Date }> = [];
+ let continuationToken: string | undefined;
+
+ do {
+ const response = await this.client.send(
+ new ListObjectsV2Command({
+ Bucket: this.bucketName,
+ Prefix: "imports/",
+ ContinuationToken: continuationToken,
+ })
+ );
+
+ if (response.Contents) {
+ for (const obj of response.Contents) {
+ if (obj.Key && obj.LastModified) {
+ allFiles.push({
+ key: obj.Key,
+ lastModified: obj.LastModified,
+ });
+ }
+ }
+ }
+
+ continuationToken = response.NextContinuationToken;
+ } while (continuationToken);
+
+ this.logger.info({ count: allFiles.length }, "Listed import files");
+ return allFiles;
+ } catch (error) {
+ this.logger.error({ error }, "Failed to list import files");
+ throw error;
+ }
+ }
+
+ /**
+ * Delete all import files older than the specified number of days
+ * Returns count of deleted files
+ */
+ async deleteOldImportFiles(daysOld: number = 1): Promise<number> {
+ if (!this.enabled || !this.client) {
+ return 0;
+ }
+
+ try {
+ const files = await this.listImportFiles();
+ const cutoffDate = new Date(Date.now() - daysOld * 24 * 60 * 60 * 1000);
+
+ const filesToDelete = files.filter(file => file.lastModified < cutoffDate);
+
+ this.logger.info(
+ { total: files.length, toDelete: filesToDelete.length, cutoffDate },
+ "Deleting old import files"
+ );
+
+ let deletedCount = 0;
+ for (const file of filesToDelete) {
+ try {
+ await this.deleteImportFile(file.key);
+ deletedCount++;
+ } catch (error) {
+ this.logger.error({ error, key: file.key }, "Failed to delete old import file");
+ // Continue with other files even if one fails
+ }
+ }
+
+ this.logger.info({ deletedCount }, "Completed deleting old import files");
+ return deletedCount;
+ } catch (error) {
+ this.logger.error({ error }, "Failed to delete old import files");
+ throw error;
+ }
+ }
}
// Singleton instance