Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better file cleanup

Edward Cuibce65713507c
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 7b2a9ff0f..9d3c53da1 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -12,6 +12,7 @@ import { ImportLimiter } from "../../services/import/importLimiter.js";
import { r2Storage } from "../../services/storage/r2StorageService.js";
import { IS_CLOUD } from "../../lib/const.js";
import { DateTime } from "luxon";
+import { deleteImportFile } from "../../services/import/utils.js";
const isValidDate = (val: string) => {
const dt = DateTime.fromFormat(val, "yyyy-MM-dd", { zone: "utc" });
@@ -151,11 +152,7 @@ export async function importSiteData(
});
} catch (queueError) {
await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");
- if (IS_CLOUD && r2Storage.isEnabled()) {
- await r2Storage.deleteImportFile(storageLocation);
- } else {
- await fs.promises.unlink(storageLocation);
- }
+ await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());
console.error("Failed to enqueue import job:", queueError);
return reply.status(500).send({ error: "Failed to initiate import process." });
}
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
new file mode 100644
index 000000000..beb0db96b
--- /dev/null
+++ b/server/src/services/import/utils.ts
@@ -0,0 +1,12 @@
+import { r2Storage } from "../storage/r2StorageService.js";
+import fs from "fs";
+
+export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean) => {
+ if (isR2Storage) {
+ await r2Storage.deleteImportFile(storageLocation);
+ console.log(`Deleted R2 import file: ${storageLocation}`);
+ } else {
+ await fs.promises.unlink(storageLocation);
+ console.log(`Deleted local import file: ${storageLocation}`);
+ }
+}
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index be2cf136e..1a21703ca 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -9,6 +9,16 @@ import { ImportStatusManager } from "../importStatusManager.js";
import { ImportLimiter } from "../importLimiter.js";
import { r2Storage } from "../../storage/r2StorageService.js";
import { DateTime } from "luxon";
+import { deleteImportFile } from "../utils.js";
+
+const getImportDataHeaders = (source: string) => {
+ switch (source) {
+ case "umami":
+ return umamiHeaders;
+ default:
+ throw new Error(`Unsupported import source: ${source}`);
+ }
+}
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
@@ -18,12 +28,7 @@ export async function registerCsvParseWorker() {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
-
- if (isR2Storage) {
- await r2Storage.deleteImportFile(storageLocation);
- } else {
- await fs.promises.unlink(storageLocation);
- }
+ await deleteImportFile(storageLocation, isR2Storage);
return;
}
@@ -37,14 +42,14 @@ export async function registerCsvParseWorker() {
console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);
const fileStream = await r2Storage.getImportFileStream(storageLocation);
stream = fileStream.pipe(parse({
- headers: umamiHeaders,
+ headers: getImportDataHeaders(source),
renameHeaders: true,
ignoreEmpty: true,
}));
} else {
console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
stream = fs.createReadStream(storageLocation).pipe(parse({
- headers: umamiHeaders,
+ headers: getImportDataHeaders(source),
renameHeaders: true,
ignoreEmpty: true,
}));
@@ -119,18 +124,7 @@ export async function registerCsvParseWorker() {
);
throw error;
} finally {
- try {
- if (isR2Storage) {
- await r2Storage.deleteImportFile(storageLocation);
- console.log(`[CSV Parser] Cleaned up R2 file: ${storageLocation}`);
- } else {
- await fs.promises.unlink(storageLocation);
- console.log(`[CSV Parser] Cleaned up local file: ${storageLocation}`);
- }
- } catch (cleanupError) {
- console.error("Error cleaning up import file:", cleanupError);
- // Don't throw here as the main processing might have succeeded
- }
+ await deleteImportFile(storageLocation, isR2Storage);
}
});
}