🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better file cleanup
Edward Cuibce65713507c
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 7b2a9ff0f..9d3c53da1 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -12,6 +12,7 @@ import { ImportLimiter } from "../../services/import/importLimiter.js";import { r2Storage } from "../../services/storage/r2StorageService.js";import { IS_CLOUD } from "../../lib/const.js";import { DateTime } from "luxon";+import { deleteImportFile } from "../../services/import/utils.js";const isValidDate = (val: string) => {const dt = DateTime.fromFormat(val, "yyyy-MM-dd", { zone: "utc" });@@ -151,11 +152,7 @@ export async function importSiteData(});} catch (queueError) {await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");- if (IS_CLOUD && r2Storage.isEnabled()) {- await r2Storage.deleteImportFile(storageLocation);- } else {- await fs.promises.unlink(storageLocation);- }+ await deleteImportFile(storageLocation, IS_CLOUD && r2Storage.isEnabled());console.error("Failed to enqueue import job:", queueError);return reply.status(500).send({ error: "Failed to initiate import process." });}diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsnew file mode 100644index 000000000..beb0db96b--- /dev/null+++ b/server/src/services/import/utils.ts@@ -0,0 +1,12 @@+import { r2Storage } from "../storage/r2StorageService.js";+import fs from "fs";++export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean) => {+ if (isR2Storage) {+ await r2Storage.deleteImportFile(storageLocation);+ console.log(`Deleted R2 import file: ${storageLocation}`);+ } else {+ await fs.promises.unlink(storageLocation);+ console.log(`Deleted local import file: ${storageLocation}`);+ }+}diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex be2cf136e..1a21703ca 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -9,6 +9,16 @@ import { ImportStatusManager } from "../importStatusManager.js";import { ImportLimiter } from "../importLimiter.js";import { r2Storage } from "../../storage/r2StorageService.js";import { DateTime } from "luxon";+import { deleteImportFile } from "../utils.js";++const getImportDataHeaders = (source: string) => {+ switch (source) {+ case "umami":+ return umamiHeaders;+ default:+ throw new Error(`Unsupported import source: ${source}`);+ }+}export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {@@ -18,12 +28,7 @@ export async function registerCsvParseWorker() {const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");-- if (isR2Storage) {- await r2Storage.deleteImportFile(storageLocation);- } else {- await fs.promises.unlink(storageLocation);- }+ await deleteImportFile(storageLocation, isR2Storage);return;}@@ -37,14 +42,14 @@ export async function registerCsvParseWorker() {console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);const fileStream = await r2Storage.getImportFileStream(storageLocation);stream = fileStream.pipe(parse({- headers: umamiHeaders,+ headers: getImportDataHeaders(source),renameHeaders: true,ignoreEmpty: true,}));} else {console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);stream = fs.createReadStream(storageLocation).pipe(parse({- headers: umamiHeaders,+ headers: getImportDataHeaders(source),renameHeaders: true,ignoreEmpty: true,}));@@ -119,18 +124,7 @@ export async function registerCsvParseWorker() {);throw error;} finally {- try {- if (isR2Storage) {- await r2Storage.deleteImportFile(storageLocation);- console.log(`[CSV Parser] Cleaned up R2 file: ${storageLocation}`);- } else {- await fs.promises.unlink(storageLocation);- console.log(`[CSV Parser] Cleaned up local file: ${storageLocation}`);- }- } catch (cleanupError) {- console.error("Error cleaning up import file:", cleanupError);- // Don't throw here as the main processing might have succeeded- }+ await deleteImportFile(storageLocation, isR2Storage);}});}