Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better import data

Edward Cuif1677437e525
diff --git a/server/src/api/import/importData.ts b/server/src/api/import/importData.ts
index 79ace229f..d5358b986 100644
--- a/server/src/api/import/importData.ts
+++ b/server/src/api/import/importData.ts
@@ -5,10 +5,10 @@ import path from "path";
import { z } from "zod";
import crypto from "crypto";
import boss from "../../lib/boss.js";
-import {getUserHasAccessToSite, getUserHasAdminAccessToSite} from "../../lib/auth-utils.js";
+import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { CSV_PARSE_QUEUE } from "../../types/import.js";
-
-const IMPORT_DIR = "/tmp/imports";
+import { ImportRateLimiter } from "../../lib/rateLimiter.js";
+import { ImportStatusManager } from "../../lib/importStatus.js";
const importDataRequestSchema = z.object({
params: z.object({
@@ -50,25 +50,50 @@ export async function importData(
return reply.status(403).send({ error: "Forbidden" });
}
+ const orgLimitResult = await ImportRateLimiter.checkRateLimit(organization);
+ if (!orgLimitResult.allowed) {
+ return reply.status(429).send({
+ error: "Organization limit exceeded",
+ message: orgLimitResult.reason,
+ });
+ }
+
const fileData = await request.file();
if (!fileData) {
return reply.status(400).send({ error: "No file uploaded." });
}
- if (!fileData.filename.endsWith(".csv")) {
+ if (fileData.file.truncated) {
+ return reply.status(400).send({ error: "File too large. Max size is 100MB." });
+ }
+
+ if (fileData.mimetype !== "text/csv" || !fileData.filename.endsWith(".csv")) {
return reply.status(400).send({
error: "Invalid file type. Only .csv files are accepted.",
});
}
- const importId = `${source}_${crypto.randomUUID()}`;
- const tempFilePath = path.join(IMPORT_DIR, importId);
+ const importDir = "/tmp/imports"; // ./tmp/imports?
+ const importId = crypto.randomUUID();
+ const savedFileName = `${importId}.csv`;
+ const tempFilePath = path.join(importDir, savedFileName);
+
+ await ImportStatusManager.createImportStatus({
+ importId,
+ siteId: Number(site),
+ organizationId: organization,
+ source,
+ status: "pending",
+ fileName: fileData.filename,
+ fileSize: fileData.file.readableLength || 0,
+ });
try {
- await fs.promises.mkdir(IMPORT_DIR, { recursive: true });
+ await fs.promises.mkdir(importDir, { recursive: true });
await pipeline(fileData.file, fs.createWriteStream(tempFilePath));
} catch (fileError) {
- console.error("🚨 Failed to save uploaded file to disk:", fileError);
+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");
+ console.error("Failed to save uploaded file to disk:", fileError);
return reply.status(500).send({ error: "Could not process file upload." });
}
@@ -81,7 +106,8 @@ export async function importData(
source,
});
} catch (queueError) {
- console.error("🚨 Failed to enqueue import job:", queueError);
+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");
+ console.error("Failed to enqueue import job:", queueError);
return reply.status(500).send({ error: "Failed to initiate import process." });
}
@@ -91,7 +117,7 @@ export async function importData(
importId,
});
} catch (error) {
- console.error("🚨 Unexpected error during import:", error);
+ console.error("Unexpected error during import:", error);
return reply.status(500).send({
error: "An unexpected error occurred. Please try again later.",
});