🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better import data
Edward Cuif1677437e525
diff --git a/server/src/api/import/importData.ts b/server/src/api/import/importData.tsindex 79ace229f..d5358b986 100644--- a/server/src/api/import/importData.ts+++ b/server/src/api/import/importData.ts@@ -5,10 +5,10 @@ import path from "path";import { z } from "zod";import crypto from "crypto";import boss from "../../lib/boss.js";-import {getUserHasAccessToSite, getUserHasAdminAccessToSite} from "../../lib/auth-utils.js";+import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { CSV_PARSE_QUEUE } from "../../types/import.js";--const IMPORT_DIR = "/tmp/imports";+import { ImportRateLimiter } from "../../lib/rateLimiter.js";+import { ImportStatusManager } from "../../lib/importStatus.js";const importDataRequestSchema = z.object({params: z.object({@@ -50,25 +50,50 @@ export async function importData(return reply.status(403).send({ error: "Forbidden" });}+ const orgLimitResult = await ImportRateLimiter.checkRateLimit(organization);+ if (!orgLimitResult.allowed) {+ return reply.status(429).send({+ error: "Organization limit exceeded",+ message: orgLimitResult.reason,+ });+ }+const fileData = await request.file();if (!fileData) {return reply.status(400).send({ error: "No file uploaded." });}- if (!fileData.filename.endsWith(".csv")) {+ if (fileData.file.truncated) {+ return reply.status(400).send({ error: "File too large. Max size is 100MB." });+ }++ if (fileData.mimetype !== "text/csv" || !fileData.filename.endsWith(".csv")) {return reply.status(400).send({error: "Invalid file type. Only .csv files are accepted.",});}- const importId = `${source}_${crypto.randomUUID()}`;- const tempFilePath = path.join(IMPORT_DIR, importId);+ const importDir = "/tmp/imports"; // ./tmp/imports?+ const importId = crypto.randomUUID();+ const savedFileName = `${importId}.csv`;+ const tempFilePath = path.join(importDir, savedFileName);++ await ImportStatusManager.createImportStatus({+ importId,+ siteId: Number(site),+ organizationId: organization,+ source,+ status: "pending",+ fileName: fileData.filename,+ fileSize: fileData.file.readableLength || 0,+ });try {- await fs.promises.mkdir(IMPORT_DIR, { recursive: true });+ await fs.promises.mkdir(importDir, { recursive: true });await pipeline(fileData.file, fs.createWriteStream(tempFilePath));} catch (fileError) {- console.error("🚨 Failed to save uploaded file to disk:", fileError);+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");+ console.error("Failed to save uploaded file to disk:", fileError);return reply.status(500).send({ error: "Could not process file upload." });}@@ -81,7 +106,8 @@ export async function importData(source,});} catch (queueError) {- console.error("🚨 Failed to enqueue import job:", queueError);+ await ImportStatusManager.updateStatus(importId, "failed", "Failed to queue import job");+ console.error("Failed to enqueue import job:", queueError);return reply.status(500).send({ error: "Failed to initiate import process." });}@@ -91,7 +117,7 @@ export async function importData(importId,});} catch (error) {- console.error("🚨 Unexpected error during import:", error);+ console.error("Unexpected error during import:", error);return reply.status(500).send({error: "An unexpected error occurred. Please try again later.",});