Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Don't load file into memory buffer

Edward Cui5fe2fb343227
diff --git a/server/src/api/import.ts b/server/src/api/import.ts
index 4103a6f04..96d63d1cb 100644
--- a/server/src/api/import.ts
+++ b/server/src/api/import.ts
@@ -1,8 +1,14 @@
import { FastifyRequest, FastifyReply } from "fastify";
-import boss from "../lib/boss.js";
-import { getUserHasAccessToSite } from "../lib/auth-utils.js";
+import { pipeline } from "stream/promises";
+import { createWriteStream } from "fs";
+import fs from "fs/promises";
+import path from "path";
import { z } from "zod";
import crypto from "crypto";
+import boss from "../lib/boss.js";
+import { getUserHasAccessToSite } from "../lib/auth-utils.js";
+
+const IMPORT_DIR = "/tmp/imports";
const importRequestSchema = z.object({
Params: z.object({
@@ -22,35 +28,41 @@ export async function importData(
const { site } = request.params;
const { source } = request.body;
- const userHasAccessToSite = await getUserHasAccessToSite(request, site);
- if (!userHasAccessToSite) {
+ const userHasAccess = await getUserHasAccessToSite(request, site);
+ if (!userHasAccess) {
return reply.status(403).send({ error: "Forbidden" });
}
- const file = await request.file();
- if (!file) {
+ const fileData = await request.file();
+ if (!fileData) {
return reply.status(400).send({ error: "No file uploaded." });
}
- if (!file.filename.endsWith(".csv")) {
- return reply
- .status(400)
- .send({ error: "Invalid file type. Only .csv files are accepted." });
+ if (!fileData.filename.endsWith(".csv")) {
+ return reply.status(400).send({ error: "Invalid file type. Only .csv files are accepted." });
}
- const importId = crypto.randomUUID();
- const fileBuffer = await file.toBuffer();
+ const importId = `${source}_${crypto.randomUUID()};`
+ const tempFilePath = path.join(IMPORT_DIR, importId);
+
+ try {
+ await fs.mkdir(IMPORT_DIR, { recursive: true });
+ await pipeline(fileData.file, createWriteStream(tempFilePath));
+ } catch (error) {
+ console.error("🚨 Failed to save uploaded file to disk:", error);
+ return reply.status(500).send({ error: "Could not process file upload." });
+ }
- await boss.send("csv-import", {
- file: {
- data: fileBuffer,
- filename: file.filename,
- mimetype: file.mimetype,
- },
- site_id: Number(site),
+ await boss.send("csv-import-initiation", {
+ filePath: tempFilePath,
+ siteId: Number(site),
importId,
source,
});
- reply.send({ success: true, importId });
+ reply.status(202).send({
+ success: true,
+ message: "File upload accepted and is now being processed.",
+ importId
+ });
}