Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cui7873ec3bb085
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index cd0fed0a7..994203756 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -9,6 +9,8 @@ import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { CSV_PARSE_QUEUE } from "../../services/import/workers/jobs.js";
import { ImportStatusManager } from "../../services/import/importStatusManager.js";
import { ImportLimiter } from "../../services/import/importLimiter.js";
+import { r2Storage } from "../../services/storage/r2StorageService.js";
+import { IS_CLOUD } from "../../lib/const.js";
const importDataRequestSchema = z.object({
params: z.object({
@@ -73,10 +75,7 @@ export async function importSiteData(
}
const organization = concurrentImportLimitResult.organizationId;
- const importDir = "/tmp/imports"; // TODO: ./tmp/imports?
const importId = crypto.randomUUID();
- const savedFileName = `${importId}.csv`;
- const tempFilePath = path.join(importDir, savedFileName);
await ImportStatusManager.createImportStatus({
importId,
@@ -88,18 +87,43 @@ export async function importSiteData(
fileSize: fileData.file.readableLength || 0,
});
+ let storageLocation: string;
+
try {
- await fs.promises.mkdir(importDir, { recursive: true });
- await pipeline(fileData.file, fs.createWriteStream(tempFilePath));
+ if (IS_CLOUD && r2Storage.isEnabled()) {
+ const r2Key = `imports/${importId}/${fileData.filename}`;
+
+ const chunks: Buffer[] = [];
+ for await (const chunk of fileData.file) {
+ chunks.push(chunk);
+ }
+ const fileBuffer = Buffer.concat(chunks);
+
+ await r2Storage.storeImportFile(r2Key, fileBuffer);
+ storageLocation = r2Key;
+
+ console.log(`[Import] File stored in R2: ${r2Key}`);
+ } else {
+ const importDir = "/tmp/imports";
+ const savedFileName = `${importId}.csv`;
+ const tempFilePath = path.join(importDir, savedFileName);
+
+ await fs.promises.mkdir(importDir, { recursive: true });
+ await pipeline(fileData.file, fs.createWriteStream(tempFilePath));
+ storageLocation = tempFilePath;
+
+ console.log(`[Import] File stored locally: ${tempFilePath}`);
+ }
} catch (fileError) {
await ImportStatusManager.updateStatus(importId, "failed", "Failed to save uploaded file");
- console.error("Failed to save uploaded file to disk:", fileError);
+ console.error("Failed to save uploaded file:", fileError);
return reply.status(500).send({ error: "Could not process file upload." });
}
try {
await boss.send(CSV_PARSE_QUEUE, {
- tempFilePath,
+ storageLocation,
+ isR2Storage: IS_CLOUD && r2Storage.isEnabled(),
organization,
site,
importId,
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 85fe153c9..1ed20fb1a 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -1,33 +1,54 @@
import fs from "fs";
import { parse } from "@fast-csv/parse";
+import { Readable } from "stream";
import boss from "../../../db/postgres/boss.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { Job } from "pg-boss";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
import { ImportStatusManager } from "../importStatusManager.js";
import { ImportLimiter } from "../importLimiter.js";
+import { r2Storage } from "../../storage/r2StorageService.js";
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
- const { site, importId, source, tempFilePath, organization } = job.data;
+ const { site, importId, source, storageLocation, isR2Storage, organization } = job.data;
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
- await fs.promises.unlink(tempFilePath);
+
+ if (isR2Storage) {
+ await r2Storage.deleteImportFile(storageLocation);
+ } else {
+ await fs.promises.unlink(storageLocation);
+ }
return;
}
const chunkSize = 1000;
let chunk: UmamiEvent[] = [];
- const stream = fs.createReadStream(tempFilePath).pipe(parse({
- headers: umamiHeaders,
- renameHeaders: true,
- ignoreEmpty: true,
- maxRows: importableEvents,
- }));
+ let stream: Readable;
+
+ if (isR2Storage) {
+ console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);
+ const fileBuffer = await r2Storage.getImportFile(storageLocation);
+ stream = Readable.from(fileBuffer).pipe(parse({
+ headers: umamiHeaders,
+ renameHeaders: true,
+ ignoreEmpty: true,
+ maxRows: importableEvents,
+ }));
+ } else {
+ console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);
+ stream = fs.createReadStream(storageLocation).pipe(parse({
+ headers: umamiHeaders,
+ renameHeaders: true,
+ ignoreEmpty: true,
+ maxRows: importableEvents,
+ }));
+ }
await ImportStatusManager.updateStatus(importId, "processing");
@@ -72,7 +93,18 @@ export async function registerCsvParseWorker() {
);
throw error;
} finally {
- await fs.promises.unlink(tempFilePath);
+ try {
+ if (isR2Storage) {
+ await r2Storage.deleteImportFile(storageLocation);
+ console.log(`[CSV Parser] Cleaned up R2 file: ${storageLocation}`);
+ } else {
+ await fs.promises.unlink(storageLocation);
+ console.log(`[CSV Parser] Cleaned up local file: ${storageLocation}`);
+ }
+ } catch (cleanupError) {
+ console.error("Error cleaning up import file:", cleanupError);
+ // Don't throw here as the main processing might have succeeded
+ }
}
});
}
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index d344b961d..17c2abbd2 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -11,7 +11,8 @@ interface ImportJob {
}
export interface CsvParseJob extends ImportJob {
- tempFilePath: string;
+ storageLocation: string; // Either local file path or R2 key
+ isR2Storage: boolean;
organization: string;
}
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 4bebebdf4..7f7f546d4 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -143,6 +143,96 @@ export class R2StorageService {
// Non-critical error, log but don't throw
}
}
+
+ /**
+ * Store an import CSV file in R2
+ * Used for temporary storage during import processing
+ */
+ async storeImportFile(key: string, fileBuffer: Buffer): Promise<void> {
+ if (!this.enabled || !this.client) {
+ throw new Error("R2 storage is not enabled");
+ }
+
+ try {
+ await this.client.send(
+ new PutObjectCommand({
+ Bucket: this.bucketName,
+ Key: key,
+ Body: fileBuffer,
+ ContentType: "text/csv",
+ Metadata: {
+ type: "import-file",
+ uploadTime: Date.now().toString(),
+ },
+ }),
+ );
+
+ console.log(`[R2Storage] Successfully stored import file: ${key}`);
+ } catch (error) {
+ console.error("[R2Storage] Failed to store import file:", error);
+ throw error;
+ }
+ }
+
+ /**
+ * Retrieve an import CSV file from R2
+ * Returns the file buffer
+ */
+ async getImportFile(key: string): Promise<Buffer> {
+ if (!this.enabled || !this.client) {
+ throw new Error("R2 storage is not enabled");
+ }
+
+ try {
+ const response = await this.client.send(
+ new GetObjectCommand({
+ Bucket: this.bucketName,
+ Key: key,
+ }),
+ );
+
+ if (!response.Body) {
+ throw new Error("Empty response body");
+ }
+
+ // Convert stream to buffer
+ const chunks: Uint8Array[] = [];
+ const stream = response.Body as Readable;
+
+ for await (const chunk of stream) {
+ chunks.push(chunk);
+ }
+
+ const buffer = Buffer.concat(chunks);
+ console.log(`[R2Storage] Successfully retrieved import file: ${key}`);
+ return buffer;
+ } catch (error) {
+ console.error("[R2Storage] Failed to retrieve import file:", error);
+ throw error;
+ }
+ }
+
+ /**
+ * Delete an import file from R2 (for cleanup after processing)
+ */
+ async deleteImportFile(key: string): Promise<void> {
+ if (!this.enabled || !this.client) {
+ return;
+ }
+
+ try {
+ await this.client.send(
+ new DeleteObjectCommand({
+ Bucket: this.bucketName,
+ Key: key,
+ }),
+ );
+ console.log(`[R2Storage] Successfully deleted import file: ${key}`);
+ } catch (error) {
+ console.error("[R2Storage] Failed to delete import file:", error);
+ // Non-critical error, log but don't throw
+ }
+ }
}
// Singleton instance