Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Import initiation wip

Edward Cuiea17c03cf1d2
diff --git a/server/src/types/import.ts b/server/src/types/import.ts
index c0fda965f..020b60683 100644
--- a/server/src/types/import.ts
+++ b/server/src/types/import.ts
@@ -2,7 +2,7 @@ export const importInitiationQueue = "import-initiation";
export const processImportChunkQueue = "process-import-chunk";
-export interface ImportJob {
+export interface ImportInitiationJob {
tempFilePath: string;
site: string;
importId: string;
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.ts
new file mode 100644
index 000000000..2153d235e
--- /dev/null
+++ b/server/src/workers/importInitiation.ts
@@ -0,0 +1,48 @@
+import fs from 'fs';
+import { parse } from '@fast-csv/parse';
+import boss from "../lib/boss.js";
+import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";
+import { Job } from "pg-boss";
+
+await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {
+ const { tempFilePath, site, importId, source } = job[0].data;
+ let chunkNumber = 0;
+ let rowCount = 0;
+ const chunkSize = 1000;
+ let chunk = [];
+
+ const stream = fs.createReadStream(tempFilePath);
+ // const parser = parse(stream);
+
+ const parser = fs.createReadStream(tempFilePath).pipe(parse());
+
+ for await (const row of parser) {
+ chunk.push(row);
+ rowCount++;
+
+ if (chunk.length >= chunkSize) {
+ await boss.send(processImportChunkQueue, {
+ importId,
+ source,
+ site,
+ chunkNumber: ++chunkNumber,
+ records: chunk,
+ });
+ chunk = [];
+ }
+ }
+
+ if (chunk.length > 0) {
+ await boss.send(processImportChunkQueue, {
+ importId,
+ source,
+ site,
+ chunkNumber: ++chunkNumber,
+ records: chunk,
+ });
+ }
+
+ console.log(`✅ Split file ${tempFilePath} into ${chunkNumber} chunks for batch ${importId}.`);
+
+ await fs.promises.unlink(tempFilePath);
+});