Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Improve import initiation

Edward Cui36e4f3bed678
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.ts
index 2153d235e..352cc149f 100644
--- a/server/src/workers/importInitiation.ts
+++ b/server/src/workers/importInitiation.ts
@@ -1,48 +1,46 @@
-import fs from 'fs';
-import { parse } from '@fast-csv/parse';
+import fs from "fs";
+import { parseStream } from "@fast-csv/parse";
import boss from "../lib/boss.js";
import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";
import { Job } from "pg-boss";
await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {
const { tempFilePath, site, importId, source } = job[0].data;
- let chunkNumber = 0;
- let rowCount = 0;
const chunkSize = 1000;
- let chunk = [];
+ let chunkNumber = 0;
+ let chunk: any[] = [];
const stream = fs.createReadStream(tempFilePath);
- // const parser = parse(stream);
-
- const parser = fs.createReadStream(tempFilePath).pipe(parse());
-
- for await (const row of parser) {
- chunk.push(row);
- rowCount++;
-
- if (chunk.length >= chunkSize) {
- await boss.send(processImportChunkQueue, {
- importId,
- source,
- site,
- chunkNumber: ++chunkNumber,
- records: chunk,
- });
- chunk = [];
- }
- }
-
- if (chunk.length > 0) {
- await boss.send(processImportChunkQueue, {
- importId,
- source,
- site,
- chunkNumber: ++chunkNumber,
- records: chunk,
- });
- }
- console.log(`✅ Split file ${tempFilePath} into ${chunkNumber} chunks for batch ${importId}.`);
+ parseStream(stream, { headers: true })
+ .on("headers", headers => console.log(headers)) // validate headers
+ .on("data", async (row) => {
+ chunk.push(row);
+
+ if (chunk.length >= chunkSize) {
+ await boss.send(processImportChunkQueue, {
+ importId,
+ source,
+ site,
+ chunkNumber: ++chunkNumber,
+ records: chunk,
+ });
+ chunk = [];
+ }
+ })
+ .on("end", async (rowCount: number) => {
+ if (chunk.length > 0) {
+ await boss.send(processImportChunkQueue, {
+ importId,
+ source,
+ site,
+ chunkNumber: ++chunkNumber,
+ records: chunk,
+ });
+ }
+ console.log(`✅ Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
+ })
+ .on("error", error => console.error("CSV parsing error:", error));
await fs.promises.unlink(tempFilePath);
});