🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Improve import initiation
Edward Cui36e4f3bed678
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.tsindex 2153d235e..352cc149f 100644--- a/server/src/workers/importInitiation.ts+++ b/server/src/workers/importInitiation.ts@@ -1,48 +1,46 @@-import fs from 'fs';-import { parse } from '@fast-csv/parse';+import fs from "fs";+import { parseStream } from "@fast-csv/parse";import boss from "../lib/boss.js";import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";import { Job } from "pg-boss";await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {const { tempFilePath, site, importId, source } = job[0].data;- let chunkNumber = 0;- let rowCount = 0;const chunkSize = 1000;- let chunk = [];+ let chunkNumber = 0;+ let chunk: any[] = [];const stream = fs.createReadStream(tempFilePath);- // const parser = parse(stream);-- const parser = fs.createReadStream(tempFilePath).pipe(parse());-- for await (const row of parser) {- chunk.push(row);- rowCount++;-- if (chunk.length >= chunkSize) {- await boss.send(processImportChunkQueue, {- importId,- source,- site,- chunkNumber: ++chunkNumber,- records: chunk,- });- chunk = [];- }- }-- if (chunk.length > 0) {- await boss.send(processImportChunkQueue, {- importId,- source,- site,- chunkNumber: ++chunkNumber,- records: chunk,- });- }- console.log(`✅ Split file ${tempFilePath} into ${chunkNumber} chunks for batch ${importId}.`);+ parseStream(stream, { headers: true })+ .on("headers", headers => console.log(headers)) // validate headers+ .on("data", async (row) => {+ chunk.push(row);++ if (chunk.length >= chunkSize) {+ await boss.send(processImportChunkQueue, {+ importId,+ source,+ site,+ chunkNumber: ++chunkNumber,+ records: chunk,+ });+ chunk = [];+ }+ })+ .on("end", async (rowCount: number) => {+ if (chunk.length > 0) {+ await boss.send(processImportChunkQueue, {+ importId,+ source,+ site,+ chunkNumber: ++chunkNumber,+ records: chunk,+ });+ }+ console.log(`✅ Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);+ })+ .on("error", error => console.error("CSV parsing error:", error));await fs.promises.unlink(tempFilePath);});