🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Import initiation wip
Edward Cuiea17c03cf1d2
diff --git a/server/src/types/import.ts b/server/src/types/import.tsindex c0fda965f..020b60683 100644--- a/server/src/types/import.ts+++ b/server/src/types/import.ts@@ -2,7 +2,7 @@ export const importInitiationQueue = "import-initiation";export const processImportChunkQueue = "process-import-chunk";-export interface ImportJob {+export interface ImportInitiationJob {tempFilePath: string;site: string;importId: string;diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.tsnew file mode 100644index 000000000..2153d235e--- /dev/null+++ b/server/src/workers/importInitiation.ts@@ -0,0 +1,48 @@+import fs from 'fs';+import { parse } from '@fast-csv/parse';+import boss from "../lib/boss.js";+import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";+import { Job } from "pg-boss";++await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {+ const { tempFilePath, site, importId, source } = job[0].data;+ let chunkNumber = 0;+ let rowCount = 0;+ const chunkSize = 1000;+ let chunk = [];++ const stream = fs.createReadStream(tempFilePath);+ // const parser = parse(stream);++ const parser = fs.createReadStream(tempFilePath).pipe(parse());++ for await (const row of parser) {+ chunk.push(row);+ rowCount++;++ if (chunk.length >= chunkSize) {+ await boss.send(processImportChunkQueue, {+ importId,+ source,+ site,+ chunkNumber: ++chunkNumber,+ records: chunk,+ });+ chunk = [];+ }+ }++ if (chunk.length > 0) {+ await boss.send(processImportChunkQueue, {+ importId,+ source,+ site,+ chunkNumber: ++chunkNumber,+ records: chunk,+ });+ }++ console.log(`✅ Split file ${tempFilePath} into ${chunkNumber} chunks for batch ${importId}.`);++ await fs.promises.unlink(tempFilePath);+});