🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Extract stream creation logic into separate functions
Edward Cui1860acdf96d9
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 97351dd85..b58003ed5 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -1,6 +1,5 @@import { access, constants } from "node:fs/promises";import { createReadStream } from "node:fs";-import { Readable } from "stream";import { parse } from "@fast-csv/parse";import { DateTime } from "luxon";import { Job } from "pg-boss";@@ -19,7 +18,27 @@ const getImportDataHeaders = (source: string) => {default:throw new Error(`Unsupported import source: ${source}`);}-}+};++const createR2FileStream = async (storageLocation: string, source: string) => {+ console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);+ const fileStream = await r2Storage.getImportFileStream(storageLocation);+ return fileStream.pipe(parse({+ headers: getImportDataHeaders(source),+ renameHeaders: true,+ ignoreEmpty: true,+ }));+};++const createLocalFileStream = async (storageLocation: string, source: string) => {+ console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);+ await access(storageLocation, constants.F_OK | constants.R_OK);+ return createReadStream(storageLocation).pipe(parse({+ headers: getImportDataHeaders(source),+ renameHeaders: true,+ ignoreEmpty: true,+ }));+};export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {@@ -37,25 +56,9 @@ export async function registerCsvParseWorker() {let chunk: UmamiEvent[] = [];let rowsProcessed = 0;- let stream: Readable;-- if (isR2Storage) {- console.log(`[CSV Parser] Reading from R2: ${storageLocation}`);- const fileStream = await r2Storage.getImportFileStream(storageLocation);- stream = fileStream.pipe(parse({- headers: getImportDataHeaders(source),- renameHeaders: true,- ignoreEmpty: true,- }));- } else {- console.log(`[CSV Parser] Reading from local disk: ${storageLocation}`);- await access(storageLocation, constants.F_OK | constants.R_OK);- stream = createReadStream(storageLocation).pipe(parse({- headers: getImportDataHeaders(source),- renameHeaders: true,- ignoreEmpty: true,- }));- }+ const stream = isR2Storage+ ? await createR2FileStream(storageLocation, source)+ : await createLocalFileStream(storageLocation, source);await ImportStatusManager.updateStatus(importId, "processing");