The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Cap memory during CSV import queue (#3816)
Pedro Ladeira20d13bddd339
diff --git a/apps/web/app/api/workspaces/[idOrSlug]/import/csv/route.ts b/apps/web/app/api/workspaces/[idOrSlug]/import/csv/route.tsindex a06ef8bf2d..db207ca323 100644--- a/apps/web/app/api/workspaces/[idOrSlug]/import/csv/route.ts+++ b/apps/web/app/api/workspaces/[idOrSlug]/import/csv/route.ts@@ -49,33 +49,65 @@ export const POST = withWorkspace(// Convert file to textconst csvText = await file.text();- // Parse CSV and add rows to Redis+ let chain: Promise<unknown> = Promise.resolve();+ let killed = false;+await new Promise<void>((resolve, reject) => {+ const fail = (error: unknown) => {+ if (killed) return;+ killed = true;+ void (async () => {+ try {+ await redis.del(+ `${redisKey}:rows`,+ `${redisKey}:created`,+ `${redisKey}:processed`,+ `${redisKey}:failed`,+ `${redisKey}:domains`,+ );+ } catch {+ // best-effort: avoid leaving a partial import queue on failure+ }+ reject(error);+ })();+ };+Papa.parse(csvText, {header: true,skipEmptyLines: true,- step: async (results) => {+ step: (results, parser) => {+ if (killed) return;rows.push(results.data as Record<string, string>);- if (rows.length >= BATCH_SIZE) {- try {- await redis.lpush(`${redisKey}:rows`, ...rows);- rows = [];- } catch (error) {- reject(error);- }- }+ if (rows.length < BATCH_SIZE) return;++ const batch = rows;+ rows = [];+ parser.pause();+ chain = chain+ .then(() => redis.lpush(`${redisKey}:rows`, ...batch))+ .then(() => {+ if (!killed) parser.resume();+ })+ .catch(fail);+ },+ complete: () => {+ void chain+ .then(() => {+ if (killed) return;+ if (rows.length > 0) {+ return redis.lpush(`${redisKey}:rows`, ...rows);+ }+ })+ .then(() => {+ if (!killed) resolve();+ })+ .catch(fail);},- complete: () => resolve(),- error: reject,+ error: fail,});});- // Add any remaining rows to Redis- if (rows.length > 0) {- await redis.lpush(`${redisKey}:rows`, ...rows);- }-// Initialize Redis countersawait Promise.all([redis.set(`${redisKey}:created`, "0"),