A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
Remove queue size checks from import worker capacity calculation (#2597)
Mohamed Bassemfba7108b1e29
diff --git a/apps/workers/workers/importWorker.ts b/apps/workers/workers/importWorker.tsindex 2f689d2e..957d3012 100644--- a/apps/workers/workers/importWorker.ts+++ b/apps/workers/workers/importWorker.ts@@ -20,7 +20,6 @@ import {importSessions,importStagingBookmarks,} from "@karakeep/db/schema";-import { LowPriorityCrawlerQueue, OpenAIQueue } from "@karakeep/shared-server";import logger, { throttledLogger } from "@karakeep/shared/logger";import {BookmarkTypes,@@ -626,33 +625,23 @@ export class ImportWorker {}/**- * Backpressure: Calculate available capacity based on number of items in flight and the health of the import queues.+ * Backpressure: Calculate available capacity based on number of items currently processing.*/private async getAvailableCapacity(): Promise<number> {- const [processingCount, crawlerQueue, openaiQueue] = await Promise.all([- db- .select({ count: count() })- .from(importStagingBookmarks)- .where(- and(- eq(importStagingBookmarks.status, "processing"),- gt(- importStagingBookmarks.processingStartedAt,- new Date(Date.now() - this.staleThresholdMs),- ),+ const processingCount = await db+ .select({ count: count() })+ .from(importStagingBookmarks)+ .where(+ and(+ eq(importStagingBookmarks.status, "processing"),+ gt(+ importStagingBookmarks.processingStartedAt,+ new Date(Date.now() - this.staleThresholdMs),),),- LowPriorityCrawlerQueue.stats(),- OpenAIQueue.stats(),- ]);-- const crawlerTotal =- crawlerQueue.pending + crawlerQueue.running + crawlerQueue.pending_retry;- const openaiTotal =- openaiQueue.pending + openaiQueue.running + openaiQueue.pending_retry;- const processingTotal = processingCount[0]?.count ?? 0;+ );- const inFlight = Math.max(crawlerTotal, openaiTotal, processingTotal);+ const inFlight = processingCount[0]?.count ?? 0;importStagingInFlightGauge.set(inFlight);return this.maxInFlight - inFlight;