Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

Remove queue size checks from import worker capacity calculation (#2597)

Mohamed Bassemfba7108b1e29
diff --git a/apps/workers/workers/importWorker.ts b/apps/workers/workers/importWorker.ts
index 2f689d2e..957d3012 100644
--- a/apps/workers/workers/importWorker.ts
+++ b/apps/workers/workers/importWorker.ts
@@ -20,7 +20,6 @@ import {
importSessions,
importStagingBookmarks,
} from "@karakeep/db/schema";
-import { LowPriorityCrawlerQueue, OpenAIQueue } from "@karakeep/shared-server";
import logger, { throttledLogger } from "@karakeep/shared/logger";
import {
BookmarkTypes,
@@ -626,33 +625,23 @@ export class ImportWorker {
}
/**
- * Backpressure: Calculate available capacity based on number of items in flight and the health of the import queues.
+ * Backpressure: Calculate available capacity based on number of items currently processing.
*/
private async getAvailableCapacity(): Promise<number> {
- const [processingCount, crawlerQueue, openaiQueue] = await Promise.all([
- db
- .select({ count: count() })
- .from(importStagingBookmarks)
- .where(
- and(
- eq(importStagingBookmarks.status, "processing"),
- gt(
- importStagingBookmarks.processingStartedAt,
- new Date(Date.now() - this.staleThresholdMs),
- ),
+ const processingCount = await db
+ .select({ count: count() })
+ .from(importStagingBookmarks)
+ .where(
+ and(
+ eq(importStagingBookmarks.status, "processing"),
+ gt(
+ importStagingBookmarks.processingStartedAt,
+ new Date(Date.now() - this.staleThresholdMs),
),
),
- LowPriorityCrawlerQueue.stats(),
- OpenAIQueue.stats(),
- ]);
-
- const crawlerTotal =
- crawlerQueue.pending + crawlerQueue.running + crawlerQueue.pending_retry;
- const openaiTotal =
- openaiQueue.pending + openaiQueue.running + openaiQueue.pending_retry;
- const processingTotal = processingCount[0]?.count ?? 0;
+ );
- const inFlight = Math.max(crawlerTotal, openaiTotal, processingTotal);
+ const inFlight = processingCount[0]?.count ?? 0;
importStagingInFlightGauge.set(inFlight);
return this.maxInFlight - inFlight;