Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix(workers): lazy import workers only when enabled

Mohamed Bassem6d013d181667
diff --git a/apps/workers/index.ts b/apps/workers/index.ts
index f5e0eda9..0bcadc0a 100644
--- a/apps/workers/index.ts
+++ b/apps/workers/index.ts
@@ -27,70 +27,89 @@ import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { shutdownPromise } from "./exit";
-import { AdminMaintenanceWorker } from "./workers/adminMaintenanceWorker";
-import { AssetPreprocessingWorker } from "./workers/assetPreprocessingWorker";
-import { BackupSchedulingWorker, BackupWorker } from "./workers/backupWorker";
-import { CrawlerWorker } from "./workers/crawlerWorker";
-import { EmbeddingsWorker } from "./workers/embeddingsWorker";
-import { FeedRefreshingWorker, FeedWorker } from "./workers/feedWorker";
-import { ImportWorker } from "./workers/importWorker";
-import { OpenAiWorker } from "./workers/inference/inferenceWorker";
-import { RuleEngineWorker } from "./workers/ruleEngineWorker";
-import { SearchIndexingWorker } from "./workers/searchWorker";
-import { VideoWorker } from "./workers/videoWorker";
-import { WebhookWorker } from "./workers/webhookWorker";
+
+let backupSchedulingWorker:
+ | typeof import("./workers/backupWorker").BackupSchedulingWorker
+ | undefined;
+let feedRefreshingWorker:
+ | typeof import("./workers/feedWorker").FeedRefreshingWorker
+ | undefined;
const workerBuilders = {
crawler: async () => {
+ const { CrawlerWorker } = await import("./workers/crawlerWorker");
await LinkCrawlerQueue.ensureInit();
return CrawlerWorker.build(LinkCrawlerQueue);
},
lowPriorityCrawler: async () => {
+ const { CrawlerWorker } = await import("./workers/crawlerWorker");
await LowPriorityCrawlerQueue.ensureInit();
return CrawlerWorker.build(LowPriorityCrawlerQueue);
},
embeddings: async () => {
+ const { EmbeddingsWorker } = await import("./workers/embeddingsWorker");
await EmbeddingsQueue.ensureInit();
return EmbeddingsWorker.build();
},
inference: async () => {
+ const { OpenAiWorker } =
+ await import("./workers/inference/inferenceWorker");
await OpenAIQueue.ensureInit();
return OpenAiWorker.build();
},
search: async () => {
+ const { SearchIndexingWorker } = await import("./workers/searchWorker");
await SearchIndexingQueue.ensureInit();
return SearchIndexingWorker.build();
},
adminMaintenance: async () => {
+ const { AdminMaintenanceWorker } =
+ await import("./workers/adminMaintenanceWorker");
await AdminMaintenanceQueue.ensureInit();
return AdminMaintenanceWorker.build();
},
video: async () => {
+ const { VideoWorker } = await import("./workers/videoWorker");
await VideoWorkerQueue.ensureInit();
return VideoWorker.build();
},
feed: async () => {
+ const { FeedRefreshingWorker, FeedWorker } =
+ await import("./workers/feedWorker");
+ feedRefreshingWorker = FeedRefreshingWorker;
await FeedQueue.ensureInit();
return FeedWorker.build();
},
assetPreprocessing: async () => {
+ const { AssetPreprocessingWorker } =
+ await import("./workers/assetPreprocessingWorker");
await AssetPreprocessingQueue.ensureInit();
return AssetPreprocessingWorker.build();
},
webhook: async () => {
+ const { WebhookWorker } = await import("./workers/webhookWorker");
await WebhookQueue.ensureInit();
return WebhookWorker.build();
},
ruleEngine: async () => {
+ const { RuleEngineWorker } = await import("./workers/ruleEngineWorker");
await RuleEngineQueue.ensureInit();
return RuleEngineWorker.build();
},
backup: async () => {
+ const { BackupSchedulingWorker, BackupWorker } =
+ await import("./workers/backupWorker");
+ backupSchedulingWorker = BackupSchedulingWorker;
await BackupQueue.ensureInit();
return BackupWorker.build();
},
} as const;
+async function buildImportWorker() {
+ const { ImportWorker } = await import("./workers/importWorker");
+ return new ImportWorker();
+}
+
type WorkerName = keyof typeof workerBuilders | "import";
const enabledWorkers = new Set(serverConfig.workers.enabledWorkers);
const disabledWorkers = new Set(serverConfig.workers.disabledWorkers);
@@ -126,18 +145,18 @@ async function main() {
await startQueue();
if (workers.some((w) => w.name === "feed")) {
- FeedRefreshingWorker.start();
+ feedRefreshingWorker?.start();
}
if (workers.some((w) => w.name === "backup")) {
- BackupSchedulingWorker.start();
+ backupSchedulingWorker?.start();
}
// Start import polling worker
- let importWorker: ImportWorker | null = null;
+ let importWorker = null;
let importWorkerPromise: Promise<void> | null = null;
if (isWorkerEnabled("import")) {
- importWorker = new ImportWorker();
+ importWorker = await buildImportWorker();
importWorkerPromise = importWorker.start();
}
@@ -159,10 +178,10 @@ async function main() {
);
if (workers.some((w) => w.name === "feed")) {
- FeedRefreshingWorker.stop();
+ feedRefreshingWorker?.stop();
}
if (workers.some((w) => w.name === "backup")) {
- BackupSchedulingWorker.stop();
+ backupSchedulingWorker?.stop();
}
if (importWorker) {
importWorker.stop();
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 364576f7..2c166e30 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1698,8 +1698,6 @@ importers:
specifier: ^5.9
version: 5.9.3
- tools/evals: {}
-
tools/seed-snapshot:
dependencies:
'@karakeep/shared':