Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/worker): liveness check in queueing -- don't take jobs when the worker is dying (#1725)
Gergő Móricz57b8e66bc8f4
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 54a9e6ff4..9eb4cd0f5 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -750,6 +750,11 @@ const workerFun = async (await sleep(cantAcceptConnectionInterval); // more sleepcontinue;+ } else if (!currentLiveness) {+ logger.info("Not accepting jobs because the liveness check failed");++ await sleep(cantAcceptConnectionInterval);+ continue;} else {cantAcceptConnectionCount = 0;}@@ -1564,9 +1569,12 @@ async function processJob(job: Job & { id: string }, token: string) {// Start all workersconst app = Express();+let currentLiveness: boolean = true;+app.get("/liveness", (req, res) => {// stalled checkif (isWorkerStalled) {+ currentLiveness = false;res.status(500).json({ ok: false });} else {// networking check@@ -1579,9 +1587,11 @@ app.get("/liveness", (req, res) => {ignoreResponse: true,}).then(() => {+ currentLiveness = true;res.status(200).json({ ok: true });}).catch(e => {_logger.error("WORKER NETWORKING CHECK FAILED", { error: e });+ currentLiveness = false;res.status(500).json({ ok: false });});}