Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): disable cacheable lookup for liveness check
Gergő Móricz904bba628dd6
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex ba9ec7036..487bf63f6 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -7,6 +7,7 @@ import { fireEngineURL } from "../engines/fire-engine/scrape";import { fetch, RequestInit, Response, FormData, Agent } from "undici";import { cacheableLookup } from "./cacheableLookup";import { log } from "console";+import dns from "dns";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -23,6 +24,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {tryCooldown?: number;mock: MockState | null;abort?: AbortSignal;+ useCacheableLookup?: boolean;};export async function robustFetch<@@ -42,6 +44,7 @@ export async function robustFetch<tryCooldown,mock,abort,+ useCacheableLookup = true,}: RobustFetchParams<Schema>): Promise<Output> {abort?.throwIfAborted();@@ -98,7 +101,7 @@ export async function robustFetch<headersTimeout: 0,bodyTimeout: 0,connect: {- lookup: cacheableLookup.lookup,+ lookup: useCacheableLookup ? cacheableLookup.lookup : dns.lookup,},}),...(body instanceof FormDatadiff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 9d344e768..eb063538c 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -464,6 +464,7 @@ app.get("/liveness", (req, res) => {logger: _logger,abort: AbortSignal.timeout(5000),ignoreResponse: true,+ useCacheableLookup: false,}).then(() => {currentLiveness = true;