Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): disable cacheable lookup for liveness check

Gergő Móricz904bba628dd6
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index ba9ec7036..487bf63f6 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -7,6 +7,7 @@ import { fireEngineURL } from "../engines/fire-engine/scrape";
import { fetch, RequestInit, Response, FormData, Agent } from "undici";
import { cacheableLookup } from "./cacheableLookup";
import { log } from "console";
+import dns from "dns";
export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string;
@@ -23,6 +24,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
tryCooldown?: number;
mock: MockState | null;
abort?: AbortSignal;
+ useCacheableLookup?: boolean;
};
export async function robustFetch<
@@ -42,6 +44,7 @@ export async function robustFetch<
tryCooldown,
mock,
abort,
+ useCacheableLookup = true,
}: RobustFetchParams<Schema>): Promise<Output> {
abort?.throwIfAborted();
@@ -98,7 +101,7 @@ export async function robustFetch<
headersTimeout: 0,
bodyTimeout: 0,
connect: {
- lookup: cacheableLookup.lookup,
+ lookup: useCacheableLookup ? cacheableLookup.lookup : dns.lookup,
},
}),
...(body instanceof FormData
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 9d344e768..eb063538c 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -464,6 +464,7 @@ app.get("/liveness", (req, res) => {
logger: _logger,
abort: AbortSignal.timeout(5000),
ignoreResponse: true,
+ useCacheableLookup: false,
})
.then(() => {
currentLiveness = true;