Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Avoid startup data layer warmup (#3899)

Abimael Martellc8496fb26980
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 1163a09e7..3aca6f20b 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -38,7 +38,6 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forci
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
-import { warmDataLayerCapabilities } from "./lib/data-layer";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -120,7 +119,6 @@ async function startServer(port = DEFAULT_PORT) {
try {
await initializeBlocklist();
initializeEngineForcing();
- await warmDataLayerCapabilities();
} catch (error) {
logger.error("Failed to initialize API startup dependencies", {
error,
diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.ts
index 9a0baaf20..ca9a16193 100644
--- a/apps/api/src/lib/data-layer.test.ts
+++ b/apps/api/src/lib/data-layer.test.ts
@@ -12,7 +12,6 @@ import {
isSuccessfulDataLayerStatusCode,
isSupportedDataLayerFormatRequest,
setDataLayerCapabilitiesForTest,
- warmDataLayerCapabilities,
} from "./data-layer";
vi.mock("undici", () => ({
@@ -70,25 +69,6 @@ describe("data layer routing", () => {
expect(fetch).toHaveBeenCalledTimes(1);
});
- it("warms capabilities before request-time URL checks", async () => {
- clearDataLayerCapabilitiesForTest();
- vi.mocked(fetch).mockResolvedValue({
- ok: true,
- status: 200,
- json: async () => ({
- ttlSeconds: 300,
- domains: ["profiles.example"],
- }),
- } as Awaited<ReturnType<typeof fetch>>);
-
- await warmDataLayerCapabilities();
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
- ).resolves.toBe(true);
-
- expect(fetch).toHaveBeenCalledTimes(1);
- });
-
it("builds a compact request log context", () => {
expect(
getDataLayerRequestLogContext(
diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.ts
index 30a7e010c..b251a6422 100644
--- a/apps/api/src/lib/data-layer.ts
+++ b/apps/api/src/lib/data-layer.ts
@@ -131,19 +131,6 @@ async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null>
return capabilities;
}
-export async function warmDataLayerCapabilities(): Promise<void> {
- if (!config.FIRE_ENGINE_BETA_URL) {
- return;
- }
-
- const capabilities = await getDataLayerCapabilities();
- rootLogger.info("Data layer capabilities warmup completed", {
- available: capabilities !== null,
- domains: capabilities?.domains.size ?? 0,
- baseDomains: capabilities?.baseDomains.size ?? 0,
- });
-}
-
function dataLayerCapabilitiesMatchUrl(
capabilities: DataLayerCapabilities,
inputUrl: string,
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 0fd9eee1b..fa2366f59 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -21,7 +21,6 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
-import { warmDataLayerCapabilities } from "../lib/data-layer";
import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
@@ -466,7 +465,6 @@ app.listen(workerPort, (error?: Error) => {
});
initializeEngineForcing();
- await warmDataLayerCapabilities();
if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {
monitorSchedulerInterval = setInterval(() => {
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
index b58df257f..63dfd6a91 100644
--- a/apps/api/src/services/worker/nuq-worker-runner.ts
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -7,7 +7,6 @@ import { register } from "prom-client";
import Express from "express";
import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";
-import { warmDataLayerCapabilities } from "../../lib/data-layer";
export type WorkerQueue = {
getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;
@@ -59,7 +58,6 @@ export async function runNuqWorker(options: {
try {
await initializeBlocklist();
initializeEngineForcing();
- await warmDataLayerCapabilities();
await options.beforeStart?.();
} catch (error) {
_logger.error("Failed to initialize NuQ worker", {