Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Avoid startup data layer warmup (#3899)
Abimael Martellc8496fb26980
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 1163a09e7..3aca6f20b 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -38,7 +38,6 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forciimport responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";-import { warmDataLayerCapabilities } from "./lib/data-layer";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -120,7 +119,6 @@ async function startServer(port = DEFAULT_PORT) {try {await initializeBlocklist();initializeEngineForcing();- await warmDataLayerCapabilities();} catch (error) {logger.error("Failed to initialize API startup dependencies", {error,diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.tsindex 9a0baaf20..ca9a16193 100644--- a/apps/api/src/lib/data-layer.test.ts+++ b/apps/api/src/lib/data-layer.test.ts@@ -12,7 +12,6 @@ import {isSuccessfulDataLayerStatusCode,isSupportedDataLayerFormatRequest,setDataLayerCapabilitiesForTest,- warmDataLayerCapabilities,} from "./data-layer";vi.mock("undici", () => ({@@ -70,25 +69,6 @@ describe("data layer routing", () => {expect(fetch).toHaveBeenCalledTimes(1);});- it("warms capabilities before request-time URL checks", async () => {- clearDataLayerCapabilitiesForTest();- vi.mocked(fetch).mockResolvedValue({- ok: true,- status: 200,- json: async () => ({- ttlSeconds: 300,- domains: ["profiles.example"],- }),- } as Awaited<ReturnType<typeof fetch>>);-- await warmDataLayerCapabilities();- await expect(- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),- ).resolves.toBe(true);-- expect(fetch).toHaveBeenCalledTimes(1);- });-it("builds a compact request log context", () => {expect(getDataLayerRequestLogContext(diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.tsindex 30a7e010c..b251a6422 100644--- a/apps/api/src/lib/data-layer.ts+++ b/apps/api/src/lib/data-layer.ts@@ -131,19 +131,6 @@ async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null>return capabilities;}-export async function warmDataLayerCapabilities(): Promise<void> {- if (!config.FIRE_ENGINE_BETA_URL) {- return;- }-- const capabilities = await getDataLayerCapabilities();- rootLogger.info("Data layer capabilities warmup completed", {- available: capabilities !== null,- domains: capabilities?.domains.size ?? 0,- baseDomains: capabilities?.baseDomains.size ?? 0,- });-}-function dataLayerCapabilitiesMatchUrl(capabilities: DataLayerCapabilities,inputUrl: string,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 0fd9eee1b..fa2366f59 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -21,7 +21,6 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";-import { warmDataLayerCapabilities } from "../lib/data-layer";import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";@@ -466,7 +465,6 @@ app.listen(workerPort, (error?: Error) => {});initializeEngineForcing();- await warmDataLayerCapabilities();if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {monitorSchedulerInterval = setInterval(() => {diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex b58df257f..63dfd6a91 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -7,7 +7,6 @@ import { register } from "prom-client";import Express from "express";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";-import { warmDataLayerCapabilities } from "../../lib/data-layer";export type WorkerQueue = {getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;@@ -59,7 +58,6 @@ export async function runNuqWorker(options: {try {await initializeBlocklist();initializeEngineForcing();- await warmDataLayerCapabilities();await options.beforeStart?.();} catch (error) {_logger.error("Failed to initialize NuQ worker", {