Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "feat: use cacheable lookup everywhere (#1559)"
Gergő Móriczd46ba959247a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex cdde350bc..febad61a4 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -35,7 +35,6 @@ env:ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}USE_GO_MARKDOWN_PARSER: true- SENTRY_ENVIRONMENT: devjobs:test:@@ -54,7 +53,6 @@ jobs:oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}tags: tag:ci- use-cache: 'true'- name: Install pnpmuses: pnpm/action-setup@v4with:diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 933a980f6..efca04793 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -18,6 +18,7 @@ import { logger } from "./lib/logger";import { adminRouter } from "./routes/admin";import http from "node:http";import https from "node:https";+import CacheableLookup from "cacheable-lookup";import { v1Router } from "./routes/v1";import expressWs from "express-ws";import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";@@ -25,7 +26,6 @@ import { ZodError } from "zod";import { v4 as uuidv4 } from "uuid";import { RateLimiterMode } from "./types";import { attachWsProxy } from "./services/agentLivecastWS";-import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";const { createBullBoard } = require("@bull-board/api");const { BullAdapter } = require("@bull-board/api/bullAdapter");@@ -34,9 +34,11 @@ const { ExpressAdapter } = require("@bull-board/express");const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;logger.info(`Number of CPUs: ${numCPUs} available`);+const cacheable = new CacheableLookup();+// Install cacheable lookup for all other requests-cacheableLookup.install(http.globalAgent);-cacheableLookup.install(https.globalAgent);+cacheable.install(http.globalAgent);+cacheable.install(https.globalAgent);// Initialize Express with WebSocket supportconst expressApp = express();diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 1a959224e..351a7742f 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -2,7 +2,6 @@ import type { Socket } from "net";import type { TLSSocket } from "tls";import * as undici from "undici";import { Address6 } from "ip-address";-import { cacheableLookup } from "../../lib/cacheableLookup";export class InsecureConnectionError extends Error {constructor() {@@ -47,7 +46,7 @@ export function makeSecureDispatcher(const agentOpts: undici.Agent.Options = {connect: {rejectUnauthorized: false, // bypass SSL failures -- this is fine- lookup: cacheableLookup.lookup,+ // lookup: secureLookup,},maxRedirections: 5000,...options,diff --git a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.tsdeleted file mode 100644index ef4f3fef1..000000000--- a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts+++ /dev/null@@ -1,4 +0,0 @@-import CacheableLookup from 'cacheable-lookup';-import dns from 'dns';--export const cacheableLookup = (process.env.SENTRY_ENVIRONMENT === "dev" ? { lookup: dns.lookup, install: () => {} } : new CacheableLookup({ lookup: false }));diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 24822f415..c64e79532 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -5,7 +5,6 @@ import { MockState, saveMock } from "./mock";import { TimeoutSignal } from "../../../controllers/v1/types";import { fireEngineURL } from "../engines/fire-engine/scrape";import { fetch, RequestInit, Response, FormData, Agent } from "undici";-import { cacheableLookup } from "./cacheableLookup";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -83,9 +82,6 @@ export async function robustFetch<dispatcher: new Agent({headersTimeout: 0,bodyTimeout: 0,- connect: {- lookup: cacheableLookup.lookup,- },}),...(body instanceof FormData? {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 553b944fd..04181c9ea 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -82,9 +82,6 @@ import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-fimport { CostTracking } from "../lib/extract/extraction-service";import { getACUCTeam } from "../controllers/auth";import Express from "express";-import http from "http";-import https from "https";-import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";configDotenv();@@ -112,10 +109,6 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();-// Install cacheable lookup for all other requests-cacheableLookup.install(http.globalAgent);-cacheableLookup.install(https.globalAgent);-async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const logger = _logger.child({module: "queue-worker",