Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Mog/cachable lookup (#1560)
Gergő Móriczbd9673e104ca
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex febad61a4..cdde350bc 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -35,6 +35,7 @@ env:ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}USE_GO_MARKDOWN_PARSER: true+ SENTRY_ENVIRONMENT: devjobs:test:@@ -53,6 +54,7 @@ jobs:oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}tags: tag:ci+ use-cache: 'true'- name: Install pnpmuses: pnpm/action-setup@v4with:diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex efca04793..933a980f6 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -18,7 +18,6 @@ import { logger } from "./lib/logger";import { adminRouter } from "./routes/admin";import http from "node:http";import https from "node:https";-import CacheableLookup from "cacheable-lookup";import { v1Router } from "./routes/v1";import expressWs from "express-ws";import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";@@ -26,6 +25,7 @@ import { ZodError } from "zod";import { v4 as uuidv4 } from "uuid";import { RateLimiterMode } from "./types";import { attachWsProxy } from "./services/agentLivecastWS";+import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";const { createBullBoard } = require("@bull-board/api");const { BullAdapter } = require("@bull-board/api/bullAdapter");@@ -34,11 +34,9 @@ const { ExpressAdapter } = require("@bull-board/express");const numCPUs = process.env.ENV === "local" ? 2 : os.cpus().length;logger.info(`Number of CPUs: ${numCPUs} available`);-const cacheable = new CacheableLookup();-// Install cacheable lookup for all other requests-cacheable.install(http.globalAgent);-cacheable.install(https.globalAgent);+cacheableLookup.install(http.globalAgent);+cacheableLookup.install(https.globalAgent);// Initialize Express with WebSocket supportconst expressApp = express();diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 351a7742f..1a959224e 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -2,6 +2,7 @@ import type { Socket } from "net";import type { TLSSocket } from "tls";import * as undici from "undici";import { Address6 } from "ip-address";+import { cacheableLookup } from "../../lib/cacheableLookup";export class InsecureConnectionError extends Error {constructor() {@@ -46,7 +47,7 @@ export function makeSecureDispatcher(const agentOpts: undici.Agent.Options = {connect: {rejectUnauthorized: false, // bypass SSL failures -- this is fine- // lookup: secureLookup,+ lookup: cacheableLookup.lookup,},maxRedirections: 5000,...options,diff --git a/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.tsnew file mode 100644index 000000000..b561967c8--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/cacheableLookup.ts@@ -0,0 +1,4 @@+import CacheableLookup from 'cacheable-lookup';+import dns from 'dns';++export const cacheableLookup = (process.env.SENTRY_ENVIRONMENT === "dev" ? { lookup: dns.lookup, install: () => {} } : new CacheableLookup({}));diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex c64e79532..24822f415 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -5,6 +5,7 @@ import { MockState, saveMock } from "./mock";import { TimeoutSignal } from "../../../controllers/v1/types";import { fireEngineURL } from "../engines/fire-engine/scrape";import { fetch, RequestInit, Response, FormData, Agent } from "undici";+import { cacheableLookup } from "./cacheableLookup";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -82,6 +83,9 @@ export async function robustFetch<dispatcher: new Agent({headersTimeout: 0,bodyTimeout: 0,+ connect: {+ lookup: cacheableLookup.lookup,+ },}),...(body instanceof FormData? {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 04181c9ea..553b944fd 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -82,6 +82,9 @@ import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-fimport { CostTracking } from "../lib/extract/extraction-service";import { getACUCTeam } from "../controllers/auth";import Express from "express";+import http from "http";+import https from "https";+import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";configDotenv();@@ -109,6 +112,10 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();+// Install cacheable lookup for all other requests+cacheableLookup.install(http.globalAgent);+cacheableLookup.install(https.globalAgent);+async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const logger = _logger.child({module: "queue-worker",