Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix sitemap parsing (ENG-3361) (#2085)
tom688bf9372352
diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.tsnew file mode 100644index 000000000..b9c9c5f1d--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts@@ -0,0 +1,62 @@+import { createRobotsChecker, fetchRobotsTxt } from "../../../lib/robots-txt";+import { scrapeTimeout } from "./lib";++describe("Robots.txt tests", () => {+ const robotsUrl = "https://www.google.com/robots.txt";+ const siteUrl = "https://www.google.com/";+ let robotsTxt: string;+ let checker: ReturnType<typeof createRobotsChecker>;++ beforeAll(async () => {+ robotsTxt = await fetchRobotsTxt(robotsUrl);+ checker = createRobotsChecker(siteUrl, robotsTxt);+ }, scrapeTimeout);++ it(+ "contains expected directives",+ () => {+ expect(robotsTxt).toContain("User-agent");+ expect(robotsTxt).toContain("Disallow");+ },+ scrapeTimeout,+ );++ it("allows and disallows URLs as per rules", () => {+ expect(+ checker.robots.isAllowed(+ "https://www.google.com/search?q=test",+ "FireCrawlAgent",+ ),+ ).toBe(false);++ expect(+ checker.robots.isAllowed(+ "https://www.google.com/search/about",+ "FireCrawlAgent",+ ),+ ).toBe(true);++ expect(+ checker.robots.isAllowed(+ "https://www.google.com/groups",+ "FireCrawlAgent",+ ),+ ).toBe(false);++ expect(+ checker.robots.isAllowed(+ "https://www.google.com/preferences",+ "FireCrawlAgent",+ ),+ ).toBe(true);+ });++ it("includes Google's sitemap URLs", () => {+ const sitemaps = checker.robots.getSitemaps();++ expect(Array.isArray(sitemaps)).toBe(true);+ expect(sitemaps.length).toBeGreaterThan(0);++ expect(sitemaps).toContain("https://www.google.com/sitemap.xml");+ });+});diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex fd94343d2..a92355372 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -21,7 +21,11 @@ import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";import { logger } from "../../lib/logger";-import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";+import {+ generateURLSplits,+ queryIndexAtDomainSplitLevelWithMeta,+ queryIndexAtSplitLevelWithMeta,+} from "../../services/index";import { redisEvictConnection } from "../../services/redis";import { performCosineSimilarityV2 } from "../../lib/map-cosine";import { MapTimeoutError } from "../../lib/error";@@ -39,19 +43,24 @@ interface MapResult {}function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {- let urlSet = new Set<string>(documents.map(x => x.url));+ let urlSet = new Set<string>(documents.map((x) => x.url));let newDocuments: MapDocument[] = [- ...Array.from(urlSet).map(x => {- let localDocs = documents.filter(y => y.url === x);- return localDocs.find(x => x.title !== undefined) || localDocs[0];+ ...Array.from(urlSet).map((x) => {+ let localDocs = documents.filter((y) => y.url === x);+ return localDocs.find((x) => x.title !== undefined) || localDocs[0];}),];return newDocuments;}-async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<MapDocument[]> {+async function queryIndex(+ url: string,+ limit: number,+ useIndex: boolean,+ includeSubdomains: boolean,+): Promise<MapDocument[]> {if (!useIndex) {return [];}@@ -63,13 +72,15 @@ async function queryIndex(url: string, limit: number, useIndex: boolean, include// TEMP: this should be altered on June 15th 2025 7AM PT - mogeryconst [domainLinks, splitLinks] = await Promise.all([- includeSubdomains ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit) : [],+ includeSubdomains+ ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)+ : [],queryIndexAtSplitLevelWithMeta(url, limit),]);return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);} else {- return (await queryIndexAtSplitLevelWithMeta(url, limit));+ return await queryIndexAtSplitLevelWithMeta(url, limit);}}@@ -181,10 +192,14 @@ export async function getMapResults({pagePromises = JSON.parse(cachedResult);} else {const fetchPage = async (page: number) => {- return fireEngineMap(mapUrl, {- numResults: resultsPerPage,- page: page,- }, abort);+ return fireEngineMap(+ mapUrl,+ {+ numResults: resultsPerPage,+ page: page,+ },+ abort,+ );};pagePromises = Array.from({ length: maxPages }, (_, i) =>@@ -199,7 +214,12 @@ export async function getMapResults({]);if (!zeroDataRetention) {- await redisEvictConnection.set(cacheKey, JSON.stringify(searchResults), "EX", 48 * 60 * 60); // Cache for 48 hours+ await redisEvictConnection.set(+ cacheKey,+ JSON.stringify(searchResults),+ "EX",+ 48 * 60 * 60,+ ); // Cache for 48 hours}if (indexResults.length > 0) {@@ -212,9 +232,11 @@ export async function getMapResults({try {await crawler.tryGetSitemap((urls) => {- mapResults.push(...urls.map(x => ({- url: x,- })));+ mapResults.push(+ ...urls.map((x) => ({+ url: x,+ })),+ );},true,false,@@ -227,17 +249,25 @@ export async function getMapResults({}if (search) {- mapResults = searchResults.flat().map<MapDocument>(x => ({- url: x.url,- title: x.title,- description: x.description,- }) satisfies MapDocument).concat(mapResults);+ mapResults = searchResults+ .flat()+ .map<MapDocument>(+ (x) =>+ ({+ url: x.url,+ title: x.title,+ description: x.description,+ }) satisfies MapDocument,+ )+ .concat(mapResults);} else {- mapResults = mapResults.concat(searchResults.flat().map(x => ({- url: x.url,- title: x.title,- description: x.description,- })));+ mapResults = mapResults.concat(+ searchResults.flat().map((x) => ({+ url: x.url,+ title: x.title,+ description: x.description,+ })),+ );}const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);@@ -255,14 +285,17 @@ export async function getMapResults({try {return {...x,- url: checkAndUpdateURLForMap(x.url, crawlerOptions.ignoreQueryParameters ?? true).url.trim(),+ url: checkAndUpdateURLForMap(+ x.url,+ crawlerOptions.ignoreQueryParameters ?? true,+ ).url.trim(),};} catch (_) {return null;}}).filter((x) => x !== null) as MapDocument[];-+// allows for subdomains to be includedmapResults = mapResults.filter((x) => isSameDomain(x.url, url));@@ -278,8 +311,8 @@ export async function getMapResults({const urlPath = urlObj.pathname;// Only apply path filtering if the URL has a significant path (not just '/' or empty)// This means we only filter by path if the user has not selected a root domain- if (urlPath && urlPath !== '/' && urlPath.length > 1) {- mapResults = mapResults.filter(x => {+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {+ mapResults = mapResults.filter((x) => {try {const linkObj = new URL(x.url);return linkObj.pathname.startsWith(urlPath);@@ -290,7 +323,9 @@ export async function getMapResults({}} catch (e) {// If URL parsing fails, continue without path filtering- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });+ logger.warn(`Failed to parse URL for path filtering: ${url}`, {+ error: e,+ });}}@@ -323,7 +358,7 @@ export async function mapController(let result: Awaited<ReturnType<typeof getMapResults>>;const abort = new AbortController();try {- result = await Promise.race([+ result = (await Promise.race([getMapResults({url: req.body.url,search: req.body.search,@@ -341,13 +376,17 @@ export async function mapController(flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,}),- ...(req.body.timeout !== undefined ? [- new Promise((resolve, reject) => setTimeout(() => {- abort.abort(new MapTimeoutError());- reject(new MapTimeoutError());- }, req.body.timeout))- ] : []),- ]) as any;+ ...(req.body.timeout !== undefined+ ? [+ new Promise((resolve, reject) =>+ setTimeout(() => {+ abort.abort(new MapTimeoutError());+ reject(new MapTimeoutError());+ }, req.body.timeout),+ ),+ ]+ : []),+ ])) as any;} catch (error) {if (error instanceof MapTimeoutError) {return res.status(408).json({@@ -361,7 +400,12 @@ export async function mapController(}// Bill the team- billTeam(req.auth.team_id, req.acuc?.sub_id ?? undefined, 1, req.acuc?.api_key_id ?? null).catch((error) => {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ 1,+ req.acuc?.api_key_id ?? null,+ ).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,);@@ -393,4 +437,4 @@ export async function mapController(};return res.status(200).json(response);-}\ No newline at end of file+}diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 0b601d87c..300909b4a 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -1,8 +1,7 @@import robotsParser, { Robot } from "robots-parser";-import axios from "axios";-import { axiosTimeout } from "./timeout";-import https from "https";+import * as undici from "undici";import { Logger } from "winston";+import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";export interface RobotsTxtChecker {robotsTxtUrl: string;@@ -18,30 +17,29 @@ export async function fetchRobotsTxt(const urlObj = new URL(url);const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;- let extraArgs: any = {};- if (skipTlsVerification) {- extraArgs.httpsAgent = new https.Agent({- rejectUnauthorized: false,- });- }-- const response = await axios.get(robotsTxtUrl, {- timeout: axiosTimeout,+ const response = await undici.fetch(robotsTxtUrl, {signal: abort,- ...extraArgs,+ dispatcher: getSecureDispatcher(skipTlsVerification),});- const contentType = (Object.entries(response.headers).find(- (x) => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? "";+ if (!response.ok) {+ throw new Error(+ `Failed to fetch robots.txt: ${response.status} ${response.statusText}`,+ );+ }++ const content = await response.text();+ const contentType = response.headers.get("content-type") || "";- if ((contentType.includes("text/html") && response.data.trim().startsWith("<")) ||- contentType.includes("application/json") ||- contentType.includes("application/xml")) {+ if (+ (contentType.includes("text/html") && content.trim().startsWith("<")) ||+ contentType.includes("application/json") ||+ contentType.includes("application/xml")+ ) {return "";}- return response.data;+ return content;}export function createRobotsChecker(@@ -67,7 +65,7 @@ export function isUrlAllowedByRobots(for (const userAgent of userAgents) {let isAllowed = robots.isAllowed(url, userAgent);-+// Handle null/undefined responses - default to true (allowed)if (isAllowed === null || isAllowed === undefined) {isAllowed = true;@@ -82,11 +80,11 @@ export function isUrlAllowedByRobots(if (isAllowed && !url.endsWith("/")) {const urlWithSlash = url + "/";let isAllowedWithSlash = robots.isAllowed(urlWithSlash, userAgent);-+if (isAllowedWithSlash == null) {isAllowedWithSlash = true;}-+// If the trailing slash version is explicitly disallowed, block itif (isAllowedWithSlash === false) {isAllowed = false;diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 272238f43..00d29a55e 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -1,17 +1,19 @@-import axios, { AxiosError } from "axios";+import { AxiosError } from "axios";import { load } from "cheerio"; // rustifiedimport { URL } from "url";import { getLinksFromSitemap } from "./sitemap";import robotsParser, { Robot } from "robots-parser";import psl from "psl";import { getURLDepth } from "./utils/maxDepthUtils";-import { axiosTimeout } from "../../lib/timeout";import { logger as _logger } from "../../lib/logger";-import https from "https";import { redisEvictConnection } from "../../services/redis";import { extractLinks } from "../../lib/html-transformer";import { filterLinks } from "../../lib/crawler";-import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";+import {+ fetchRobotsTxt,+ createRobotsChecker,+ isUrlAllowedByRobots,+} from "../../lib/robots-txt";import { ScrapeJobTimeoutError } from "../../lib/error";export const SITEMAP_LIMIT = 100;@@ -32,7 +34,7 @@ export enum DenialReason {BACKWARD_CRAWLING = "URL cannot be crawled unless crawlEntireDomain is set to true",SOCIAL_MEDIA = "URL is a social media or email link",EXTERNAL_LINK = "External URL not allowed",- SECTION_LINK = "URL contains section anchor (#)"+ SECTION_LINK = "URL contains section anchor (#)",}export interface FilterLinksResult {@@ -122,7 +124,11 @@ export class WebCrawler {this.ignoreRobotsTxt = ignoreRobotsTxt ?? false;this.regexOnFullURL = regexOnFullURL ?? false;this.zeroDataRetention = zeroDataRetention ?? false;- this.logger = _logger.child({ crawlId: this.jobId, module: "WebCrawler", zeroDataRetention: this.zeroDataRetention });+ this.logger = _logger.child({+ crawlId: this.jobId,+ module: "WebCrawler",+ zeroDataRetention: this.zeroDataRetention,+ });this.maxDiscoveryDepth = maxDiscoveryDepth;this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;}@@ -136,8 +142,11 @@ export class WebCrawler {const denialReasons = new Map<string, string>();if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {- this.logger.debug("Max discovery depth hit, filtering off all links", { currentDiscoveryDepth: this.currentDiscoveryDepth, maxDiscoveryDepth: this.maxDiscoveryDepth });- sitemapLinks.forEach(link => {+ this.logger.debug("Max discovery depth hit, filtering off all links", {+ currentDiscoveryDepth: this.currentDiscoveryDepth,+ maxDiscoveryDepth: this.maxDiscoveryDepth,+ });+ sitemapLinks.forEach((link) => {denialReasons.set(link, "Maximum discovery depth reached");});return { links: [], denialReasons };@@ -275,7 +284,9 @@ export class WebCrawler {!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)) {if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {- this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`);+ this.logger.debug(+ `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,+ );}denialReasons.set(link, DenialReason.BACKWARD_CRAWLING);return false;@@ -284,7 +295,9 @@ export class WebCrawler {const isAllowed = this.ignoreRobotsTxt? true- : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);+ : ((this.robots.isAllowed(link, "FireCrawlAgent") ||+ this.robots.isAllowed(link, "FirecrawlAgent")) ??+ true);// Check if the link is disallowed by robots.txtif (!isAllowed) {this.logger.debug(`Link disallowed by robots.txt: ${link}`, {@@ -316,7 +329,10 @@ export class WebCrawler {return { links: filteredLinks, denialReasons };}- public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {+ public async getRobotsTxt(+ skipTlsVerification = false,+ abort?: AbortSignal,+ ): Promise<string> {return fetchRobotsTxt(this.initialUrl, skipTlsVerification, abort);}@@ -325,7 +341,9 @@ export class WebCrawler {const checker = createRobotsChecker(this.initialUrl, txt);this.robots = checker.robots;this.robotsTxtUrl = checker.robotsTxtUrl;- const delay = this.robots.getCrawlDelay("FireCrawlAgent") || this.robots.getCrawlDelay("FirecrawlAgent");+ const delay =+ this.robots.getCrawlDelay("FireCrawlAgent") ||+ this.robots.getCrawlDelay("FirecrawlAgent");this.robotsCrawlDelay = delay !== undefined ? delay : null;}@@ -360,7 +378,9 @@ export class WebCrawler {return await urlsHandler(urls);} else {let filteredLinksResult = await this.filterLinks(- [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl).allowed),+ [...new Set(urls)].filter(+ (x) => this.filterURL(x, this.initialUrl).allowed,+ ),leftOfLimit,this.maxCrawledDepth,fromMap,@@ -396,15 +416,23 @@ export class WebCrawler {// Allow sitemaps to be cached for 48 hours if they are requested from /map// - mogery- const maxAge = (fromMap && !onlySitemap) ? 48 * 60 * 60 * 1000 : 0;+ const maxAge = fromMap && !onlySitemap ? 48 * 60 * 60 * 1000 : 0;try {let count = (await Promise.race([Promise.all([- this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock, maxAge),+ this.tryFetchSitemapLinks(+ this.initialUrl,+ _urlsHandler,+ abort,+ mock,+ maxAge,+ ),...this.robots.getSitemaps()- .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge)),+ .map((x) =>+ this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge),+ ),]).then((results) => results.reduce((a, x) => a + x, 0)),timeoutPromise,])) as number;@@ -560,18 +588,28 @@ export class WebCrawler {public async extractLinksFromHTML(html: string, url: string) {try {- return [...new Set((await this.extractLinksFromHTMLRust(html, url)).map(x => {- try {- return new URL(x, url).href- } catch (e) {- return null;- }- }).filter(x => x !== null) as string[])];+ return [+ ...new Set(+ (await this.extractLinksFromHTMLRust(html, url))+ .map((x) => {+ try {+ return new URL(x, url).href;+ } catch (e) {+ return null;+ }+ })+ .filter((x) => x !== null) as string[],+ ),+ ];} catch (error) {- this.logger.warn("Failed to call html-transformer! Falling back to cheerio...", {- error,- module: "scrapeURL", method: "extractMetadata"- });+ this.logger.warn(+ "Failed to call html-transformer! Falling back to cheerio...",+ {+ error,+ module: "scrapeURL",+ method: "extractMetadata",+ },+ );}return this.extractLinksFromHTMLCheerio(html, url);@@ -581,9 +619,7 @@ export class WebCrawler {url: string,ignoreRobotsTxt: boolean = false,): boolean {- return ignoreRobotsTxt- ? true- : isUrlAllowedByRobots(url, this.robots);+ return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);}private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {@@ -636,13 +672,13 @@ export class WebCrawler {if (!link.includes("#")) {return true;}-+// Check if the hash fragment looks like a route (contains forward slashes and has substantial content)const hashPart = link.split("#")[1];if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {return true;}-+return false;}@@ -660,14 +696,14 @@ export class WebCrawler {try {const linkUrl = new URL(link, this.baseUrl);const baseUrl = new URL(this.baseUrl);-+const linkParsed = psl.parse(linkUrl.hostname);const baseParsed = psl.parse(baseUrl.hostname);-+if (!linkParsed?.domain || !baseParsed?.domain) {return false;}-+return linkParsed.domain === baseParsed.domain;} catch (error) {return false;@@ -749,7 +785,13 @@ export class WebCrawler {// Try to get sitemap from the provided URL firsttry {sitemapCount = await getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },+ {+ sitemapUrl,+ urlsHandler,+ mode: "fire-engine",+ maxAge,+ zeroDataRetention: this.zeroDataRetention,+ },this.logger,this.jobId,this.sitemapsHit,@@ -792,7 +834,7 @@ export class WebCrawler {try {const linkUrl = new URL(link);return linkUrl.hostname.endsWith(hostname);- } catch { }+ } catch {}}),);},@@ -834,7 +876,13 @@ export class WebCrawler {const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;try {sitemapCount += await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },+ {+ sitemapUrl: baseUrlSitemap,+ urlsHandler,+ mode: "fire-engine",+ maxAge,+ zeroDataRetention: this.zeroDataRetention,+ },this.logger,this.jobId,this.sitemapsHit,@@ -854,7 +902,13 @@ export class WebCrawler {// ignore 404} else {sitemapCount += await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },+ {+ sitemapUrl: baseUrlSitemap,+ urlsHandler,+ mode: "fire-engine",+ maxAge,+ zeroDataRetention: this.zeroDataRetention,+ },this.logger,this.jobId,this.sitemapsHit,@@ -867,7 +921,10 @@ export class WebCrawler {}if (this.sitemapsHit.size >= SITEMAP_LIMIT) {- this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });+ this.logger.warn("Sitemap limit hit!", {+ crawlId: this.jobId,+ url: this.baseUrl,+ });}return sitemapCount;diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 99bd7c545..b22e41e7c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -4,9 +4,8 @@ import { Meta } from "../..";import { SSLError } from "../../error";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import {+ getSecureDispatcher,InsecureConnectionError,- secureDispatcher,- secureDispatcherSkipTlsVerification,} from "../utils/safeFetch";import { MockState, saveMock } from "../../lib/mock";import { TextDecoder } from "util";@@ -26,7 +25,7 @@ export async function scrapeURLWithFetch(let response: {url: string;- body: string,+ body: string;status: number;headers: [string, string][];};@@ -53,7 +52,7 @@ export async function scrapeURLWithFetch(} else {try {const x = await undici.fetch(meta.rewrittenUrl ?? meta.url, {- dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,+ dispatcher: getSecureDispatcher(meta.options.skipTlsVerification),redirect: "follow",headers: meta.options.headers,signal: meta.abort.asSignal(),@@ -61,13 +60,18 @@ export async function scrapeURLWithFetch(const buf = Buffer.from(await x.arrayBuffer());let text = buf.toString("utf8");- const charset = (text.match(/<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i) ?? [])[1]+ const charset = (text.match(+ /<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i,+ ) ?? [])[1];try {if (charset) {text = new TextDecoder(charset.trim()).decode(buf);}} catch (error) {- meta.logger.warn("Failed to re-parse with correct charset", { charset, error })+ meta.logger.warn("Failed to re-parse with correct charset", {+ charset,+ error,+ });}response = {@@ -78,10 +82,7 @@ export async function scrapeURLWithFetch(};if (meta.mock === null) {- await saveMock(- mockOptions,- response,- );+ await saveMock(mockOptions, response);}} catch (error) {if (@@ -89,7 +90,12 @@ export async function scrapeURLWithFetch(error.cause instanceof InsecureConnectionError) {throw error.cause;- } else if (error instanceof Error && error.message === "fetch failed" && error.cause && (error.cause as any).code === "CERT_HAS_EXPIRED") {+ } else if (+ error instanceof Error &&+ error.message === "fetch failed" &&+ error.cause &&+ (error.cause as any).code === "CERT_HAS_EXPIRED"+ ) {throw new SSLError(meta.options.skipTlsVerification);} else {throw error;@@ -106,9 +112,9 @@ export async function scrapeURLWithFetch(url: response.url,html: response.body,statusCode: response.status,- contentType: (response.headers.find(- (x) => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,+ contentType:+ (response.headers.find((x) => x[0].toLowerCase() === "content-type") ??+ [])[1] ?? undefined,proxyUsed: "basic",};diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex 4ccd52c53..87bbdccda 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -5,7 +5,7 @@ import { EngineError } from "../../error";import { Writable } from "stream";import { v4 as uuid } from "uuid";import * as undici from "undici";-import { secureDispatcher } from "./safeFetch";+import { getSecureDispatcher } from "./safeFetch";export async function fetchFileToBuffer(url: string,@@ -17,7 +17,7 @@ export async function fetchFileToBuffer(const response = await undici.fetch(url, {...init,redirect: "follow",- dispatcher: secureDispatcher,+ dispatcher: getSecureDispatcher(),});return {response,@@ -40,7 +40,7 @@ export async function downloadFile(const response = await undici.fetch(url, {...init,redirect: "follow",- dispatcher: secureDispatcher,+ dispatcher: getSecureDispatcher(),});// This should never happen in the current state of JS/Undici (2024), but let's check anyways.diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex b18d9af7c..406f5a833 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -18,7 +18,7 @@ export function isIPPrivate(address: string): boolean {return addr.range() !== "unicast";}-export function makeSecureDispatcher(skipTlsVerification: boolean = false) {+export function makeSecureDispatcher(skipTlsVerification: boolean) {const agentOpts: undici.Agent.Options = {maxRedirections: 5000,};@@ -62,5 +62,8 @@ export function makeSecureDispatcher(skipTlsVerification: boolean = false) {return agent;}-export const secureDispatcher = makeSecureDispatcher(false);-export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);+const secureDispatcher = makeSecureDispatcher(false);+const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);++export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>+ skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.tsindex 005d3061b..cdc4046a5 100644--- a/apps/api/src/services/webhook.ts+++ b/apps/api/src/services/webhook.ts@@ -1,5 +1,4 @@import undici from "undici";-import { isIPPrivate, secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";import { logger as _logger, logger } from "../lib/logger";import { supabase_rr_service, supabase_service } from "./supabase";import { WebhookEventType } from "../types";@@ -7,22 +6,30 @@ import { configDotenv } from "dotenv";import { z } from "zod";import { webhookSchema } from "../controllers/v1/types";import { redisEvictConnection } from "./redis";+import { getSecureDispatcher, isIPPrivate } from "../scraper/scrapeURL/engines/utils/safeFetch";configDotenv();const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";const WEBHOOK_INSERT_BATCH_SIZE = 1000;async function addWebhookInsertJob(data: any) {- await redisEvictConnection.rpush(WEBHOOK_INSERT_QUEUE_KEY, JSON.stringify(data));+ await redisEvictConnection.rpush(+ WEBHOOK_INSERT_QUEUE_KEY,+ JSON.stringify(data),+ );}export async function getWebhookInsertQueueLength(): Promise<number> {- return await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY) ?? 0;+ return (await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY)) ?? 0;}async function getWebhookInsertJobs(): Promise<any[]> {- const jobs = (await redisEvictConnection.lpop(WEBHOOK_INSERT_QUEUE_KEY, WEBHOOK_INSERT_BATCH_SIZE)) ?? [];- return jobs.map(x => JSON.parse(x));+ const jobs =+ (await redisEvictConnection.lpop(+ WEBHOOK_INSERT_QUEUE_KEY,+ WEBHOOK_INSERT_BATCH_SIZE,+ )) ?? [];+ return jobs.map((x) => JSON.parse(x));}export async function processWebhookInsertJobs() {@@ -30,12 +37,17 @@ export async function processWebhookInsertJobs() {if (jobs.length === 0) {return;}- logger.info(`Webhook inserter found jobs to insert`, { jobCount: jobs.length });+ logger.info(`Webhook inserter found jobs to insert`, {+ jobCount: jobs.length,+ });try {await supabase_service.from("webhook_logs").insert(jobs);logger.info(`Webhook inserter inserted jobs`, { jobCount: jobs.length });} catch (error) {- logger.error(`Webhook inserter failed to insert jobs`, { error, jobCount: jobs.length });+ logger.error(`Webhook inserter failed to insert jobs`, {+ error,+ jobCount: jobs.length,+ });}}@@ -47,7 +59,7 @@ async function logWebhook(data: {scrapeId?: string;url: string;statusCode?: number;- event: WebhookEventType+ event: WebhookEventType;}) {try {await addWebhookInsertJob({@@ -61,7 +73,15 @@ async function logWebhook(data: {event: data.event,});} catch (error) {- _logger.error("Error logging webhook", { error, crawlId: data.crawlId, scrapeId: data.scrapeId, teamId: data.teamId, team_id: data.teamId, module: "webhook", method: "logWebhook" });+ _logger.error("Error logging webhook", {+ error,+ crawlId: data.crawlId,+ scrapeId: data.scrapeId,+ teamId: data.teamId,+ team_id: data.teamId,+ module: "webhook",+ method: "logWebhook",+ });}}@@ -78,16 +98,17 @@ export const callWebhook = async ({teamId: string;crawlId: string;scrapeId?: string;- webhook?: z.infer<typeof webhookSchema>,- v1: boolean,+ webhook?: z.infer<typeof webhookSchema>;+ v1: boolean;data: any | null;- eventType: WebhookEventType,+ eventType: WebhookEventType;awaitWebhook?: boolean;}) => {const logger = _logger.child({module: "webhook",method: "callWebhook",- teamId, team_id: teamId,+ teamId,+ team_id: teamId,crawlId,scrapeId,eventType,@@ -126,12 +147,9 @@ export const callWebhook = async ({.eq("team_id", teamId).limit(1);if (error) {- logger.error(- `Error fetching webhook URL for team`,- {- error,- },- );+ logger.error(`Error fetching webhook URL for team`, {+ error,+ });return null;}@@ -204,7 +222,7 @@ export const callWebhook = async ({: undefined,metadata: webhookUrl.metadata || undefined,}),- dispatcher: secureDispatcher,+ dispatcher: getSecureDispatcher(),signal: AbortSignal.timeout(v1 ? 10000 : 30000), // 10 seconds timeout (v1)});if (!res.ok) {@@ -220,12 +238,9 @@ export const callWebhook = async ({statusCode: res.status,});} catch (error) {- logger.error(- `Failed to send webhook`,- {- error,- },- );+ logger.error(`Failed to send webhook`, {+ error,+ });logWebhook({success: false,teamId,@@ -233,35 +248,44 @@ export const callWebhook = async ({scrapeId,url: webhookUrl.url,event: eventType,- error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),- statusCode: typeof (error as any)?.status === "number" ? (error as any).status : undefined,+ error:+ error instanceof Error+ ? error.message+ : typeof error === "string"+ ? error+ : undefined,+ statusCode:+ typeof (error as any)?.status === "number"+ ? (error as any).status+ : undefined,});}} else {- undici.fetch(webhookUrl.url, {- method: "POST",- headers: {- "Content-Type": "application/json",- ...webhookUrl.headers,- },- body: JSON.stringify({- success: !v1- ? data.success- : eventType === "crawl.page"+ undici+ .fetch(webhookUrl.url, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ ...webhookUrl.headers,+ },+ body: JSON.stringify({+ success: !v1? data.success- : true,- type: eventType,- [v1 ? "id" : "jobId"]: crawlId,- data: dataToSend,- error: !v1- ? data?.error || undefined- : eventType === "crawl.page"+ : eventType === "crawl.page"+ ? data.success+ : true,+ type: eventType,+ [v1 ? "id" : "jobId"]: crawlId,+ data: dataToSend,+ error: !v1? data?.error || undefined- : undefined,- metadata: webhookUrl.metadata || undefined,- }),- dispatcher: secureDispatcher,- })+ : eventType === "crawl.page"+ ? data?.error || undefined+ : undefined,+ metadata: webhookUrl.metadata || undefined,+ }),+ dispatcher: getSecureDispatcher(),+ }).then((res) => {if (!res.ok) {throw { status: res.status };@@ -277,12 +301,9 @@ export const callWebhook = async ({});}).catch((error) => {- logger.error(- `Failed to send webhook`,- {- error,- },- );+ logger.error(`Failed to send webhook`, {+ error,+ });logWebhook({success: false,teamId,@@ -290,17 +311,22 @@ export const callWebhook = async ({scrapeId,url: webhookUrl.url,event: eventType,- error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),- statusCode: typeof (error as any)?.status === "number" ? (error as any).status : undefined,+ error:+ error instanceof Error+ ? error.message+ : typeof error === "string"+ ? error+ : undefined,+ statusCode:+ typeof (error as any)?.status === "number"+ ? (error as any).status+ : undefined,});});}} catch (error) {- logger.warn(- `Error sending webhook`,- {- error,- },- );+ logger.warn(`Error sending webhook`, {+ error,+ });}};