Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix sitemap parsing (ENG-3361) (#2085)

tom688bf9372352
diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
new file mode 100644
index 000000000..b9c9c5f1d
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
@@ -0,0 +1,62 @@
+import { createRobotsChecker, fetchRobotsTxt } from "../../../lib/robots-txt";
+import { scrapeTimeout } from "./lib";
+
+describe("Robots.txt tests", () => {
+ const robotsUrl = "https://www.google.com/robots.txt";
+ const siteUrl = "https://www.google.com/";
+ let robotsTxt: string;
+ let checker: ReturnType<typeof createRobotsChecker>;
+
+ beforeAll(async () => {
+ robotsTxt = await fetchRobotsTxt(robotsUrl);
+ checker = createRobotsChecker(siteUrl, robotsTxt);
+ }, scrapeTimeout);
+
+ it(
+ "contains expected directives",
+ () => {
+ expect(robotsTxt).toContain("User-agent");
+ expect(robotsTxt).toContain("Disallow");
+ },
+ scrapeTimeout,
+ );
+
+ it("allows and disallows URLs as per rules", () => {
+ expect(
+ checker.robots.isAllowed(
+ "https://www.google.com/search?q=test",
+ "FireCrawlAgent",
+ ),
+ ).toBe(false);
+
+ expect(
+ checker.robots.isAllowed(
+ "https://www.google.com/search/about",
+ "FireCrawlAgent",
+ ),
+ ).toBe(true);
+
+ expect(
+ checker.robots.isAllowed(
+ "https://www.google.com/groups",
+ "FireCrawlAgent",
+ ),
+ ).toBe(false);
+
+ expect(
+ checker.robots.isAllowed(
+ "https://www.google.com/preferences",
+ "FireCrawlAgent",
+ ),
+ ).toBe(true);
+ });
+
+ it("includes Google's sitemap URLs", () => {
+ const sitemaps = checker.robots.getSitemaps();
+
+ expect(Array.isArray(sitemaps)).toBe(true);
+ expect(sitemaps.length).toBeGreaterThan(0);
+
+ expect(sitemaps).toContain("https://www.google.com/sitemap.xml");
+ });
+});
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index fd94343d2..a92355372 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -21,7 +21,11 @@ import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { logger } from "../../lib/logger";
-import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";
+import {
+ generateURLSplits,
+ queryIndexAtDomainSplitLevelWithMeta,
+ queryIndexAtSplitLevelWithMeta,
+} from "../../services/index";
import { redisEvictConnection } from "../../services/redis";
import { performCosineSimilarityV2 } from "../../lib/map-cosine";
import { MapTimeoutError } from "../../lib/error";
@@ -39,19 +43,24 @@ interface MapResult {
}
function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
- let urlSet = new Set<string>(documents.map(x => x.url));
+ let urlSet = new Set<string>(documents.map((x) => x.url));
let newDocuments: MapDocument[] = [
- ...Array.from(urlSet).map(x => {
- let localDocs = documents.filter(y => y.url === x);
- return localDocs.find(x => x.title !== undefined) || localDocs[0];
+ ...Array.from(urlSet).map((x) => {
+ let localDocs = documents.filter((y) => y.url === x);
+ return localDocs.find((x) => x.title !== undefined) || localDocs[0];
}),
];
return newDocuments;
}
-async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<MapDocument[]> {
+async function queryIndex(
+ url: string,
+ limit: number,
+ useIndex: boolean,
+ includeSubdomains: boolean,
+): Promise<MapDocument[]> {
if (!useIndex) {
return [];
}
@@ -63,13 +72,15 @@ async function queryIndex(url: string, limit: number, useIndex: boolean, include
// TEMP: this should be altered on June 15th 2025 7AM PT - mogery
const [domainLinks, splitLinks] = await Promise.all([
- includeSubdomains ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit) : [],
+ includeSubdomains
+ ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)
+ : [],
queryIndexAtSplitLevelWithMeta(url, limit),
]);
return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);
} else {
- return (await queryIndexAtSplitLevelWithMeta(url, limit));
+ return await queryIndexAtSplitLevelWithMeta(url, limit);
}
}
@@ -181,10 +192,14 @@ export async function getMapResults({
pagePromises = JSON.parse(cachedResult);
} else {
const fetchPage = async (page: number) => {
- return fireEngineMap(mapUrl, {
- numResults: resultsPerPage,
- page: page,
- }, abort);
+ return fireEngineMap(
+ mapUrl,
+ {
+ numResults: resultsPerPage,
+ page: page,
+ },
+ abort,
+ );
};
pagePromises = Array.from({ length: maxPages }, (_, i) =>
@@ -199,7 +214,12 @@ export async function getMapResults({
]);
if (!zeroDataRetention) {
- await redisEvictConnection.set(cacheKey, JSON.stringify(searchResults), "EX", 48 * 60 * 60); // Cache for 48 hours
+ await redisEvictConnection.set(
+ cacheKey,
+ JSON.stringify(searchResults),
+ "EX",
+ 48 * 60 * 60,
+ ); // Cache for 48 hours
}
if (indexResults.length > 0) {
@@ -212,9 +232,11 @@ export async function getMapResults({
try {
await crawler.tryGetSitemap(
(urls) => {
- mapResults.push(...urls.map(x => ({
- url: x,
- })));
+ mapResults.push(
+ ...urls.map((x) => ({
+ url: x,
+ })),
+ );
},
true,
false,
@@ -227,17 +249,25 @@ export async function getMapResults({
}
if (search) {
- mapResults = searchResults.flat().map<MapDocument>(x => ({
- url: x.url,
- title: x.title,
- description: x.description,
- }) satisfies MapDocument).concat(mapResults);
+ mapResults = searchResults
+ .flat()
+ .map<MapDocument>(
+ (x) =>
+ ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ }) satisfies MapDocument,
+ )
+ .concat(mapResults);
} else {
- mapResults = mapResults.concat(searchResults.flat().map(x => ({
- url: x.url,
- title: x.title,
- description: x.description,
- })));
+ mapResults = mapResults.concat(
+ searchResults.flat().map((x) => ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ })),
+ );
}
const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
@@ -255,14 +285,17 @@ export async function getMapResults({
try {
return {
...x,
- url: checkAndUpdateURLForMap(x.url, crawlerOptions.ignoreQueryParameters ?? true).url.trim(),
+ url: checkAndUpdateURLForMap(
+ x.url,
+ crawlerOptions.ignoreQueryParameters ?? true,
+ ).url.trim(),
};
} catch (_) {
return null;
}
})
.filter((x) => x !== null) as MapDocument[];
-
+
// allows for subdomains to be included
mapResults = mapResults.filter((x) => isSameDomain(x.url, url));
@@ -278,8 +311,8 @@ export async function getMapResults({
const urlPath = urlObj.pathname;
// Only apply path filtering if the URL has a significant path (not just '/' or empty)
// This means we only filter by path if the user has not selected a root domain
- if (urlPath && urlPath !== '/' && urlPath.length > 1) {
- mapResults = mapResults.filter(x => {
+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {
+ mapResults = mapResults.filter((x) => {
try {
const linkObj = new URL(x.url);
return linkObj.pathname.startsWith(urlPath);
@@ -290,7 +323,9 @@ export async function getMapResults({
}
} catch (e) {
// If URL parsing fails, continue without path filtering
- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });
+ logger.warn(`Failed to parse URL for path filtering: ${url}`, {
+ error: e,
+ });
}
}
@@ -323,7 +358,7 @@ export async function mapController(
let result: Awaited<ReturnType<typeof getMapResults>>;
const abort = new AbortController();
try {
- result = await Promise.race([
+ result = (await Promise.race([
getMapResults({
url: req.body.url,
search: req.body.search,
@@ -341,13 +376,17 @@ export async function mapController(
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
}),
- ...(req.body.timeout !== undefined ? [
- new Promise((resolve, reject) => setTimeout(() => {
- abort.abort(new MapTimeoutError());
- reject(new MapTimeoutError());
- }, req.body.timeout))
- ] : []),
- ]) as any;
+ ...(req.body.timeout !== undefined
+ ? [
+ new Promise((resolve, reject) =>
+ setTimeout(() => {
+ abort.abort(new MapTimeoutError());
+ reject(new MapTimeoutError());
+ }, req.body.timeout),
+ ),
+ ]
+ : []),
+ ])) as any;
} catch (error) {
if (error instanceof MapTimeoutError) {
return res.status(408).json({
@@ -361,7 +400,12 @@ export async function mapController(
}
// Bill the team
- billTeam(req.auth.team_id, req.acuc?.sub_id ?? undefined, 1, req.acuc?.api_key_id ?? null).catch((error) => {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ 1,
+ req.acuc?.api_key_id ?? null,
+ ).catch((error) => {
logger.error(
`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,
);
@@ -393,4 +437,4 @@ export async function mapController(
};
return res.status(200).json(response);
-}
\ No newline at end of file
+}
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 0b601d87c..300909b4a 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -1,8 +1,7 @@
import robotsParser, { Robot } from "robots-parser";
-import axios from "axios";
-import { axiosTimeout } from "./timeout";
-import https from "https";
+import * as undici from "undici";
import { Logger } from "winston";
+import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
export interface RobotsTxtChecker {
robotsTxtUrl: string;
@@ -18,30 +17,29 @@ export async function fetchRobotsTxt(
const urlObj = new URL(url);
const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
- let extraArgs: any = {};
- if (skipTlsVerification) {
- extraArgs.httpsAgent = new https.Agent({
- rejectUnauthorized: false,
- });
- }
-
- const response = await axios.get(robotsTxtUrl, {
- timeout: axiosTimeout,
+ const response = await undici.fetch(robotsTxtUrl, {
signal: abort,
- ...extraArgs,
+ dispatcher: getSecureDispatcher(skipTlsVerification),
});
- const contentType = (Object.entries(response.headers).find(
- (x) => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? "";
+ if (!response.ok) {
+ throw new Error(
+ `Failed to fetch robots.txt: ${response.status} ${response.statusText}`,
+ );
+ }
+
+ const content = await response.text();
+ const contentType = response.headers.get("content-type") || "";
- if ((contentType.includes("text/html") && response.data.trim().startsWith("<")) ||
- contentType.includes("application/json") ||
- contentType.includes("application/xml")) {
+ if (
+ (contentType.includes("text/html") && content.trim().startsWith("<")) ||
+ contentType.includes("application/json") ||
+ contentType.includes("application/xml")
+ ) {
return "";
}
- return response.data;
+ return content;
}
export function createRobotsChecker(
@@ -67,7 +65,7 @@ export function isUrlAllowedByRobots(
for (const userAgent of userAgents) {
let isAllowed = robots.isAllowed(url, userAgent);
-
+
// Handle null/undefined responses - default to true (allowed)
if (isAllowed === null || isAllowed === undefined) {
isAllowed = true;
@@ -82,11 +80,11 @@ export function isUrlAllowedByRobots(
if (isAllowed && !url.endsWith("/")) {
const urlWithSlash = url + "/";
let isAllowedWithSlash = robots.isAllowed(urlWithSlash, userAgent);
-
+
if (isAllowedWithSlash == null) {
isAllowedWithSlash = true;
}
-
+
// If the trailing slash version is explicitly disallowed, block it
if (isAllowedWithSlash === false) {
isAllowed = false;
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 272238f43..00d29a55e 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -1,17 +1,19 @@
-import axios, { AxiosError } from "axios";
+import { AxiosError } from "axios";
import { load } from "cheerio"; // rustified
import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import robotsParser, { Robot } from "robots-parser";
import psl from "psl";
import { getURLDepth } from "./utils/maxDepthUtils";
-import { axiosTimeout } from "../../lib/timeout";
import { logger as _logger } from "../../lib/logger";
-import https from "https";
import { redisEvictConnection } from "../../services/redis";
import { extractLinks } from "../../lib/html-transformer";
import { filterLinks } from "../../lib/crawler";
-import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";
+import {
+ fetchRobotsTxt,
+ createRobotsChecker,
+ isUrlAllowedByRobots,
+} from "../../lib/robots-txt";
import { ScrapeJobTimeoutError } from "../../lib/error";
export const SITEMAP_LIMIT = 100;
@@ -32,7 +34,7 @@ export enum DenialReason {
BACKWARD_CRAWLING = "URL cannot be crawled unless crawlEntireDomain is set to true",
SOCIAL_MEDIA = "URL is a social media or email link",
EXTERNAL_LINK = "External URL not allowed",
- SECTION_LINK = "URL contains section anchor (#)"
+ SECTION_LINK = "URL contains section anchor (#)",
}
export interface FilterLinksResult {
@@ -122,7 +124,11 @@ export class WebCrawler {
this.ignoreRobotsTxt = ignoreRobotsTxt ?? false;
this.regexOnFullURL = regexOnFullURL ?? false;
this.zeroDataRetention = zeroDataRetention ?? false;
- this.logger = _logger.child({ crawlId: this.jobId, module: "WebCrawler", zeroDataRetention: this.zeroDataRetention });
+ this.logger = _logger.child({
+ crawlId: this.jobId,
+ module: "WebCrawler",
+ zeroDataRetention: this.zeroDataRetention,
+ });
this.maxDiscoveryDepth = maxDiscoveryDepth;
this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;
}
@@ -136,8 +142,11 @@ export class WebCrawler {
const denialReasons = new Map<string, string>();
if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {
- this.logger.debug("Max discovery depth hit, filtering off all links", { currentDiscoveryDepth: this.currentDiscoveryDepth, maxDiscoveryDepth: this.maxDiscoveryDepth });
- sitemapLinks.forEach(link => {
+ this.logger.debug("Max discovery depth hit, filtering off all links", {
+ currentDiscoveryDepth: this.currentDiscoveryDepth,
+ maxDiscoveryDepth: this.maxDiscoveryDepth,
+ });
+ sitemapLinks.forEach((link) => {
denialReasons.set(link, "Maximum discovery depth reached");
});
return { links: [], denialReasons };
@@ -275,7 +284,9 @@ export class WebCrawler {
!normalizedLink.pathname.startsWith(normalizedInitialUrl.pathname)
) {
if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
- this.logger.debug(`${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`);
+ this.logger.debug(
+ `${link} BACKWARDS FAIL ${normalizedLink.pathname} ${normalizedInitialUrl.pathname}`,
+ );
}
denialReasons.set(link, DenialReason.BACKWARD_CRAWLING);
return false;
@@ -284,7 +295,9 @@ export class WebCrawler {
const isAllowed = this.ignoreRobotsTxt
? true
- : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);
+ : ((this.robots.isAllowed(link, "FireCrawlAgent") ||
+ this.robots.isAllowed(link, "FirecrawlAgent")) ??
+ true);
// Check if the link is disallowed by robots.txt
if (!isAllowed) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -316,7 +329,10 @@ export class WebCrawler {
return { links: filteredLinks, denialReasons };
}
- public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {
+ public async getRobotsTxt(
+ skipTlsVerification = false,
+ abort?: AbortSignal,
+ ): Promise<string> {
return fetchRobotsTxt(this.initialUrl, skipTlsVerification, abort);
}
@@ -325,7 +341,9 @@ export class WebCrawler {
const checker = createRobotsChecker(this.initialUrl, txt);
this.robots = checker.robots;
this.robotsTxtUrl = checker.robotsTxtUrl;
- const delay = this.robots.getCrawlDelay("FireCrawlAgent") || this.robots.getCrawlDelay("FirecrawlAgent");
+ const delay =
+ this.robots.getCrawlDelay("FireCrawlAgent") ||
+ this.robots.getCrawlDelay("FirecrawlAgent");
this.robotsCrawlDelay = delay !== undefined ? delay : null;
}
@@ -360,7 +378,9 @@ export class WebCrawler {
return await urlsHandler(urls);
} else {
let filteredLinksResult = await this.filterLinks(
- [...new Set(urls)].filter(x => this.filterURL(x, this.initialUrl).allowed),
+ [...new Set(urls)].filter(
+ (x) => this.filterURL(x, this.initialUrl).allowed,
+ ),
leftOfLimit,
this.maxCrawledDepth,
fromMap,
@@ -396,15 +416,23 @@ export class WebCrawler {
// Allow sitemaps to be cached for 48 hours if they are requested from /map
// - mogery
- const maxAge = (fromMap && !onlySitemap) ? 48 * 60 * 60 * 1000 : 0;
+ const maxAge = fromMap && !onlySitemap ? 48 * 60 * 60 * 1000 : 0;
try {
let count = (await Promise.race([
Promise.all([
- this.tryFetchSitemapLinks(this.initialUrl, _urlsHandler, abort, mock, maxAge),
+ this.tryFetchSitemapLinks(
+ this.initialUrl,
+ _urlsHandler,
+ abort,
+ mock,
+ maxAge,
+ ),
...this.robots
.getSitemaps()
- .map((x) => this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge)),
+ .map((x) =>
+ this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge),
+ ),
]).then((results) => results.reduce((a, x) => a + x, 0)),
timeoutPromise,
])) as number;
@@ -560,18 +588,28 @@ export class WebCrawler {
public async extractLinksFromHTML(html: string, url: string) {
try {
- return [...new Set((await this.extractLinksFromHTMLRust(html, url)).map(x => {
- try {
- return new URL(x, url).href
- } catch (e) {
- return null;
- }
- }).filter(x => x !== null) as string[])];
+ return [
+ ...new Set(
+ (await this.extractLinksFromHTMLRust(html, url))
+ .map((x) => {
+ try {
+ return new URL(x, url).href;
+ } catch (e) {
+ return null;
+ }
+ })
+ .filter((x) => x !== null) as string[],
+ ),
+ ];
} catch (error) {
- this.logger.warn("Failed to call html-transformer! Falling back to cheerio...", {
- error,
- module: "scrapeURL", method: "extractMetadata"
- });
+ this.logger.warn(
+ "Failed to call html-transformer! Falling back to cheerio...",
+ {
+ error,
+ module: "scrapeURL",
+ method: "extractMetadata",
+ },
+ );
}
return this.extractLinksFromHTMLCheerio(html, url);
@@ -581,9 +619,7 @@ export class WebCrawler {
url: string,
ignoreRobotsTxt: boolean = false,
): boolean {
- return ignoreRobotsTxt
- ? true
- : isUrlAllowedByRobots(url, this.robots);
+ return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);
}
private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {
@@ -636,13 +672,13 @@ export class WebCrawler {
if (!link.includes("#")) {
return true;
}
-
+
// Check if the hash fragment looks like a route (contains forward slashes and has substantial content)
const hashPart = link.split("#")[1];
if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {
return true;
}
-
+
return false;
}
@@ -660,14 +696,14 @@ export class WebCrawler {
try {
const linkUrl = new URL(link, this.baseUrl);
const baseUrl = new URL(this.baseUrl);
-
+
const linkParsed = psl.parse(linkUrl.hostname);
const baseParsed = psl.parse(baseUrl.hostname);
-
+
if (!linkParsed?.domain || !baseParsed?.domain) {
return false;
}
-
+
return linkParsed.domain === baseParsed.domain;
} catch (error) {
return false;
@@ -749,7 +785,13 @@ export class WebCrawler {
// Try to get sitemap from the provided URL first
try {
sitemapCount = await getLinksFromSitemap(
- { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },
+ {
+ sitemapUrl,
+ urlsHandler,
+ mode: "fire-engine",
+ maxAge,
+ zeroDataRetention: this.zeroDataRetention,
+ },
this.logger,
this.jobId,
this.sitemapsHit,
@@ -792,7 +834,7 @@ export class WebCrawler {
try {
const linkUrl = new URL(link);
return linkUrl.hostname.endsWith(hostname);
- } catch { }
+ } catch {}
}),
);
},
@@ -834,7 +876,13 @@ export class WebCrawler {
const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;
try {
sitemapCount += await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },
+ {
+ sitemapUrl: baseUrlSitemap,
+ urlsHandler,
+ mode: "fire-engine",
+ maxAge,
+ zeroDataRetention: this.zeroDataRetention,
+ },
this.logger,
this.jobId,
this.sitemapsHit,
@@ -854,7 +902,13 @@ export class WebCrawler {
// ignore 404
} else {
sitemapCount += await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention },
+ {
+ sitemapUrl: baseUrlSitemap,
+ urlsHandler,
+ mode: "fire-engine",
+ maxAge,
+ zeroDataRetention: this.zeroDataRetention,
+ },
this.logger,
this.jobId,
this.sitemapsHit,
@@ -867,7 +921,10 @@ export class WebCrawler {
}
if (this.sitemapsHit.size >= SITEMAP_LIMIT) {
- this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
+ this.logger.warn("Sitemap limit hit!", {
+ crawlId: this.jobId,
+ url: this.baseUrl,
+ });
}
return sitemapCount;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index 99bd7c545..b22e41e7c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -4,9 +4,8 @@ import { Meta } from "../..";
import { SSLError } from "../../error";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
+ getSecureDispatcher,
InsecureConnectionError,
- secureDispatcher,
- secureDispatcherSkipTlsVerification,
} from "../utils/safeFetch";
import { MockState, saveMock } from "../../lib/mock";
import { TextDecoder } from "util";
@@ -26,7 +25,7 @@ export async function scrapeURLWithFetch(
let response: {
url: string;
- body: string,
+ body: string;
status: number;
headers: [string, string][];
};
@@ -53,7 +52,7 @@ export async function scrapeURLWithFetch(
} else {
try {
const x = await undici.fetch(meta.rewrittenUrl ?? meta.url, {
- dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,
+ dispatcher: getSecureDispatcher(meta.options.skipTlsVerification),
redirect: "follow",
headers: meta.options.headers,
signal: meta.abort.asSignal(),
@@ -61,13 +60,18 @@ export async function scrapeURLWithFetch(
const buf = Buffer.from(await x.arrayBuffer());
let text = buf.toString("utf8");
- const charset = (text.match(/<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i) ?? [])[1]
+ const charset = (text.match(
+ /<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i,
+ ) ?? [])[1];
try {
if (charset) {
text = new TextDecoder(charset.trim()).decode(buf);
}
} catch (error) {
- meta.logger.warn("Failed to re-parse with correct charset", { charset, error })
+ meta.logger.warn("Failed to re-parse with correct charset", {
+ charset,
+ error,
+ });
}
response = {
@@ -78,10 +82,7 @@ export async function scrapeURLWithFetch(
};
if (meta.mock === null) {
- await saveMock(
- mockOptions,
- response,
- );
+ await saveMock(mockOptions, response);
}
} catch (error) {
if (
@@ -89,7 +90,12 @@ export async function scrapeURLWithFetch(
error.cause instanceof InsecureConnectionError
) {
throw error.cause;
- } else if (error instanceof Error && error.message === "fetch failed" && error.cause && (error.cause as any).code === "CERT_HAS_EXPIRED") {
+ } else if (
+ error instanceof Error &&
+ error.message === "fetch failed" &&
+ error.cause &&
+ (error.cause as any).code === "CERT_HAS_EXPIRED"
+ ) {
throw new SSLError(meta.options.skipTlsVerification);
} else {
throw error;
@@ -106,9 +112,9 @@ export async function scrapeURLWithFetch(
url: response.url,
html: response.body,
statusCode: response.status,
- contentType: (response.headers.find(
- (x) => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
+ contentType:
+ (response.headers.find((x) => x[0].toLowerCase() === "content-type") ??
+ [])[1] ?? undefined,
proxyUsed: "basic",
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
index 4ccd52c53..87bbdccda 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -5,7 +5,7 @@ import { EngineError } from "../../error";
import { Writable } from "stream";
import { v4 as uuid } from "uuid";
import * as undici from "undici";
-import { secureDispatcher } from "./safeFetch";
+import { getSecureDispatcher } from "./safeFetch";
export async function fetchFileToBuffer(
url: string,
@@ -17,7 +17,7 @@ export async function fetchFileToBuffer(
const response = await undici.fetch(url, {
...init,
redirect: "follow",
- dispatcher: secureDispatcher,
+ dispatcher: getSecureDispatcher(),
});
return {
response,
@@ -40,7 +40,7 @@ export async function downloadFile(
const response = await undici.fetch(url, {
...init,
redirect: "follow",
- dispatcher: secureDispatcher,
+ dispatcher: getSecureDispatcher(),
});
// This should never happen in the current state of JS/Undici (2024), but let's check anyways.
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index b18d9af7c..406f5a833 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -18,7 +18,7 @@ export function isIPPrivate(address: string): boolean {
return addr.range() !== "unicast";
}
-export function makeSecureDispatcher(skipTlsVerification: boolean = false) {
+export function makeSecureDispatcher(skipTlsVerification: boolean) {
const agentOpts: undici.Agent.Options = {
maxRedirections: 5000,
};
@@ -62,5 +62,8 @@ export function makeSecureDispatcher(skipTlsVerification: boolean = false) {
return agent;
}
-export const secureDispatcher = makeSecureDispatcher(false);
-export const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
+const secureDispatcher = makeSecureDispatcher(false);
+const secureDispatcherSkipTlsVerification = makeSecureDispatcher(true);
+
+export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>
+ skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;
diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts
index 005d3061b..cdc4046a5 100644
--- a/apps/api/src/services/webhook.ts
+++ b/apps/api/src/services/webhook.ts
@@ -1,5 +1,4 @@
import undici from "undici";
-import { isIPPrivate, secureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
import { logger as _logger, logger } from "../lib/logger";
import { supabase_rr_service, supabase_service } from "./supabase";
import { WebhookEventType } from "../types";
@@ -7,22 +6,30 @@ import { configDotenv } from "dotenv";
import { z } from "zod";
import { webhookSchema } from "../controllers/v1/types";
import { redisEvictConnection } from "./redis";
+import { getSecureDispatcher, isIPPrivate } from "../scraper/scrapeURL/engines/utils/safeFetch";
configDotenv();
const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";
const WEBHOOK_INSERT_BATCH_SIZE = 1000;
async function addWebhookInsertJob(data: any) {
- await redisEvictConnection.rpush(WEBHOOK_INSERT_QUEUE_KEY, JSON.stringify(data));
+ await redisEvictConnection.rpush(
+ WEBHOOK_INSERT_QUEUE_KEY,
+ JSON.stringify(data),
+ );
}
export async function getWebhookInsertQueueLength(): Promise<number> {
- return await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY) ?? 0;
+ return (await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY)) ?? 0;
}
async function getWebhookInsertJobs(): Promise<any[]> {
- const jobs = (await redisEvictConnection.lpop(WEBHOOK_INSERT_QUEUE_KEY, WEBHOOK_INSERT_BATCH_SIZE)) ?? [];
- return jobs.map(x => JSON.parse(x));
+ const jobs =
+ (await redisEvictConnection.lpop(
+ WEBHOOK_INSERT_QUEUE_KEY,
+ WEBHOOK_INSERT_BATCH_SIZE,
+ )) ?? [];
+ return jobs.map((x) => JSON.parse(x));
}
export async function processWebhookInsertJobs() {
@@ -30,12 +37,17 @@ export async function processWebhookInsertJobs() {
if (jobs.length === 0) {
return;
}
- logger.info(`Webhook inserter found jobs to insert`, { jobCount: jobs.length });
+ logger.info(`Webhook inserter found jobs to insert`, {
+ jobCount: jobs.length,
+ });
try {
await supabase_service.from("webhook_logs").insert(jobs);
logger.info(`Webhook inserter inserted jobs`, { jobCount: jobs.length });
} catch (error) {
- logger.error(`Webhook inserter failed to insert jobs`, { error, jobCount: jobs.length });
+ logger.error(`Webhook inserter failed to insert jobs`, {
+ error,
+ jobCount: jobs.length,
+ });
}
}
@@ -47,7 +59,7 @@ async function logWebhook(data: {
scrapeId?: string;
url: string;
statusCode?: number;
- event: WebhookEventType
+ event: WebhookEventType;
}) {
try {
await addWebhookInsertJob({
@@ -61,7 +73,15 @@ async function logWebhook(data: {
event: data.event,
});
} catch (error) {
- _logger.error("Error logging webhook", { error, crawlId: data.crawlId, scrapeId: data.scrapeId, teamId: data.teamId, team_id: data.teamId, module: "webhook", method: "logWebhook" });
+ _logger.error("Error logging webhook", {
+ error,
+ crawlId: data.crawlId,
+ scrapeId: data.scrapeId,
+ teamId: data.teamId,
+ team_id: data.teamId,
+ module: "webhook",
+ method: "logWebhook",
+ });
}
}
@@ -78,16 +98,17 @@ export const callWebhook = async ({
teamId: string;
crawlId: string;
scrapeId?: string;
- webhook?: z.infer<typeof webhookSchema>,
- v1: boolean,
+ webhook?: z.infer<typeof webhookSchema>;
+ v1: boolean;
data: any | null;
- eventType: WebhookEventType,
+ eventType: WebhookEventType;
awaitWebhook?: boolean;
}) => {
const logger = _logger.child({
module: "webhook",
method: "callWebhook",
- teamId, team_id: teamId,
+ teamId,
+ team_id: teamId,
crawlId,
scrapeId,
eventType,
@@ -126,12 +147,9 @@ export const callWebhook = async ({
.eq("team_id", teamId)
.limit(1);
if (error) {
- logger.error(
- `Error fetching webhook URL for team`,
- {
- error,
- },
- );
+ logger.error(`Error fetching webhook URL for team`, {
+ error,
+ });
return null;
}
@@ -204,7 +222,7 @@ export const callWebhook = async ({
: undefined,
metadata: webhookUrl.metadata || undefined,
}),
- dispatcher: secureDispatcher,
+ dispatcher: getSecureDispatcher(),
signal: AbortSignal.timeout(v1 ? 10000 : 30000), // 10 seconds timeout (v1)
});
if (!res.ok) {
@@ -220,12 +238,9 @@ export const callWebhook = async ({
statusCode: res.status,
});
} catch (error) {
- logger.error(
- `Failed to send webhook`,
- {
- error,
- },
- );
+ logger.error(`Failed to send webhook`, {
+ error,
+ });
logWebhook({
success: false,
teamId,
@@ -233,35 +248,44 @@ export const callWebhook = async ({
scrapeId,
url: webhookUrl.url,
event: eventType,
- error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),
- statusCode: typeof (error as any)?.status === "number" ? (error as any).status : undefined,
+ error:
+ error instanceof Error
+ ? error.message
+ : typeof error === "string"
+ ? error
+ : undefined,
+ statusCode:
+ typeof (error as any)?.status === "number"
+ ? (error as any).status
+ : undefined,
});
}
} else {
- undici.fetch(webhookUrl.url, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- ...webhookUrl.headers,
- },
- body: JSON.stringify({
- success: !v1
- ? data.success
- : eventType === "crawl.page"
+ undici
+ .fetch(webhookUrl.url, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...webhookUrl.headers,
+ },
+ body: JSON.stringify({
+ success: !v1
? data.success
- : true,
- type: eventType,
- [v1 ? "id" : "jobId"]: crawlId,
- data: dataToSend,
- error: !v1
- ? data?.error || undefined
- : eventType === "crawl.page"
+ : eventType === "crawl.page"
+ ? data.success
+ : true,
+ type: eventType,
+ [v1 ? "id" : "jobId"]: crawlId,
+ data: dataToSend,
+ error: !v1
? data?.error || undefined
- : undefined,
- metadata: webhookUrl.metadata || undefined,
- }),
- dispatcher: secureDispatcher,
- })
+ : eventType === "crawl.page"
+ ? data?.error || undefined
+ : undefined,
+ metadata: webhookUrl.metadata || undefined,
+ }),
+ dispatcher: getSecureDispatcher(),
+ })
.then((res) => {
if (!res.ok) {
throw { status: res.status };
@@ -277,12 +301,9 @@ export const callWebhook = async ({
});
})
.catch((error) => {
- logger.error(
- `Failed to send webhook`,
- {
- error,
- },
- );
+ logger.error(`Failed to send webhook`, {
+ error,
+ });
logWebhook({
success: false,
teamId,
@@ -290,17 +311,22 @@ export const callWebhook = async ({
scrapeId,
url: webhookUrl.url,
event: eventType,
- error: error instanceof Error ? error.message : (typeof error === "string" ? error : undefined),
- statusCode: typeof (error as any)?.status === "number" ? (error as any).status : undefined,
+ error:
+ error instanceof Error
+ ? error.message
+ : typeof error === "string"
+ ? error
+ : undefined,
+ statusCode:
+ typeof (error as any)?.status === "number"
+ ? (error as any).status
+ : undefined,
});
});
}
} catch (error) {
- logger.warn(
- `Error sending webhook`,
- {
- error,
- },
- );
+ logger.warn(`Error sending webhook`, {
+ error,
+ });
}
};