Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: resolve redirects before mapping urls (#2439)
tom2c23e321e87f
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex e78b9c490..475d45c3f 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -124,4 +124,25 @@ describe("Map tests", () => {},60000,);++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "handles redirects correctly",+ async () => {+ const response = await map(+ {+ url: "http://firecrawl.com",+ limit: 5,+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.links.length).toBeGreaterThan(0);+ expect(+ response.body.links.every(link => link.url.includes("firecrawl.dev")),+ ).toBe(true);+ },+ 60000,+ );});diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex c6816c694..3c2ae31e9 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -11,6 +11,7 @@ import {checkAndUpdateURLForMap,isSameDomain,isSameSubdomain,+ resolveRedirects,} from "./validateUrl";import { fireEngineMap } from "../search/fireEngine";import { redisEvictConnection } from "../services/redis";@@ -111,6 +112,8 @@ export async function getMapResults({}): Promise<MapResult> {const functionStartTime = Date.now();+ url = await resolveRedirects(url, abort);+const id = uuidv4();let mapResults: MapDocument[] = [];const zeroDataRetention = flags?.forceZDR ?? false;diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.tsindex bca847423..0cad62730 100644--- a/apps/api/src/lib/validateUrl.ts+++ b/apps/api/src/lib/validateUrl.ts@@ -1,3 +1,6 @@+import * as undici from "undici";+import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";+export const protocolIncluded = (url: string) => {// if :// not in the start of the url assume http (maybe https?)// regex checks if :// appears before any .@@ -189,3 +192,39 @@ export function removeDuplicateUrls(urls: string[]): string[] {return [...new Set(Array.from(urlMap.values()))];}++export async function resolveRedirects(+ url: string,+ abort?: AbortSignal,+): Promise<string> {+ if (!protocolIncluded(url)) {+ url = `http://${url}`;+ }++ try {+ const response = await undici.fetch(url, {+ method: "HEAD",+ redirect: "follow",+ dispatcher: getSecureDispatcher(false),+ signal: abort,+ });++ return response.url;+ } catch (error) {+ if (abort?.aborted) throw error;++ try {+ const response = await undici.fetch(url, {+ method: "GET",+ redirect: "follow",+ dispatcher: getSecureDispatcher(false),+ signal: abort,+ });++ return response.url;+ } catch (getError) {+ if (abort?.aborted) throw getError;+ return url;+ }+ }+}