Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: resolve redirects before mapping urls (#2439)

tom2c23e321e87f
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index e78b9c490..475d45c3f 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -124,4 +124,25 @@ describe("Map tests", () => {
},
60000,
);
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "handles redirects correctly",
+ async () => {
+ const response = await map(
+ {
+ url: "http://firecrawl.com",
+ limit: 5,
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.links.length).toBeGreaterThan(0);
+ expect(
+ response.body.links.every(link => link.url.includes("firecrawl.dev")),
+ ).toBe(true);
+ },
+ 60000,
+ );
});
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index c6816c694..3c2ae31e9 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -11,6 +11,7 @@ import {
checkAndUpdateURLForMap,
isSameDomain,
isSameSubdomain,
+ resolveRedirects,
} from "./validateUrl";
import { fireEngineMap } from "../search/fireEngine";
import { redisEvictConnection } from "../services/redis";
@@ -111,6 +112,8 @@ export async function getMapResults({
}): Promise<MapResult> {
const functionStartTime = Date.now();
+ url = await resolveRedirects(url, abort);
+
const id = uuidv4();
let mapResults: MapDocument[] = [];
const zeroDataRetention = flags?.forceZDR ?? false;
diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.ts
index bca847423..0cad62730 100644
--- a/apps/api/src/lib/validateUrl.ts
+++ b/apps/api/src/lib/validateUrl.ts
@@ -1,3 +1,6 @@
+import * as undici from "undici";
+import { getSecureDispatcher } from "../scraper/scrapeURL/engines/utils/safeFetch";
+
export const protocolIncluded = (url: string) => {
// if :// not in the start of the url assume http (maybe https?)
// regex checks if :// appears before any .
@@ -189,3 +192,39 @@ export function removeDuplicateUrls(urls: string[]): string[] {
return [...new Set(Array.from(urlMap.values()))];
}
+
+export async function resolveRedirects(
+ url: string,
+ abort?: AbortSignal,
+): Promise<string> {
+ if (!protocolIncluded(url)) {
+ url = `http://${url}`;
+ }
+
+ try {
+ const response = await undici.fetch(url, {
+ method: "HEAD",
+ redirect: "follow",
+ dispatcher: getSecureDispatcher(false),
+ signal: abort,
+ });
+
+ return response.url;
+ } catch (error) {
+ if (abort?.aborted) throw error;
+
+ try {
+ const response = await undici.fetch(url, {
+ method: "GET",
+ redirect: "follow",
+ dispatcher: getSecureDispatcher(false),
+ signal: abort,
+ });
+
+ return response.url;
+ } catch (getError) {
+ if (abort?.aborted) throw getError;
+ return url;
+ }
+ }
+}