Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): custom header support for /map (#2593)

tomeba72e75f530
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 42930369e..2d385385d 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -97,6 +97,7 @@ export async function getMapResults({
useIndex = true,
timeout,
location,
+ headers,
id: providedId,
}: {
url: string;
@@ -116,6 +117,7 @@ export async function getMapResults({
useIndex?: boolean;
timeout?: number;
location?: ScrapeOptions["location"];
+ headers?: Record<string, string>;
id?: string;
}): Promise<MapResult> {
const id = providedId ?? uuidv7();
@@ -133,6 +135,7 @@ export async function getMapResults({
},
scrapeOptions: scrapeOptions.parse({
...(location ? { location } : {}),
+ ...(headers ? { headers } : {}),
}),
internalOptions: { teamId },
team_id: teamId,
@@ -422,6 +425,7 @@ export async function mapController(
useIndex: req.body.useIndex,
timeout: req.body.timeout,
location: req.body.location,
+ headers: req.body.headers,
id: mapId,
}),
...(req.body.timeout !== undefined
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index fb550f156..3d87d64b3 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -946,6 +946,7 @@ const mapRequestSchemaBase = crawlerOptions
filterByPath: z.boolean().prefault(true),
useIndex: z.boolean().prefault(true),
location: locationSchema,
+ headers: z.record(z.string(), z.string()).optional(),
});
export const mapRequestSchema = mapRequestSchemaBase.strict();
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index afdba5f15..14599f0f6 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -91,6 +91,7 @@ export async function mapController(
flags: req.acuc?.flags ?? null,
useIndex: req.body.useIndex,
location: req.body.location,
+ headers: req.body.headers,
id: mapId,
}),
...(req.body.timeout !== undefined
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index ac0b5c75e..64d1028e8 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -924,6 +924,7 @@ const mapRequestSchemaBase = crawlerOptions
filterByPath: z.boolean().prefault(true),
useIndex: z.boolean().prefault(true),
location: locationSchema,
+ headers: z.record(z.string(), z.string()).optional(),
});
export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 84af3fe96..83cfe7c9c 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -548,6 +548,7 @@ export function crawlToCrawler(
currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,
zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false,
location: sc.scrapeOptions?.location,
+ headers: sc.scrapeOptions?.headers,
});
if (sc.robots !== undefined) {
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 00757fdcc..4df034292 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -91,6 +91,7 @@ export async function getMapResults({
flags,
useIndex = true,
location,
+ headers,
maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,
id: providedId,
}: {
@@ -109,6 +110,7 @@ export async function getMapResults({
flags: TeamFlags | null;
useIndex?: boolean;
location?: ScrapeOptions["location"];
+ headers?: Record<string, string>;
maxFireEngineResults?: number;
id?: string;
}): Promise<MapResult> {
@@ -137,6 +139,7 @@ export async function getMapResults({
},
scrapeOptions: scrapeOptions.parse({
...(location ? { location } : {}),
+ ...(headers ? { headers } : {}),
}),
internalOptions: { teamId },
team_id: teamId,
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 91aba1216..31d33f18c 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -72,6 +72,7 @@ export class WebCrawler {
private currentDiscoveryDepth: number;
private zeroDataRetention: boolean;
private location?: ScrapeOptions["location"];
+ private headers?: Record<string, string>;
constructor({
jobId,
@@ -92,6 +93,7 @@ export class WebCrawler {
currentDiscoveryDepth,
zeroDataRetention,
location,
+ headers,
}: {
jobId: string;
initialUrl: string;
@@ -111,6 +113,7 @@ export class WebCrawler {
currentDiscoveryDepth?: number;
zeroDataRetention?: boolean;
location?: ScrapeOptions["location"];
+ headers?: Record<string, string>;
}) {
this.jobId = jobId;
this.initialUrl = initialUrl;
@@ -139,6 +142,7 @@ export class WebCrawler {
this.maxDiscoveryDepth = maxDiscoveryDepth;
this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;
this.location = location;
+ this.headers = headers;
}
public setBaseUrl(newBase: string): void {
@@ -744,6 +748,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ headers: this.headers,
},
this.logger,
this.jobId,
@@ -802,6 +807,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ headers: this.headers,
},
this.logger,
this.jobId,
@@ -845,6 +851,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ headers: this.headers,
},
this.logger,
this.jobId,
@@ -872,6 +879,7 @@ export class WebCrawler {
maxAge,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ headers: this.headers,
},
this.logger,
this.jobId,
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index d7faa57bd..81824a3d0 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -33,6 +33,7 @@ export async function getLinksFromSitemap(
maxAge = 0,
zeroDataRetention,
location,
+ headers,
}: {
sitemapUrl: string;
urlsHandler(urls: string[]): unknown;
@@ -40,6 +41,7 @@ export async function getLinksFromSitemap(
maxAge?: number;
zeroDataRetention: boolean;
location?: ScrapeOptions["location"];
+ headers?: Record<string, string>;
},
logger: Logger,
crawlId: string,
@@ -64,7 +66,9 @@ export async function getLinksFromSitemap(
const isGzip = sitemapUrl.toLowerCase().endsWith(".gz");
if (isGzip) {
try {
- const { buffer } = await fetchFileToBuffer(sitemapUrl);
+ const { buffer } = await fetchFileToBuffer(sitemapUrl, false, {
+ headers,
+ });
const decompressed = await gunzipAsync(buffer);
content = decompressed.toString("utf-8");
} catch (error) {
@@ -106,6 +110,7 @@ export async function getLinksFromSitemap(
useMock: mock,
maxAge,
...(location ? { location } : {}),
+ ...(headers ? { headers } : {}),
}),
{
forceEngine,
@@ -209,6 +214,7 @@ export async function getLinksFromSitemap(
mode,
zeroDataRetention,
location,
+ headers,
maxAge,
},
logger,
@@ -241,6 +247,7 @@ export async function getLinksFromSitemap(
mode,
zeroDataRetention,
location,
+ headers,
maxAge,
},
logger,
@@ -289,6 +296,7 @@ export async function getLinksFromSitemap(
mode,
zeroDataRetention,
location,
+ headers,
maxAge,
},
logger,