Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): custom header support for /map (#2593)
tomeba72e75f530
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 42930369e..2d385385d 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -97,6 +97,7 @@ export async function getMapResults({useIndex = true,timeout,location,+ headers,id: providedId,}: {url: string;@@ -116,6 +117,7 @@ export async function getMapResults({useIndex?: boolean;timeout?: number;location?: ScrapeOptions["location"];+ headers?: Record<string, string>;id?: string;}): Promise<MapResult> {const id = providedId ?? uuidv7();@@ -133,6 +135,7 @@ export async function getMapResults({},scrapeOptions: scrapeOptions.parse({...(location ? { location } : {}),+ ...(headers ? { headers } : {}),}),internalOptions: { teamId },team_id: teamId,@@ -422,6 +425,7 @@ export async function mapController(useIndex: req.body.useIndex,timeout: req.body.timeout,location: req.body.location,+ headers: req.body.headers,id: mapId,}),...(req.body.timeout !== undefineddiff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex fb550f156..3d87d64b3 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -946,6 +946,7 @@ const mapRequestSchemaBase = crawlerOptionsfilterByPath: z.boolean().prefault(true),useIndex: z.boolean().prefault(true),location: locationSchema,+ headers: z.record(z.string(), z.string()).optional(),});export const mapRequestSchema = mapRequestSchemaBase.strict();diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex afdba5f15..14599f0f6 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -91,6 +91,7 @@ export async function mapController(flags: req.acuc?.flags ?? null,useIndex: req.body.useIndex,location: req.body.location,+ headers: req.body.headers,id: mapId,}),...(req.body.timeout !== undefineddiff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex ac0b5c75e..64d1028e8 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -924,6 +924,7 @@ const mapRequestSchemaBase = crawlerOptionsfilterByPath: z.boolean().prefault(true),useIndex: z.boolean().prefault(true),location: locationSchema,+ headers: z.record(z.string(), z.string()).optional(),});export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 84af3fe96..83cfe7c9c 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -548,6 +548,7 @@ export function crawlToCrawler(currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false,location: sc.scrapeOptions?.location,+ headers: sc.scrapeOptions?.headers,});if (sc.robots !== undefined) {diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 00757fdcc..4df034292 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -91,6 +91,7 @@ export async function getMapResults({flags,useIndex = true,location,+ headers,maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,id: providedId,}: {@@ -109,6 +110,7 @@ export async function getMapResults({flags: TeamFlags | null;useIndex?: boolean;location?: ScrapeOptions["location"];+ headers?: Record<string, string>;maxFireEngineResults?: number;id?: string;}): Promise<MapResult> {@@ -137,6 +139,7 @@ export async function getMapResults({},scrapeOptions: scrapeOptions.parse({...(location ? { location } : {}),+ ...(headers ? { headers } : {}),}),internalOptions: { teamId },team_id: teamId,diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 91aba1216..31d33f18c 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -72,6 +72,7 @@ export class WebCrawler {private currentDiscoveryDepth: number;private zeroDataRetention: boolean;private location?: ScrapeOptions["location"];+ private headers?: Record<string, string>;constructor({jobId,@@ -92,6 +93,7 @@ export class WebCrawler {currentDiscoveryDepth,zeroDataRetention,location,+ headers,}: {jobId: string;initialUrl: string;@@ -111,6 +113,7 @@ export class WebCrawler {currentDiscoveryDepth?: number;zeroDataRetention?: boolean;location?: ScrapeOptions["location"];+ headers?: Record<string, string>;}) {this.jobId = jobId;this.initialUrl = initialUrl;@@ -139,6 +142,7 @@ export class WebCrawler {this.maxDiscoveryDepth = maxDiscoveryDepth;this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;this.location = location;+ this.headers = headers;}public setBaseUrl(newBase: string): void {@@ -744,6 +748,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ headers: this.headers,},this.logger,this.jobId,@@ -802,6 +807,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ headers: this.headers,},this.logger,this.jobId,@@ -845,6 +851,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ headers: this.headers,},this.logger,this.jobId,@@ -872,6 +879,7 @@ export class WebCrawler {maxAge,zeroDataRetention: this.zeroDataRetention,location: this.location,+ headers: this.headers,},this.logger,this.jobId,diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex d7faa57bd..81824a3d0 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -33,6 +33,7 @@ export async function getLinksFromSitemap(maxAge = 0,zeroDataRetention,location,+ headers,}: {sitemapUrl: string;urlsHandler(urls: string[]): unknown;@@ -40,6 +41,7 @@ export async function getLinksFromSitemap(maxAge?: number;zeroDataRetention: boolean;location?: ScrapeOptions["location"];+ headers?: Record<string, string>;},logger: Logger,crawlId: string,@@ -64,7 +66,9 @@ export async function getLinksFromSitemap(const isGzip = sitemapUrl.toLowerCase().endsWith(".gz");if (isGzip) {try {- const { buffer } = await fetchFileToBuffer(sitemapUrl);+ const { buffer } = await fetchFileToBuffer(sitemapUrl, false, {+ headers,+ });const decompressed = await gunzipAsync(buffer);content = decompressed.toString("utf-8");} catch (error) {@@ -106,6 +110,7 @@ export async function getLinksFromSitemap(useMock: mock,maxAge,...(location ? { location } : {}),+ ...(headers ? { headers } : {}),}),{forceEngine,@@ -209,6 +214,7 @@ export async function getLinksFromSitemap(mode,zeroDataRetention,location,+ headers,maxAge,},logger,@@ -241,6 +247,7 @@ export async function getLinksFromSitemap(mode,zeroDataRetention,location,+ headers,maxAge,},logger,@@ -289,6 +296,7 @@ export async function getLinksFromSitemap(mode,zeroDataRetention,location,+ headers,maxAge,},logger,