Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas4e9eda7657e4
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 2b556a20a..93962e910 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -1,9 +1,10 @@import { Response } from "express";import { z } from "zod";-import { ErrorResponse, RequestWithAuth, crawlRequestSchema } from "./types";+import { ErrorResponse, RequestWithAuth } from "./types";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";+import { getMapResults } from "../../lib/map-utils";// Define the request schema for params preview// Only url and prompt are required/relevant for preview@@ -60,10 +61,40 @@ export async function crawlParamsPreviewController(prompt: parsedBody.prompt,});- // Generate crawler options from prompt+ // First, get the website structure by mapping URLs+ logger.debug("Getting website structure for crawl params preview");+ const mapResult = await getMapResults({+ url: parsedBody.url,+ limit: 120, // Limit to 120 URLs as requested+ includeSubdomains: true,+ crawlerOptions: {},+ teamId: req.auth.team_id,+ flags: req.acuc?.flags ?? null,+ allowExternalLinks: false,+ useIndex: true,+ maxFireEngineResults: 500, // Use smaller limit for preview+ });++ const websiteUrls = mapResult.mapResults.map(doc => doc.url);++ logger.debug("Found website URLs for crawl params preview", {+ urlCount: websiteUrls.length,+ sampleUrls: websiteUrls.slice(0, 5), // Log first 5 URLs for debugging+ });++ // Create enhanced prompt with website structure+ const enhancedPrompt = `${parsedBody.prompt}++--- WEBSITE STRUCTURE ---+The website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first 120 URLs):+${websiteUrls.slice(0, 120).join('\n')}++Based on this structure and the user's request, generate appropriate crawler options.`;++ // Generate crawler options from enhanced promptconst costTracking = new CostTracking();const { extract } = await generateCrawlerOptionsFromPrompt(- parsedBody.prompt,+ enhancedPrompt,logger,costTracking,{ teamId: req.auth.team_id },@@ -71,8 +102,9 @@ export async function crawlParamsPreviewController(const generatedOptions = extract || {};- logger.debug("Generated crawler options from prompt", {- prompt: parsedBody.prompt,+ logger.debug("Generated crawler options from enhanced prompt", {+ originalPrompt: parsedBody.prompt,+ websiteUrlCount: websiteUrls.length,generatedOptions: generatedOptions,});diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 51d64de0f..3ca36d7fa 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -1,355 +1,21 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";import {mapRequestSchema,RequestWithAuth,- scrapeOptions,- ScrapeOptions,- TeamFlags,MapRequest,- MapDocument,MapResponse,MAX_MAP_LIMIT,} from "./types";-import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";-import {- checkAndUpdateURLForMap,- isSameDomain,- isSameSubdomain,-} from "../../lib/validateUrl";-import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";import { logger } from "../../lib/logger";-import {- generateURLSplits,- queryIndexAtDomainSplitLevelWithMeta,- queryIndexAtSplitLevelWithMeta,-} from "../../services/index";-import { redisEvictConnection } from "../../services/redis";-import { performCosineSimilarityV2 } from "../../lib/map-cosine";import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";+import { getMapResults, MapResult } from "../../lib/map-utils";configDotenv();-// Max Links that "Smart /map" can return-const MAX_FIRE_ENGINE_RESULTS = 500;--interface MapResult {- success: boolean;- job_id: string;- time_taken: number;- mapResults: MapDocument[];-}--function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {- const urlMap = new Map<string, MapDocument>();-- for (const doc of documents) {- const existing = urlMap.get(doc.url);-- if (!existing) {- urlMap.set(doc.url, doc);- } else if (doc.title !== undefined && existing.title === undefined) {- urlMap.set(doc.url, doc);- }- }-- return Array.from(urlMap.values());-}--async function queryIndex(- url: string,- limit: number,- useIndex: boolean,- includeSubdomains: boolean,-): Promise<MapDocument[]> {- if (!useIndex) {- return [];- }-- const urlSplits = generateURLSplits(url);- if (urlSplits.length === 1) {- const urlObj = new URL(url);- const hostname = urlObj.hostname;-- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery- const [domainLinks, splitLinks] = await Promise.all([- includeSubdomains- ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)- : [],- queryIndexAtSplitLevelWithMeta(url, limit),- ]);-- return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);- } else {- return await queryIndexAtSplitLevelWithMeta(url, limit);- }-}--async function getMapResults({- url,- search,- limit = MAX_MAP_LIMIT,- includeSubdomains = true,- crawlerOptions = {},- teamId,- allowExternalLinks,- abort = new AbortController().signal, // noop- filterByPath = true,- flags,- useIndex = true,- location,-}: {- url: string;- search?: string;- limit?: number;- includeSubdomains?: boolean;- crawlerOptions?: any;- teamId: string;- origin?: string;- includeMetadata?: boolean;- allowExternalLinks?: boolean;- abort?: AbortSignal;- mock?: string;- filterByPath?: boolean;- flags: TeamFlags;- useIndex?: boolean;- location?: ScrapeOptions["location"];-}): Promise<MapResult> {- const functionStartTime = Date.now();-- const id = uuidv4();- let mapResults: MapDocument[] = [];- const zeroDataRetention = flags?.forceZDR ?? false;-- const sc: StoredCrawl = {- originUrl: url,- crawlerOptions: {- ...crawlerOptions,- limit: crawlerOptions.sitemapOnly ? 10000000 : limit,- scrapeOptions: undefined,- },- scrapeOptions: scrapeOptions.parse({- ...(location ? { location } : {}),- }),- internalOptions: { teamId },- team_id: teamId,- createdAt: Date.now(),- zeroDataRetention,- };-- const crawler = crawlToCrawler(id, sc, flags);-- try {- sc.robots = await crawler.getRobotsTxt(false, abort);- crawler.importRobotsTxt(sc.robots);- } catch (_) {- // Robots.txt fetch failed, continue without it- }-- // If sitemapOnly is true, only get links from sitemap- if (crawlerOptions.sitemap === "only") {- const sitemap = await crawler.tryGetSitemap(- urls => {- urls.forEach(x => {- mapResults.push({- url: x,- });- });- },- true,- true,- crawlerOptions.timeout ?? 30000,- abort,- crawlerOptions.useMock,- );-- if (sitemap > 0) {- mapResults = mapResults- .slice(1)- .map(x => {- try {- return {- ...x,- url: checkAndUpdateURLForMap(x.url).url.trim(),- };- } catch (_) {- return null;- }- })- .filter(x => x !== null) as MapDocument[];- }- } else {- let urlWithoutWww = url.replace("www.", "");- let mapUrl =- search && allowExternalLinks- ? `${search} ${urlWithoutWww}`- : search- ? `${search} site:${urlWithoutWww}`- : `site:${url}`;-- const resultsPerPage = 100;- const maxPages = Math.ceil(- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,- );-- const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = await redisEvictConnection.get(cacheKey);-- let pagePromises: (Promise<any> | any)[];-- if (cachedResult) {- pagePromises = JSON.parse(cachedResult);- } else {- const fetchPage = async (page: number) => {- return await fireEngineMap(- mapUrl,- {- numResults: resultsPerPage,- page: page,- },- abort,- );- };-- pagePromises = Array.from({ length: maxPages }, (_, i) =>- fetchPage(i + 1),- );- }-- const [indexResults, searchResults] = await Promise.all([- queryIndex(url, limit, useIndex, includeSubdomains),- Promise.all(pagePromises),- ]);-- if (!zeroDataRetention) {- await redisEvictConnection.set(- cacheKey,- JSON.stringify(searchResults),- "EX",- 48 * 60 * 60,- ); // Cache for 48 hours- }-- if (indexResults.length > 0) {- mapResults.push(...indexResults);- }-- if (crawlerOptions.sitemap === "include") {- try {- await crawler.tryGetSitemap(- urls => {- mapResults.push(- ...urls.map(x => ({- url: x,- })),- );- },- true,- false,- crawlerOptions.timeout ?? 30000,- abort,- );- } catch (e) {- logger.warn("tryGetSitemap threw an error", { error: e });- }- }-- if (search) {- mapResults = searchResults- .flat()- .map<MapDocument>(- x =>- ({- url: x.url,- title: x.title,- description: x.description,- }) satisfies MapDocument,- )- .concat(mapResults);- } else {- mapResults = mapResults.concat(- searchResults.flat().map(x => ({- url: x.url,- title: x.title,- description: x.description,- })),- );- }-- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);- if (mapResults.length > minumumCutoff) {- mapResults = mapResults.slice(0, minumumCutoff);- }-- if (search) {- const searchQuery = search.toLowerCase();- mapResults = performCosineSimilarityV2(mapResults, searchQuery);- }- }-- mapResults = mapResults- .map(x => {- try {- return {- ...x,- url: checkAndUpdateURLForMap(- x.url,- crawlerOptions.ignoreQueryParameters ?? true,- ).url.trim(),- };- } catch (_) {- return null;- }- })- .filter(x => x !== null) as MapDocument[];-- mapResults = mapResults.filter(x => isSameDomain(x.url, url));-- if (!includeSubdomains) {- mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));- }-- if (filterByPath && !allowExternalLinks) {- try {- const urlObj = new URL(url);- const urlPath = urlObj.pathname;- // Only apply path filtering if the URL has a significant path (not just '/' or empty)- // This means we only filter by path if the user has not selected a root domain- if (urlPath && urlPath !== "/" && urlPath.length > 1) {- mapResults = mapResults.filter(x => {- try {- const linkObj = new URL(x.url);- return linkObj.pathname.startsWith(urlPath);- } catch (e) {- return false;- }- });- }- } catch (e) {- // If URL parsing fails, continue without path filtering- logger.warn(`Failed to parse URL for path filtering: ${url}`, {- error: e,- });- }- }-- mapResults = dedupeMapDocumentArray(mapResults);- mapResults = mapResults.slice(0, limit);-- const totalTimeMs = Date.now() - functionStartTime;-- return {- success: true,- mapResults,- job_id: id,- time_taken: totalTimeMs,- };-}-export async function mapController(req: RequestWithAuth<{}, MapResponse, MapRequest>,res: Response<MapResponse>,@@ -371,7 +37,7 @@ export async function mapController(teamId: req.auth.team_id,});- let result: Awaited<ReturnType<typeof getMapResults>>;+ let result: MapResult;const abort = new AbortController();try {result = (await Promise.race([@@ -386,6 +52,7 @@ export async function mapController(},origin: req.body.origin,teamId: req.auth.team_id,+ allowExternalLinks: req.body.allowExternalLinks,abort: abort.signal,mock: req.body.useMock,filterByPath: req.body.filterByPath !== false,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsnew file mode 100644index 000000000..058f4b20a--- /dev/null+++ b/apps/api/src/lib/map-utils.ts@@ -0,0 +1,324 @@+import { v4 as uuidv4 } from "uuid";+import { TeamFlags, MapDocument, scrapeOptions, ScrapeOptions } from "../controllers/v2/types";+import { crawlToCrawler, StoredCrawl } from "./crawl-redis";+import { checkAndUpdateURLForMap, isSameDomain, isSameSubdomain } from "./validateUrl";+import { fireEngineMap } from "../search/fireEngine";+import { redisEvictConnection } from "../services/redis";+import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";+import { performCosineSimilarityV2 } from "./map-cosine";++// Max Links that "Smart /map" can return+const MAX_FIRE_ENGINE_RESULTS = 500;+const MAX_MAP_LIMIT = 1000;++export interface MapResult {+ success: boolean;+ job_id: string;+ time_taken: number;+ mapResults: MapDocument[];+}++export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {+ const urlMap = new Map<string, MapDocument>();++ for (const doc of documents) {+ const existing = urlMap.get(doc.url);++ if (!existing) {+ urlMap.set(doc.url, doc);+ } else if (doc.title !== undefined && existing.title === undefined) {+ urlMap.set(doc.url, doc);+ }+ }++ return Array.from(urlMap.values());+}++export async function queryIndex(+ url: string,+ limit: number,+ useIndex: boolean,+ includeSubdomains: boolean,+): Promise<MapDocument[]> {+ if (!useIndex) {+ return [];+ }++ const urlSplits = generateURLSplits(url);+ if (urlSplits.length === 1) {+ const urlObj = new URL(url);+ const hostname = urlObj.hostname;++ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery+ const [domainLinks, splitLinks] = await Promise.all([+ includeSubdomains+ ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)+ : [],+ queryIndexAtSplitLevelWithMeta(url, limit),+ ]);++ return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);+ } else {+ return await queryIndexAtSplitLevelWithMeta(url, limit);+ }+}++export async function getMapResults({+ url,+ search,+ limit = MAX_MAP_LIMIT,+ includeSubdomains = true,+ crawlerOptions = {},+ teamId,+ allowExternalLinks,+ abort = new AbortController().signal,+ filterByPath = true,+ flags,+ useIndex = true,+ location,+ maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,+}: {+ url: string;+ search?: string;+ limit?: number;+ includeSubdomains?: boolean;+ crawlerOptions?: any;+ teamId: string;+ origin?: string;+ includeMetadata?: boolean;+ allowExternalLinks?: boolean;+ abort?: AbortSignal;+ mock?: string;+ filterByPath?: boolean;+ flags: TeamFlags | null;+ useIndex?: boolean;+ location?: ScrapeOptions["location"];+ maxFireEngineResults?: number;+}): Promise<MapResult> {+ const functionStartTime = Date.now();++ const id = uuidv4();+ let mapResults: MapDocument[] = [];+ const zeroDataRetention = flags?.forceZDR ?? false;++ const sc: StoredCrawl = {+ originUrl: url,+ crawlerOptions: {+ ...crawlerOptions,+ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,+ scrapeOptions: undefined,+ },+ scrapeOptions: scrapeOptions.parse({+ ...(location ? { location } : {}),+ }),+ internalOptions: { teamId },+ team_id: teamId,+ createdAt: Date.now(),+ zeroDataRetention,+ };++ const crawler = crawlToCrawler(id, sc, flags);++ try {+ sc.robots = await crawler.getRobotsTxt(false, abort);+ crawler.importRobotsTxt(sc.robots);+ } catch (_) {+ // Robots.txt fetch failed, continue without it+ }++ // If sitemapOnly is true, only get links from sitemap+ if (crawlerOptions.sitemap === "only") {+ const sitemap = await crawler.tryGetSitemap(+ urls => {+ urls.forEach(x => {+ mapResults.push({+ url: x,+ });+ });+ },+ true,+ true,+ crawlerOptions.timeout ?? 30000,+ abort,+ crawlerOptions.useMock,+ );++ if (sitemap > 0) {+ mapResults = mapResults+ .slice(1)+ .map(x => {+ try {+ return {+ ...x,+ url: checkAndUpdateURLForMap(x.url).url.trim(),+ };+ } catch (_) {+ return null;+ }+ })+ .filter(x => x !== null) as MapDocument[];+ }+ } else {+ let urlWithoutWww = url.replace("www.", "");+ let mapUrl =+ search && allowExternalLinks+ ? `${search} ${urlWithoutWww}`+ : search+ ? `${search} site:${urlWithoutWww}`+ : `site:${url}`;++ const resultsPerPage = 100;+ const maxPages = Math.ceil(+ Math.min(maxFireEngineResults, limit) / resultsPerPage,+ );++ const cacheKey = `fireEngineMap:${mapUrl}`;+ const cachedResult = await redisEvictConnection.get(cacheKey);++ let pagePromises: (Promise<any> | any)[];++ if (cachedResult) {+ pagePromises = JSON.parse(cachedResult);+ } else {+ const fetchPage = async (page: number) => {+ return await fireEngineMap(+ mapUrl,+ {+ numResults: resultsPerPage,+ page: page,+ },+ abort,+ );+ };++ pagePromises = Array.from({ length: maxPages }, (_, i) =>+ fetchPage(i + 1),+ );+ }++ const [indexResults, searchResults] = await Promise.all([+ queryIndex(url, limit, useIndex, includeSubdomains),+ Promise.all(pagePromises),+ ]);++ if (!zeroDataRetention) {+ await redisEvictConnection.set(+ cacheKey,+ JSON.stringify(searchResults),+ "EX",+ 48 * 60 * 60,+ ); // Cache for 48 hours+ }++ if (indexResults.length > 0) {+ mapResults.push(...indexResults);+ }++ if (crawlerOptions.sitemap === "include") {+ try {+ await crawler.tryGetSitemap(+ urls => {+ mapResults.push(+ ...urls.map(x => ({+ url: x,+ })),+ );+ },+ true,+ false,+ crawlerOptions.timeout ?? 30000,+ abort,+ );+ } catch (e) {+ // Silently handle sitemap errors+ }+ }++ if (search) {+ mapResults = searchResults+ .flat()+ .map<MapDocument>(+ x =>+ ({+ url: x.url,+ title: x.title,+ description: x.description,+ }) satisfies MapDocument,+ )+ .concat(mapResults);+ } else {+ mapResults = mapResults.concat(+ searchResults.flat().map(x => ({+ url: x.url,+ title: x.title,+ description: x.description,+ })),+ );+ }++ const minimumCutoff = Math.min(MAX_MAP_LIMIT, limit);+ if (mapResults.length > minimumCutoff) {+ mapResults = mapResults.slice(0, minimumCutoff);+ }++ if (search) {+ const searchQuery = search.toLowerCase();+ mapResults = performCosineSimilarityV2(mapResults, searchQuery);+ }+ }++ mapResults = mapResults+ .map(x => {+ try {+ return {+ ...x,+ url: checkAndUpdateURLForMap(+ x.url,+ crawlerOptions.ignoreQueryParameters ?? true,+ ).url.trim(),+ };+ } catch (_) {+ return null;+ }+ })+ .filter(x => x !== null) as MapDocument[];++ mapResults = mapResults.filter(x => isSameDomain(x.url, url));++ if (!includeSubdomains) {+ mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));+ }++ if (filterByPath && !allowExternalLinks) {+ try {+ const urlObj = new URL(url);+ const urlPath = urlObj.pathname;+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)+ // This means we only filter by path if the user has not selected a root domain+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {+ mapResults = mapResults.filter(x => {+ try {+ const linkObj = new URL(x.url);+ return linkObj.pathname.startsWith(urlPath);+ } catch (e) {+ return false;+ }+ });+ }+ } catch (e) {+ // If URL parsing fails, continue without path filtering+ }+ }++ mapResults = dedupeMapDocumentArray(mapResults);+ mapResults = mapResults.slice(0, limit);++ const totalTimeMs = Date.now() - functionStartTime;++ return {+ success: true,+ mapResults,+ job_id: id,+ time_taken: totalTimeMs,+ };+}