Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas4e9eda7657e4
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 2b556a20a..93962e910 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -1,9 +1,10 @@
import { Response } from "express";
import { z } from "zod";
-import { ErrorResponse, RequestWithAuth, crawlRequestSchema } from "./types";
+import { ErrorResponse, RequestWithAuth } from "./types";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
+import { getMapResults } from "../../lib/map-utils";
// Define the request schema for params preview
// Only url and prompt are required/relevant for preview
@@ -60,10 +61,40 @@ export async function crawlParamsPreviewController(
prompt: parsedBody.prompt,
});
- // Generate crawler options from prompt
+ // First, get the website structure by mapping URLs
+ logger.debug("Getting website structure for crawl params preview");
+ const mapResult = await getMapResults({
+ url: parsedBody.url,
+ limit: 120, // Limit to 120 URLs as requested
+ includeSubdomains: true,
+ crawlerOptions: {},
+ teamId: req.auth.team_id,
+ flags: req.acuc?.flags ?? null,
+ allowExternalLinks: false,
+ useIndex: true,
+ maxFireEngineResults: 500, // Use smaller limit for preview
+ });
+
+ const websiteUrls = mapResult.mapResults.map(doc => doc.url);
+
+ logger.debug("Found website URLs for crawl params preview", {
+ urlCount: websiteUrls.length,
+ sampleUrls: websiteUrls.slice(0, 5), // Log first 5 URLs for debugging
+ });
+
+ // Create enhanced prompt with website structure
+ const enhancedPrompt = `${parsedBody.prompt}
+
+--- WEBSITE STRUCTURE ---
+The website has the following URL structure (${websiteUrls.length} URLs found, here is a sample of the first 120 URLs):
+${websiteUrls.slice(0, 120).join('\n')}
+
+Based on this structure and the user's request, generate appropriate crawler options.`;
+
+ // Generate crawler options from enhanced prompt
const costTracking = new CostTracking();
const { extract } = await generateCrawlerOptionsFromPrompt(
- parsedBody.prompt,
+ enhancedPrompt,
logger,
costTracking,
{ teamId: req.auth.team_id },
@@ -71,8 +102,9 @@ export async function crawlParamsPreviewController(
const generatedOptions = extract || {};
- logger.debug("Generated crawler options from prompt", {
- prompt: parsedBody.prompt,
+ logger.debug("Generated crawler options from enhanced prompt", {
+ originalPrompt: parsedBody.prompt,
+ websiteUrlCount: websiteUrls.length,
generatedOptions: generatedOptions,
});
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 51d64de0f..3ca36d7fa 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -1,355 +1,21 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
import {
mapRequestSchema,
RequestWithAuth,
- scrapeOptions,
- ScrapeOptions,
- TeamFlags,
MapRequest,
- MapDocument,
MapResponse,
MAX_MAP_LIMIT,
} from "./types";
-import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
-import {
- checkAndUpdateURLForMap,
- isSameDomain,
- isSameSubdomain,
-} from "../../lib/validateUrl";
-import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { logger } from "../../lib/logger";
-import {
- generateURLSplits,
- queryIndexAtDomainSplitLevelWithMeta,
- queryIndexAtSplitLevelWithMeta,
-} from "../../services/index";
-import { redisEvictConnection } from "../../services/redis";
-import { performCosineSimilarityV2 } from "../../lib/map-cosine";
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
+import { getMapResults, MapResult } from "../../lib/map-utils";
configDotenv();
-// Max Links that "Smart /map" can return
-const MAX_FIRE_ENGINE_RESULTS = 500;
-
-interface MapResult {
- success: boolean;
- job_id: string;
- time_taken: number;
- mapResults: MapDocument[];
-}
-
-function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
- const urlMap = new Map<string, MapDocument>();
-
- for (const doc of documents) {
- const existing = urlMap.get(doc.url);
-
- if (!existing) {
- urlMap.set(doc.url, doc);
- } else if (doc.title !== undefined && existing.title === undefined) {
- urlMap.set(doc.url, doc);
- }
- }
-
- return Array.from(urlMap.values());
-}
-
-async function queryIndex(
- url: string,
- limit: number,
- useIndex: boolean,
- includeSubdomains: boolean,
-): Promise<MapDocument[]> {
- if (!useIndex) {
- return [];
- }
-
- const urlSplits = generateURLSplits(url);
- if (urlSplits.length === 1) {
- const urlObj = new URL(url);
- const hostname = urlObj.hostname;
-
- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
- const [domainLinks, splitLinks] = await Promise.all([
- includeSubdomains
- ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)
- : [],
- queryIndexAtSplitLevelWithMeta(url, limit),
- ]);
-
- return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);
- } else {
- return await queryIndexAtSplitLevelWithMeta(url, limit);
- }
-}
-
-async function getMapResults({
- url,
- search,
- limit = MAX_MAP_LIMIT,
- includeSubdomains = true,
- crawlerOptions = {},
- teamId,
- allowExternalLinks,
- abort = new AbortController().signal, // noop
- filterByPath = true,
- flags,
- useIndex = true,
- location,
-}: {
- url: string;
- search?: string;
- limit?: number;
- includeSubdomains?: boolean;
- crawlerOptions?: any;
- teamId: string;
- origin?: string;
- includeMetadata?: boolean;
- allowExternalLinks?: boolean;
- abort?: AbortSignal;
- mock?: string;
- filterByPath?: boolean;
- flags: TeamFlags;
- useIndex?: boolean;
- location?: ScrapeOptions["location"];
-}): Promise<MapResult> {
- const functionStartTime = Date.now();
-
- const id = uuidv4();
- let mapResults: MapDocument[] = [];
- const zeroDataRetention = flags?.forceZDR ?? false;
-
- const sc: StoredCrawl = {
- originUrl: url,
- crawlerOptions: {
- ...crawlerOptions,
- limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
- scrapeOptions: undefined,
- },
- scrapeOptions: scrapeOptions.parse({
- ...(location ? { location } : {}),
- }),
- internalOptions: { teamId },
- team_id: teamId,
- createdAt: Date.now(),
- zeroDataRetention,
- };
-
- const crawler = crawlToCrawler(id, sc, flags);
-
- try {
- sc.robots = await crawler.getRobotsTxt(false, abort);
- crawler.importRobotsTxt(sc.robots);
- } catch (_) {
- // Robots.txt fetch failed, continue without it
- }
-
- // If sitemapOnly is true, only get links from sitemap
- if (crawlerOptions.sitemap === "only") {
- const sitemap = await crawler.tryGetSitemap(
- urls => {
- urls.forEach(x => {
- mapResults.push({
- url: x,
- });
- });
- },
- true,
- true,
- crawlerOptions.timeout ?? 30000,
- abort,
- crawlerOptions.useMock,
- );
-
- if (sitemap > 0) {
- mapResults = mapResults
- .slice(1)
- .map(x => {
- try {
- return {
- ...x,
- url: checkAndUpdateURLForMap(x.url).url.trim(),
- };
- } catch (_) {
- return null;
- }
- })
- .filter(x => x !== null) as MapDocument[];
- }
- } else {
- let urlWithoutWww = url.replace("www.", "");
- let mapUrl =
- search && allowExternalLinks
- ? `${search} ${urlWithoutWww}`
- : search
- ? `${search} site:${urlWithoutWww}`
- : `site:${url}`;
-
- const resultsPerPage = 100;
- const maxPages = Math.ceil(
- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,
- );
-
- const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = await redisEvictConnection.get(cacheKey);
-
- let pagePromises: (Promise<any> | any)[];
-
- if (cachedResult) {
- pagePromises = JSON.parse(cachedResult);
- } else {
- const fetchPage = async (page: number) => {
- return await fireEngineMap(
- mapUrl,
- {
- numResults: resultsPerPage,
- page: page,
- },
- abort,
- );
- };
-
- pagePromises = Array.from({ length: maxPages }, (_, i) =>
- fetchPage(i + 1),
- );
- }
-
- const [indexResults, searchResults] = await Promise.all([
- queryIndex(url, limit, useIndex, includeSubdomains),
- Promise.all(pagePromises),
- ]);
-
- if (!zeroDataRetention) {
- await redisEvictConnection.set(
- cacheKey,
- JSON.stringify(searchResults),
- "EX",
- 48 * 60 * 60,
- ); // Cache for 48 hours
- }
-
- if (indexResults.length > 0) {
- mapResults.push(...indexResults);
- }
-
- if (crawlerOptions.sitemap === "include") {
- try {
- await crawler.tryGetSitemap(
- urls => {
- mapResults.push(
- ...urls.map(x => ({
- url: x,
- })),
- );
- },
- true,
- false,
- crawlerOptions.timeout ?? 30000,
- abort,
- );
- } catch (e) {
- logger.warn("tryGetSitemap threw an error", { error: e });
- }
- }
-
- if (search) {
- mapResults = searchResults
- .flat()
- .map<MapDocument>(
- x =>
- ({
- url: x.url,
- title: x.title,
- description: x.description,
- }) satisfies MapDocument,
- )
- .concat(mapResults);
- } else {
- mapResults = mapResults.concat(
- searchResults.flat().map(x => ({
- url: x.url,
- title: x.title,
- description: x.description,
- })),
- );
- }
-
- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
- if (mapResults.length > minumumCutoff) {
- mapResults = mapResults.slice(0, minumumCutoff);
- }
-
- if (search) {
- const searchQuery = search.toLowerCase();
- mapResults = performCosineSimilarityV2(mapResults, searchQuery);
- }
- }
-
- mapResults = mapResults
- .map(x => {
- try {
- return {
- ...x,
- url: checkAndUpdateURLForMap(
- x.url,
- crawlerOptions.ignoreQueryParameters ?? true,
- ).url.trim(),
- };
- } catch (_) {
- return null;
- }
- })
- .filter(x => x !== null) as MapDocument[];
-
- mapResults = mapResults.filter(x => isSameDomain(x.url, url));
-
- if (!includeSubdomains) {
- mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));
- }
-
- if (filterByPath && !allowExternalLinks) {
- try {
- const urlObj = new URL(url);
- const urlPath = urlObj.pathname;
- // Only apply path filtering if the URL has a significant path (not just '/' or empty)
- // This means we only filter by path if the user has not selected a root domain
- if (urlPath && urlPath !== "/" && urlPath.length > 1) {
- mapResults = mapResults.filter(x => {
- try {
- const linkObj = new URL(x.url);
- return linkObj.pathname.startsWith(urlPath);
- } catch (e) {
- return false;
- }
- });
- }
- } catch (e) {
- // If URL parsing fails, continue without path filtering
- logger.warn(`Failed to parse URL for path filtering: ${url}`, {
- error: e,
- });
- }
- }
-
- mapResults = dedupeMapDocumentArray(mapResults);
- mapResults = mapResults.slice(0, limit);
-
- const totalTimeMs = Date.now() - functionStartTime;
-
- return {
- success: true,
- mapResults,
- job_id: id,
- time_taken: totalTimeMs,
- };
-}
-
export async function mapController(
req: RequestWithAuth<{}, MapResponse, MapRequest>,
res: Response<MapResponse>,
@@ -371,7 +37,7 @@ export async function mapController(
teamId: req.auth.team_id,
});
- let result: Awaited<ReturnType<typeof getMapResults>>;
+ let result: MapResult;
const abort = new AbortController();
try {
result = (await Promise.race([
@@ -386,6 +52,7 @@ export async function mapController(
},
origin: req.body.origin,
teamId: req.auth.team_id,
+ allowExternalLinks: req.body.allowExternalLinks,
abort: abort.signal,
mock: req.body.useMock,
filterByPath: req.body.filterByPath !== false,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
new file mode 100644
index 000000000..058f4b20a
--- /dev/null
+++ b/apps/api/src/lib/map-utils.ts
@@ -0,0 +1,324 @@
+import { v4 as uuidv4 } from "uuid";
+import { TeamFlags, MapDocument, scrapeOptions, ScrapeOptions } from "../controllers/v2/types";
+import { crawlToCrawler, StoredCrawl } from "./crawl-redis";
+import { checkAndUpdateURLForMap, isSameDomain, isSameSubdomain } from "./validateUrl";
+import { fireEngineMap } from "../search/fireEngine";
+import { redisEvictConnection } from "../services/redis";
+import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";
+import { performCosineSimilarityV2 } from "./map-cosine";
+
+// Max Links that "Smart /map" can return
+const MAX_FIRE_ENGINE_RESULTS = 500;
+const MAX_MAP_LIMIT = 1000;
+
+export interface MapResult {
+ success: boolean;
+ job_id: string;
+ time_taken: number;
+ mapResults: MapDocument[];
+}
+
+export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
+ const urlMap = new Map<string, MapDocument>();
+
+ for (const doc of documents) {
+ const existing = urlMap.get(doc.url);
+
+ if (!existing) {
+ urlMap.set(doc.url, doc);
+ } else if (doc.title !== undefined && existing.title === undefined) {
+ urlMap.set(doc.url, doc);
+ }
+ }
+
+ return Array.from(urlMap.values());
+}
+
+export async function queryIndex(
+ url: string,
+ limit: number,
+ useIndex: boolean,
+ includeSubdomains: boolean,
+): Promise<MapDocument[]> {
+ if (!useIndex) {
+ return [];
+ }
+
+ const urlSplits = generateURLSplits(url);
+ if (urlSplits.length === 1) {
+ const urlObj = new URL(url);
+ const hostname = urlObj.hostname;
+
+ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
+ const [domainLinks, splitLinks] = await Promise.all([
+ includeSubdomains
+ ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit)
+ : [],
+ queryIndexAtSplitLevelWithMeta(url, limit),
+ ]);
+
+ return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);
+ } else {
+ return await queryIndexAtSplitLevelWithMeta(url, limit);
+ }
+}
+
+export async function getMapResults({
+ url,
+ search,
+ limit = MAX_MAP_LIMIT,
+ includeSubdomains = true,
+ crawlerOptions = {},
+ teamId,
+ allowExternalLinks,
+ abort = new AbortController().signal,
+ filterByPath = true,
+ flags,
+ useIndex = true,
+ location,
+ maxFireEngineResults = MAX_FIRE_ENGINE_RESULTS,
+}: {
+ url: string;
+ search?: string;
+ limit?: number;
+ includeSubdomains?: boolean;
+ crawlerOptions?: any;
+ teamId: string;
+ origin?: string;
+ includeMetadata?: boolean;
+ allowExternalLinks?: boolean;
+ abort?: AbortSignal;
+ mock?: string;
+ filterByPath?: boolean;
+ flags: TeamFlags | null;
+ useIndex?: boolean;
+ location?: ScrapeOptions["location"];
+ maxFireEngineResults?: number;
+}): Promise<MapResult> {
+ const functionStartTime = Date.now();
+
+ const id = uuidv4();
+ let mapResults: MapDocument[] = [];
+ const zeroDataRetention = flags?.forceZDR ?? false;
+
+ const sc: StoredCrawl = {
+ originUrl: url,
+ crawlerOptions: {
+ ...crawlerOptions,
+ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
+ scrapeOptions: undefined,
+ },
+ scrapeOptions: scrapeOptions.parse({
+ ...(location ? { location } : {}),
+ }),
+ internalOptions: { teamId },
+ team_id: teamId,
+ createdAt: Date.now(),
+ zeroDataRetention,
+ };
+
+ const crawler = crawlToCrawler(id, sc, flags);
+
+ try {
+ sc.robots = await crawler.getRobotsTxt(false, abort);
+ crawler.importRobotsTxt(sc.robots);
+ } catch (_) {
+ // Robots.txt fetch failed, continue without it
+ }
+
+ // If sitemapOnly is true, only get links from sitemap
+ if (crawlerOptions.sitemap === "only") {
+ const sitemap = await crawler.tryGetSitemap(
+ urls => {
+ urls.forEach(x => {
+ mapResults.push({
+ url: x,
+ });
+ });
+ },
+ true,
+ true,
+ crawlerOptions.timeout ?? 30000,
+ abort,
+ crawlerOptions.useMock,
+ );
+
+ if (sitemap > 0) {
+ mapResults = mapResults
+ .slice(1)
+ .map(x => {
+ try {
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(x.url).url.trim(),
+ };
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter(x => x !== null) as MapDocument[];
+ }
+ } else {
+ let urlWithoutWww = url.replace("www.", "");
+ let mapUrl =
+ search && allowExternalLinks
+ ? `${search} ${urlWithoutWww}`
+ : search
+ ? `${search} site:${urlWithoutWww}`
+ : `site:${url}`;
+
+ const resultsPerPage = 100;
+ const maxPages = Math.ceil(
+ Math.min(maxFireEngineResults, limit) / resultsPerPage,
+ );
+
+ const cacheKey = `fireEngineMap:${mapUrl}`;
+ const cachedResult = await redisEvictConnection.get(cacheKey);
+
+ let pagePromises: (Promise<any> | any)[];
+
+ if (cachedResult) {
+ pagePromises = JSON.parse(cachedResult);
+ } else {
+ const fetchPage = async (page: number) => {
+ return await fireEngineMap(
+ mapUrl,
+ {
+ numResults: resultsPerPage,
+ page: page,
+ },
+ abort,
+ );
+ };
+
+ pagePromises = Array.from({ length: maxPages }, (_, i) =>
+ fetchPage(i + 1),
+ );
+ }
+
+ const [indexResults, searchResults] = await Promise.all([
+ queryIndex(url, limit, useIndex, includeSubdomains),
+ Promise.all(pagePromises),
+ ]);
+
+ if (!zeroDataRetention) {
+ await redisEvictConnection.set(
+ cacheKey,
+ JSON.stringify(searchResults),
+ "EX",
+ 48 * 60 * 60,
+ ); // Cache for 48 hours
+ }
+
+ if (indexResults.length > 0) {
+ mapResults.push(...indexResults);
+ }
+
+ if (crawlerOptions.sitemap === "include") {
+ try {
+ await crawler.tryGetSitemap(
+ urls => {
+ mapResults.push(
+ ...urls.map(x => ({
+ url: x,
+ })),
+ );
+ },
+ true,
+ false,
+ crawlerOptions.timeout ?? 30000,
+ abort,
+ );
+ } catch (e) {
+ // Silently handle sitemap errors
+ }
+ }
+
+ if (search) {
+ mapResults = searchResults
+ .flat()
+ .map<MapDocument>(
+ x =>
+ ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ }) satisfies MapDocument,
+ )
+ .concat(mapResults);
+ } else {
+ mapResults = mapResults.concat(
+ searchResults.flat().map(x => ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ })),
+ );
+ }
+
+ const minimumCutoff = Math.min(MAX_MAP_LIMIT, limit);
+ if (mapResults.length > minimumCutoff) {
+ mapResults = mapResults.slice(0, minimumCutoff);
+ }
+
+ if (search) {
+ const searchQuery = search.toLowerCase();
+ mapResults = performCosineSimilarityV2(mapResults, searchQuery);
+ }
+ }
+
+ mapResults = mapResults
+ .map(x => {
+ try {
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(
+ x.url,
+ crawlerOptions.ignoreQueryParameters ?? true,
+ ).url.trim(),
+ };
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter(x => x !== null) as MapDocument[];
+
+ mapResults = mapResults.filter(x => isSameDomain(x.url, url));
+
+ if (!includeSubdomains) {
+ mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));
+ }
+
+ if (filterByPath && !allowExternalLinks) {
+ try {
+ const urlObj = new URL(url);
+ const urlPath = urlObj.pathname;
+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)
+ // This means we only filter by path if the user has not selected a root domain
+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {
+ mapResults = mapResults.filter(x => {
+ try {
+ const linkObj = new URL(x.url);
+ return linkObj.pathname.startsWith(urlPath);
+ } catch (e) {
+ return false;
+ }
+ });
+ }
+ } catch (e) {
+ // If URL parsing fails, continue without path filtering
+ }
+ }
+
+ mapResults = dedupeMapDocumentArray(mapResults);
+ mapResults = mapResults.slice(0, limit);
+
+ const totalTimeMs = Date.now() - functionStartTime;
+
+ return {
+ success: true,
+ mapResults,
+ job_id: id,
+ time_taken: totalTimeMs,
+ };
+}