Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): implement URL modification handling in crawl controller
AdemÃlson Tonato8136e7fa6d2d
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b9affea48..b5fe0a4af 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -14,12 +14,17 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transfimport { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";+import { modifyCrawlUrl } from "../../utils/url-utils";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,res: Response<CrawlResponse>,) {const preNormalizedBody = req.body;++ // Check for URL modification before parsing+ const urlModificationInfo = modifyCrawlUrl(preNormalizedBody.url);+req.body = crawlRequestSchema.parse(req.body);const permissions = checkPermissions(req.body, req.acuc?.flags);@@ -219,6 +224,9 @@ export async function crawlController(success: true,id,url: `${protocol}://${req.get("host")}/v2/crawl/${id}`,+ ...(urlModificationInfo.wasModified && {+ warning: `The URL you provided included a '/*' suffix, which has been removed to ensure a more targeted and efficient crawl.`,+ }),...(req.body.prompt && {promptGeneratedOptions: promptGeneratedOptions,finalCrawlerOptions: finalCrawlerOptions,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex dd7c0c168..bb9e20f87 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -19,8 +19,10 @@ import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";import { integrationSchema } from "../../utils/integration";import { webhookSchema } from "../../services/webhook/schema";+import { modifyCrawlUrl } from "../../utils/url-utils";-export const url = z.preprocess(+// Base URL schema with common validation logic+const BASE_URL_SCHEMA = z.preprocess(x => {if (!protocolIncluded(x as string)) {x = `http://${x}`;@@ -60,6 +62,12 @@ export const url = z.preprocess(// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);+// Standard URL schema+const URL = BASE_URL_SCHEMA;++// Crawl URL schema with modification handling+const CRAWL_URL = BASE_URL_SCHEMA.transform(url => modifyCrawlUrl(url));+const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";@@ -563,8 +571,7 @@ const ajv = new Ajv();const extractOptions = z.object({- urls: url- .array()+ urls: URL.array().max(10, "Maximum of 10 URLs allowed per request while in beta.").optional(),prompt: z.string().max(10000).optional(),@@ -641,7 +648,7 @@ export type ExtractRequestInput = z.input<typeof extractRequestSchema>;export const scrapeRequestSchema = baseScrapeOptions.extend({- url,+ url: URL,origin: z.string().optional().default("api"),integration: integrationSchema.optional().transform(val => val || null),zeroDataRetention: z.boolean().optional(),@@ -655,7 +662,7 @@ export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;export const batchScrapeRequestSchema = baseScrapeOptions.extend({- urls: url.array(),+ urls: URL.array(),origin: z.string().optional().default("api"),integration: integrationSchema.optional().transform(val => val || null),webhook: webhookSchema.optional(),@@ -718,7 +725,7 @@ type CrawlerOptions = z.infer<typeof crawlerOptions>;export const crawlRequestSchema = crawlerOptions.extend({- url,+ url: CRAWL_URL,origin: z.string().optional().default("api"),integration: integrationSchema.optional().transform(val => val || null),scrapeOptions: baseScrapeOptions.default({}),@@ -733,6 +740,7 @@ export const crawlRequestSchema = crawlerOptions.transform(x => {return {...x,+ url: x.url.url, // Extract the actual URL from the CRAWL_URL resultscrapeOptions: extractTransform(x.scrapeOptions),};});@@ -757,7 +765,7 @@ export const MAX_MAP_LIMIT = 100000;export const mapRequestSchema = crawlerOptions.omit({ sitemap: true, ignoreQueryParameters: true }).extend({- url,+ url: URL,origin: z.string().optional().default("api"),integration: integrationSchema.optional().transform(val => val || null),includeSubdomains: z.boolean().default(true),@@ -943,6 +951,7 @@ export type CrawlResponse =success: true;id: string;url: string;+ warning?: string;};export type BatchScrapeResponse =@@ -1556,7 +1565,7 @@ export type TokenUsage = {};const generateLLMsTextRequestSchema = z.object({- url: url.describe("The URL to generate text from"),+ url: URL.describe("The URL to generate text from"),maxUrls: z.number().min(1)diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex ab0c2eb82..dbef33a44 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -1,10 +1,23 @@import { v4 as uuidv4 } from "uuid";-import { TeamFlags, MapDocument, scrapeOptions, ScrapeOptions } from "../controllers/v2/types";+import {+ TeamFlags,+ MapDocument,+ scrapeOptions,+ ScrapeOptions,+} from "../controllers/v2/types";import { crawlToCrawler, StoredCrawl } from "./crawl-redis";-import { checkAndUpdateURLForMap, isSameDomain, isSameSubdomain } from "./validateUrl";+import {+ checkAndUpdateURLForMap,+ isSameDomain,+ isSameSubdomain,+} from "./validateUrl";import { fireEngineMap } from "../search/fireEngine";import { redisEvictConnection } from "../services/redis";-import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";+import {+ generateURLSplits,+ queryIndexAtDomainSplitLevelWithMeta,+ queryIndexAtSplitLevelWithMeta,+} from "../services/index";import { performCosineSimilarityV2 } from "./map-cosine";import { Logger } from "winston";@@ -19,7 +32,7 @@ export interface MapResult {mapResults: MapDocument[];}-export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {+function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {const urlMap = new Map<string, MapDocument>();for (const doc of documents) {@@ -35,7 +48,7 @@ export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[]return Array.from(urlMap.values());}-export async function queryIndex(+async function queryIndex(url: string,limit: number,useIndex: boolean,diff --git a/apps/api/src/utils/url-utils.ts b/apps/api/src/utils/url-utils.tsnew file mode 100644index 000000000..1c848b38b--- /dev/null+++ b/apps/api/src/utils/url-utils.ts@@ -0,0 +1,27 @@+/**+ * Utility functions for URL handling and modification+ */++/**+ * Modifies a crawl URL by stripping the /* suffix if present+ * @param url - The URL to potentially modify+ * @returns Object containing the modified URL, modification flag, and original URL+ */+export function modifyCrawlUrl(url: string): {+ url: string;+ wasModified: boolean;+ originalUrl: string;+} {+ if (url.endsWith("/*")) {+ return {+ url: url.slice(0, -2),+ wasModified: true,+ originalUrl: url,+ };+ }+ return {+ url: url,+ wasModified: false,+ originalUrl: url,+ };+}