Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "feat(crawl): implement URL modification handling in crawl controller" (#2558)
tom1400e227abcd
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b87bce480..b36521368 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -20,7 +20,6 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transfimport { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";-import { modifyCrawlUrl } from "../../utils/url-utils";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";@@ -29,10 +28,6 @@ export async function crawlController(res: Response<CrawlResponse>,) {const preNormalizedBody = req.body;-- // Check for URL modification before parsing- const urlModificationInfo = modifyCrawlUrl(preNormalizedBody.url);-req.body = crawlRequestSchema.parse(req.body);const permissions = checkPermissions(req.body, req.acuc?.flags);@@ -250,9 +245,6 @@ export async function crawlController(success: true,id,url: `${protocol}://${req.get("host")}/v2/crawl/${id}`,- ...(urlModificationInfo.wasModified && {- warning: `The URL you provided included a '/*' suffix, which has been removed to ensure a more targeted and efficient crawl.`,- }),...(req.body.prompt && {promptGeneratedOptions: promptGeneratedOptions,finalCrawlerOptions: finalCrawlerOptions,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex d9080c7e6..856917296 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -21,11 +21,10 @@ import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";import { integrationSchema } from "../../utils/integration";import { webhookSchema } from "../../services/webhook/schema";-import { modifyCrawlUrl } from "../../utils/url-utils";import { BrandingProfile } from "../../types/branding";// Base URL schema with common validation logic-const BASE_URL_SCHEMA = z.preprocess(+const URL = z.preprocess(x => {if (!protocolIncluded(x as string)) {x = `http://${x}`;@@ -71,12 +70,6 @@ const BASE_URL_SCHEMA = z.preprocess(// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),);-// Standard URL schema-export const URL = BASE_URL_SCHEMA;--// Crawl URL schema with modification handling-const CRAWL_URL = BASE_URL_SCHEMA.transform(url => modifyCrawlUrl(url));-const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";@@ -851,7 +844,7 @@ export const crawlerOptions = z.strictObject({type CrawlerOptions = z.infer<typeof crawlerOptions>;const crawlRequestSchemaBase = crawlerOptions.extend({- url: CRAWL_URL,+ url: URL,origin: z.string().optional().prefault("api"),integration: integrationSchema.optional().transform(val => val || null),scrapeOptions: baseScrapeOptions.prefault(() => baseScrapeOptions.parse({})),@@ -868,7 +861,7 @@ export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase)const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});return {...x,- url: x.url.url, // Extract the actual URL from the CRAWL_URL result+ url: x.url,scrapeOptions: extractTransformRequired(scrapeOptionsValue),};});@@ -1104,7 +1097,6 @@ export type CrawlResponse =success: true;id: string;url: string;- warning?: string;};export type BatchScrapeResponse =diff --git a/apps/api/src/utils/url-utils.ts b/apps/api/src/utils/url-utils.tsdeleted file mode 100644index 1c848b38b..000000000--- a/apps/api/src/utils/url-utils.ts+++ /dev/null@@ -1,27 +0,0 @@-/**- * Utility functions for URL handling and modification- */--/**- * Modifies a crawl URL by stripping the /* suffix if present- * @param url - The URL to potentially modify- * @returns Object containing the modified URL, modification flag, and original URL- */-export function modifyCrawlUrl(url: string): {- url: string;- wasModified: boolean;- originalUrl: string;-} {- if (url.endsWith("/*")) {- return {- url: url.slice(0, -2),- wasModified: true,- originalUrl: url,- };- }- return {- url: url,- wasModified: false,- originalUrl: url,- };-}