Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "feat(crawl): implement URL modification handling in crawl controller" (#2558)

tom1400e227abcd
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index b87bce480..b36521368 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -20,7 +20,6 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transf
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
-import { modifyCrawlUrl } from "../../utils/url-utils";
import { crawlGroup } from "../../services/worker/nuq";
import { logRequest } from "../../services/logging/log_job";
@@ -29,10 +28,6 @@ export async function crawlController(
res: Response<CrawlResponse>,
) {
const preNormalizedBody = req.body;
-
- // Check for URL modification before parsing
- const urlModificationInfo = modifyCrawlUrl(preNormalizedBody.url);
-
req.body = crawlRequestSchema.parse(req.body);
const permissions = checkPermissions(req.body, req.acuc?.flags);
@@ -250,9 +245,6 @@ export async function crawlController(
success: true,
id,
url: `${protocol}://${req.get("host")}/v2/crawl/${id}`,
- ...(urlModificationInfo.wasModified && {
- warning: `The URL you provided included a '/*' suffix, which has been removed to ensure a more targeted and efficient crawl.`,
- }),
...(req.body.prompt && {
promptGeneratedOptions: promptGeneratedOptions,
finalCrawlerOptions: finalCrawlerOptions,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index d9080c7e6..856917296 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -21,11 +21,10 @@ import { ErrorCodes } from "../../lib/error";
import Ajv from "ajv";
import { integrationSchema } from "../../utils/integration";
import { webhookSchema } from "../../services/webhook/schema";
-import { modifyCrawlUrl } from "../../utils/url-utils";
import { BrandingProfile } from "../../types/branding";
// Base URL schema with common validation logic
-const BASE_URL_SCHEMA = z.preprocess(
+const URL = z.preprocess(
x => {
if (!protocolIncluded(x as string)) {
x = `http://${x}`;
@@ -71,12 +70,6 @@ const BASE_URL_SCHEMA = z.preprocess(
// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
);
-// Standard URL schema
-export const URL = BASE_URL_SCHEMA;
-
-// Crawl URL schema with modification handling
-const CRAWL_URL = BASE_URL_SCHEMA.transform(url => modifyCrawlUrl(url));
-
const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
@@ -851,7 +844,7 @@ export const crawlerOptions = z.strictObject({
type CrawlerOptions = z.infer<typeof crawlerOptions>;
const crawlRequestSchemaBase = crawlerOptions.extend({
- url: CRAWL_URL,
+ url: URL,
origin: z.string().optional().prefault("api"),
integration: integrationSchema.optional().transform(val => val || null),
scrapeOptions: baseScrapeOptions.prefault(() => baseScrapeOptions.parse({})),
@@ -868,7 +861,7 @@ export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase)
const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});
return {
...x,
- url: x.url.url, // Extract the actual URL from the CRAWL_URL result
+ url: x.url,
scrapeOptions: extractTransformRequired(scrapeOptionsValue),
};
});
@@ -1104,7 +1097,6 @@ export type CrawlResponse =
success: true;
id: string;
url: string;
- warning?: string;
};
export type BatchScrapeResponse =
diff --git a/apps/api/src/utils/url-utils.ts b/apps/api/src/utils/url-utils.ts
deleted file mode 100644
index 1c848b38b..000000000
--- a/apps/api/src/utils/url-utils.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Utility functions for URL handling and modification
- */
-
-/**
- * Modifies a crawl URL by stripping the /* suffix if present
- * @param url - The URL to potentially modify
- * @returns Object containing the modified URL, modification flag, and original URL
- */
-export function modifyCrawlUrl(url: string): {
- url: string;
- wasModified: boolean;
- originalUrl: string;
-} {
- if (url.endsWith("/*")) {
- return {
- url: url.slice(0, -2),
- wasModified: true,
- originalUrl: url,
- };
- }
- return {
- url: url,
- wasModified: false,
- originalUrl: url,
- };
-}