Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(crawl): implement URL modification handling in crawl controller

Ademílson Tonato8136e7fa6d2d
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index b9affea48..b5fe0a4af 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -14,12 +14,17 @@ import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transf
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
+import { modifyCrawlUrl } from "../../utils/url-utils";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
res: Response<CrawlResponse>,
) {
const preNormalizedBody = req.body;
+
+ // Check for URL modification before parsing
+ const urlModificationInfo = modifyCrawlUrl(preNormalizedBody.url);
+
req.body = crawlRequestSchema.parse(req.body);
const permissions = checkPermissions(req.body, req.acuc?.flags);
@@ -219,6 +224,9 @@ export async function crawlController(
success: true,
id,
url: `${protocol}://${req.get("host")}/v2/crawl/${id}`,
+ ...(urlModificationInfo.wasModified && {
+ warning: `The URL you provided included a '/*' suffix, which has been removed to ensure a more targeted and efficient crawl.`,
+ }),
...(req.body.prompt && {
promptGeneratedOptions: promptGeneratedOptions,
finalCrawlerOptions: finalCrawlerOptions,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index dd7c0c168..bb9e20f87 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -19,8 +19,10 @@ import { ErrorCodes } from "../../lib/error";
import Ajv from "ajv";
import { integrationSchema } from "../../utils/integration";
import { webhookSchema } from "../../services/webhook/schema";
+import { modifyCrawlUrl } from "../../utils/url-utils";
-export const url = z.preprocess(
+// Base URL schema with common validation logic
+const BASE_URL_SCHEMA = z.preprocess(
x => {
if (!protocolIncluded(x as string)) {
x = `http://${x}`;
@@ -60,6 +62,12 @@ export const url = z.preprocess(
// .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
);
+// Standard URL schema
+const URL = BASE_URL_SCHEMA;
+
+// Crawl URL schema with modification handling
+const CRAWL_URL = BASE_URL_SCHEMA.transform(url => modifyCrawlUrl(url));
+
const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
@@ -563,8 +571,7 @@ const ajv = new Ajv();
const extractOptions = z
.object({
- urls: url
- .array()
+ urls: URL.array()
.max(10, "Maximum of 10 URLs allowed per request while in beta.")
.optional(),
prompt: z.string().max(10000).optional(),
@@ -641,7 +648,7 @@ export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
export const scrapeRequestSchema = baseScrapeOptions
.extend({
- url,
+ url: URL,
origin: z.string().optional().default("api"),
integration: integrationSchema.optional().transform(val => val || null),
zeroDataRetention: z.boolean().optional(),
@@ -655,7 +662,7 @@ export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;
export const batchScrapeRequestSchema = baseScrapeOptions
.extend({
- urls: url.array(),
+ urls: URL.array(),
origin: z.string().optional().default("api"),
integration: integrationSchema.optional().transform(val => val || null),
webhook: webhookSchema.optional(),
@@ -718,7 +725,7 @@ type CrawlerOptions = z.infer<typeof crawlerOptions>;
export const crawlRequestSchema = crawlerOptions
.extend({
- url,
+ url: CRAWL_URL,
origin: z.string().optional().default("api"),
integration: integrationSchema.optional().transform(val => val || null),
scrapeOptions: baseScrapeOptions.default({}),
@@ -733,6 +740,7 @@ export const crawlRequestSchema = crawlerOptions
.transform(x => {
return {
...x,
+ url: x.url.url, // Extract the actual URL from the CRAWL_URL result
scrapeOptions: extractTransform(x.scrapeOptions),
};
});
@@ -757,7 +765,7 @@ export const MAX_MAP_LIMIT = 100000;
export const mapRequestSchema = crawlerOptions
.omit({ sitemap: true, ignoreQueryParameters: true })
.extend({
- url,
+ url: URL,
origin: z.string().optional().default("api"),
integration: integrationSchema.optional().transform(val => val || null),
includeSubdomains: z.boolean().default(true),
@@ -943,6 +951,7 @@ export type CrawlResponse =
success: true;
id: string;
url: string;
+ warning?: string;
};
export type BatchScrapeResponse =
@@ -1556,7 +1565,7 @@ export type TokenUsage = {
};
const generateLLMsTextRequestSchema = z.object({
- url: url.describe("The URL to generate text from"),
+ url: URL.describe("The URL to generate text from"),
maxUrls: z
.number()
.min(1)
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index ab0c2eb82..dbef33a44 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -1,10 +1,23 @@
import { v4 as uuidv4 } from "uuid";
-import { TeamFlags, MapDocument, scrapeOptions, ScrapeOptions } from "../controllers/v2/types";
+import {
+ TeamFlags,
+ MapDocument,
+ scrapeOptions,
+ ScrapeOptions,
+} from "../controllers/v2/types";
import { crawlToCrawler, StoredCrawl } from "./crawl-redis";
-import { checkAndUpdateURLForMap, isSameDomain, isSameSubdomain } from "./validateUrl";
+import {
+ checkAndUpdateURLForMap,
+ isSameDomain,
+ isSameSubdomain,
+} from "./validateUrl";
import { fireEngineMap } from "../search/fireEngine";
import { redisEvictConnection } from "../services/redis";
-import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../services/index";
+import {
+ generateURLSplits,
+ queryIndexAtDomainSplitLevelWithMeta,
+ queryIndexAtSplitLevelWithMeta,
+} from "../services/index";
import { performCosineSimilarityV2 } from "./map-cosine";
import { Logger } from "winston";
@@ -19,7 +32,7 @@ export interface MapResult {
mapResults: MapDocument[];
}
-export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
+function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
const urlMap = new Map<string, MapDocument>();
for (const doc of documents) {
@@ -35,7 +48,7 @@ export function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[]
return Array.from(urlMap.values());
}
-export async function queryIndex(
+async function queryIndex(
url: string,
limit: number,
useIndex: boolean,
diff --git a/apps/api/src/utils/url-utils.ts b/apps/api/src/utils/url-utils.ts
new file mode 100644
index 000000000..1c848b38b
--- /dev/null
+++ b/apps/api/src/utils/url-utils.ts
@@ -0,0 +1,27 @@
+/**
+ * Utility functions for URL handling and modification
+ */
+
+/**
+ * Modifies a crawl URL by stripping the /* suffix if present
+ * @param url - The URL to potentially modify
+ * @returns Object containing the modified URL, modification flag, and original URL
+ */
+export function modifyCrawlUrl(url: string): {
+ url: string;
+ wasModified: boolean;
+ originalUrl: string;
+} {
+ if (url.endsWith("/*")) {
+ return {
+ url: url.slice(0, -2),
+ wasModified: true,
+ originalUrl: url,
+ };
+ }
+ return {
+ url: url,
+ wasModified: false,
+ originalUrl: url,
+ };
+}