Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
move param to the right place
Gergő Móriczdc3a4e27fdf0
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 8a01e5021..dd0faff78 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -143,7 +143,6 @@ export const scrapeOptions = z.object({}).optional(),skipTlsVerification: z.boolean().default(false),removeBase64Images: z.boolean().default(true),- deduplicateSimilarURLs: z.boolean().default(true),}).strict(strictMessage)@@ -200,6 +199,7 @@ const crawlerOptions = z.object({allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME???allowExternalLinks: z.boolean().default(false),ignoreSitemap: z.boolean().default(true),+ deduplicateSimilarURLs: z.boolean().default(true),}).strict(strictMessage);// export type CrawlerOptions = {diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex f1c1d9568..3d9182637 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -107,7 +107,7 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promiseurl = normalizeURL(url);let res: boolean;- if (!sc.scrapeOptions.deduplicateSimilarURLs) {+ if (!sc.crawlerOptions.deduplicateSimilarURLs) {res = (await redisConnection.sadd("crawl:" + id + ":visited", url)) !== 0} else {const urlO = new URL(url);diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 5c83170b8..4f68f3037 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -88,7 +88,6 @@ export interface CrawlScrapeOptions {};skipTlsVerification?: boolean;removeBase64Images?: boolean;- deduplicateSimilarURLs?: boolean;}export type Action = {@@ -151,6 +150,7 @@ export interface CrawlParams {ignoreSitemap?: boolean;scrapeOptions?: CrawlScrapeOptions;webhook?: string;+ deduplicateSimilarURLs?: boolean;}/**