Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): never invalidate first crawl scrape if redirects
Móricz Gergő7d73ebdbf12e
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 90c96cf64..6cda426aa 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -558,6 +558,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {crawl_id: job.data.crawl_id,webhook: job.data.webhook,v1: job.data.v1,+ isCrawlSourceScrape: true,},{priority: 15,@@ -721,9 +722,13 @@ async function processJob(job: Job & { id: string }, token: string) {if (crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL) === null) {- throw new Error(- "Redirected target URL is not allowed by crawlOptions",- ); // TODO: make this its own error type that is ignored by error tracking+ if (job.data.isCrawlSourceScrape) {+ // TODO: re-fetch sitemap for redirect target domain+ } else {+ throw new Error(+ "Redirected target URL is not allowed by crawlOptions",+ ); // TODO: make this its own error type that is ignored by error tracking+ }}if (isUrlBlocked(doc.metadata.url)) {diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 8f6a39d9c..4a43e75d8 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -44,6 +44,7 @@ export interface WebScraperOptions {webhook?: z.infer<typeof webhookSchema>;v1?: boolean;is_scrape?: boolean;+ isCrawlSourceScrape?: boolean;}export interface RunWebScraperParams {