Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
add comment
Móricz Gergő7a03275575c7
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 6cda426aa..9e8952166 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -724,6 +724,7 @@ async function processJob(job: Job & { id: string }, token: string) {) {if (job.data.isCrawlSourceScrape) {// TODO: re-fetch sitemap for redirect target domain+ // TODO: reset crawl source url to new target} else {throw new Error("Redirected target URL is not allowed by crawlOptions",