Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

add comment

Móricz Gergő7a03275575c7
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 6cda426aa..9e8952166 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -724,6 +724,7 @@ async function processJob(job: Job & { id: string }, token: string) {
) {
if (job.data.isCrawlSourceScrape) {
// TODO: re-fetch sitemap for redirect target domain
+ // TODO: reset crawl source url to new target
} else {
throw new Error(
"Redirected target URL is not allowed by crawlOptions",