Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl): never invalidate first crawl scrape if redirects

Móricz Gergő7d73ebdbf12e
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 90c96cf64..6cda426aa 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -558,6 +558,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
crawl_id: job.data.crawl_id,
webhook: job.data.webhook,
v1: job.data.v1,
+ isCrawlSourceScrape: true,
},
{
priority: 15,
@@ -721,9 +722,13 @@ async function processJob(job: Job & { id: string }, token: string) {
if (
crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL) === null
) {
- throw new Error(
- "Redirected target URL is not allowed by crawlOptions",
- ); // TODO: make this its own error type that is ignored by error tracking
+ if (job.data.isCrawlSourceScrape) {
+ // TODO: re-fetch sitemap for redirect target domain
+ } else {
+ throw new Error(
+ "Redirected target URL is not allowed by crawlOptions",
+ ); // TODO: make this its own error type that is ignored by error tracking
+ }
}
if (isUrlBlocked(doc.metadata.url)) {
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 8f6a39d9c..4a43e75d8 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -44,6 +44,7 @@ export interface WebScraperOptions {
webhook?: z.infer<typeof webhookSchema>;
v1?: boolean;
is_scrape?: boolean;
+ isCrawlSourceScrape?: boolean;
}
export interface RunWebScraperParams {