Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): do not follow redirect URLs if they are not allowed by the crawl options

Gergő Móricz9005757de347
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 3cc04a11e..bf57f63f3 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -42,6 +42,11 @@ export const logger = winston.createLogger({
},
}),
transports: [
+ ...(process.env.FIRECRAWL_LOG_TO_FILE ? [
+ new winston.transports.File({
+ filename: "firecrawl-" + (process.argv[1].includes("worker") ? "worker" : "app") + "-" + crypto.randomUUID() + ".log",
+ })
+ ] : []),
new winston.transports.Console({
format: winston.format.combine(
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index e8c8bdf30..48e6f7fd0 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -46,6 +46,8 @@ import {
removeConcurrencyLimitActiveJob,
takeConcurrencyLimitedJob,
} from "../lib/concurrency-limit";
+import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
+import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
configDotenv();
class RacedRedirectError extends Error {
@@ -620,6 +622,15 @@ async function processJob(job: Job & { id: string }, token: string) {
normalizeURL(doc.metadata.url, sc) !==
normalizeURL(doc.metadata.sourceURL, sc)
) {
+ const crawler = crawlToCrawler(job.data.crawl_id, sc);
+ if (crawler.filterURL(doc.metadata.url, doc.metadata.sourceURL) === null) {
+ throw new Error("Redirected target URL is not allowed by crawlOptions"); // TODO: make this its own error type that is ignored by error tracking
+ }
+
+ if (isUrlBlocked(doc.metadata.url)) {
+ throw new Error(BLOCKLISTED_URL_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
+ }
+
const p1 = generateURLPermutations(normalizeURL(doc.metadata.url, sc));
const p2 = generateURLPermutations(
normalizeURL(doc.metadata.sourceURL, sc),