Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): add unnormalizedSourceURL for url matching DX (FIR-2137) (#1601)
Gergő Móricza36c6a4f401d
diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.tsindex f3e9e585b..8d4c39468 100644--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts@@ -48,4 +48,12 @@ describe("Batch scrape tests", () => {}, 180000);});}++ it.concurrent("sourceURL stays unnormalized", async () => {+ const response = await batchScrape({+ urls: ["https://firecrawl.dev/?pagewanted=all&et_blog"],+ });++ expect(response.body.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");+ }, 35000);});diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 4703964e6..9c2a9e2d5 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -366,4 +366,12 @@ describe("Scrape tests", () => {}, 30000);});}++ it.concurrent("sourceURL stays unnormalized", async () => {+ const response = await scrape({+ url: "https://firecrawl.dev/?pagewanted=all&et_blog",+ });++ expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");+ }, 35000);});diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 326aba5f3..d14f8cd7f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -22,7 +22,6 @@ import { getJobPriority } from "../../lib/job-priority";import { addScrapeJobs } from "../../services/queue-jobs";import { callWebhook } from "../../services/webhook";import { logger as _logger } from "../../lib/logger";-import { CostTracking } from "../../lib/extract/extraction-service";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";@@ -30,6 +29,8 @@ export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,res: Response<BatchScrapeResponse>,) {+ const preNormalizedBody = { ...req.body };+if (req.body?.ignoreInvalidURLs === true) {req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body);} else {@@ -46,6 +47,7 @@ export async function batchScrapeController(});let urls = req.body.urls;+ let unnormalizedURLs = preNormalizedBody.urls;let invalidURLs: string[] | undefined = undefined;if (req.body.ignoreInvalidURLs) {@@ -53,11 +55,13 @@ export async function batchScrapeController(let pendingURLs = urls;urls = [];+ unnormalizedURLs = [];for (const u of pendingURLs) {try {const nu = urlSchema.parse(u);if (!isUrlBlocked(nu, req.acuc?.flags ?? null)) {urls.push(nu);+ unnormalizedURLs.push(u);} else {invalidURLs.push(u);}@@ -86,12 +90,6 @@ export async function batchScrapeController(await logCrawl(id, req.auth.team_id);}- let { remainingCredits } = req.account!;- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (!useDbAuthentication) {- remainingCredits = Infinity;- }-const sc: StoredCrawl = req.body.appendToId? ((await getCrawl(req.body.appendToId)) as StoredCrawl): {@@ -127,7 +125,7 @@ export async function batchScrapeController(delete (scrapeOptions as any).urls;delete (scrapeOptions as any).appendToId;- const jobs = urls.map((x) => {+ const jobs = urls.map((x, i) => {return {data: {url: x,@@ -142,6 +140,7 @@ export async function batchScrapeController(webhook: req.body.webhook,internalOptions: {saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ unnormalizedSourceURL: unnormalizedURLs[i],},},opts: {diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 8da72dae6..092d86e8c 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -51,6 +51,7 @@ export async function scrapeController(internalOptions: {teamId: req.auth.team_id,saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ unnormalizedSourceURL: preNormalizedBody.url,},origin: req.body.origin,is_scrape: true,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 85254f372..fbc0c53b3 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -182,6 +182,7 @@ export type InternalOptions = {fromCache?: boolean; // Indicates if the document was retrieved from cacheabort?: AbortSignal;urlInvisibleInCurrentCrawl?: boolean;+ unnormalizedSourceURL?: string;saveScrapeResultToGCS?: boolean; // Passed along to fire-engine};@@ -373,7 +374,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {screenshot: result.result.screenshot,actions: result.result.actions,metadata: {- sourceURL: meta.url,+ sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,url: result.result.url,statusCode: result.result.statusCode,error: result.result.error,