Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add minAge parameter to scrape (#2452)
tom9f4f011a7834
diff --git a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts b/apps/api/src/__tests__/snips/v2/scrape-cache.test.tsindex e6d3b5bb8..529e6b8c9 100644--- a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts@@ -1,5 +1,5 @@import { describeIf, TEST_PRODUCTION } from "../lib";-import { Identity, idmux, scrapeTimeout, scrape } from "./lib";+import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";import crypto from "crypto";describeIf(TEST_PRODUCTION)("V2 Scrape Default maxAge", () => {@@ -119,4 +119,76 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Default maxAge", () => {},scrapeTimeout * 2 + 20000,);++ test(+ "should return error if cached data does not meet minAge requirement",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ // First scrape to populate cache+ const data1 = await scrape(+ {+ url,+ },+ identity,+ );++ expect(data1).toBeDefined();+ expect(data1.metadata.cacheState).toBe("miss");++ // Wait for index to be populated+ await new Promise(resolve => setTimeout(resolve, 20000));++ // Second scrape with minAge should fail+ const response = await scrapeRaw(+ {+ url,+ minAge: 60000,+ },+ identity,+ );++ expect(response.statusCode).toBe(404);+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("SCRAPE_NO_CACHED_DATA");+ },+ scrapeTimeout * 2 + 20000,+ );++ test(+ "should return cached data if it meets minAge requirement",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?testId=" + id;++ // First scrape to populate cache+ const data1 = await scrape(+ {+ url,+ },+ identity,+ );++ expect(data1).toBeDefined();+ expect(data1.metadata.cacheState).toBe("miss");++ // Wait for index to be populated and for data to age+ await new Promise(resolve => setTimeout(resolve, 35000));++ // Second scrape with minAge should hit cache+ const data2 = await scrape(+ {+ url,+ minAge: 30000,+ },+ identity,+ );++ expect(data2).toBeDefined();+ expect(data2.metadata.cacheState).toBe("hit");+ expect(data2.metadata.cachedAt).toBeDefined();+ },+ scrapeTimeout * 2 + 35000,+ );});diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex de66d59b2..d70be9cca 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -243,6 +243,17 @@ export async function scrapeController(});}+ if (e.code === "SCRAPE_NO_CACHED_DATA") {+ setSpanAttributes(span, {+ "scrape.status_code": 404,+ });+ return res.status(404).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 6593c1308..1599426c5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -553,6 +553,7 @@ const baseScrapeOptions = zblockAds: z.boolean().default(true),proxy: z.enum(["basic", "stealth", "auto"]).default("auto"),maxAge: z.number().int().gte(0).safe().optional(),+ minAge: z.number().int().gte(0).safe().optional(),storeInCache: z.boolean().default(true),// @deprecated__searchPreviewToken: z.string().optional(),diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 680c2137b..21936f81c 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -22,6 +22,7 @@ import {SiteError,SSLError,ProxySelectionError,+ NoCachedDataError,} from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -41,6 +42,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,+ SCRAPE_NO_CACHED_DATA: NoCachedDataError,SCRAPE_ACTION_ERROR: ActionError,SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,SCRAPE_SITEMAP_ERROR: SitemapError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 9bf3c90cd..6076748b9 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -16,6 +16,7 @@ export type ErrorCodes =| "SCRAPE_UNSUPPORTED_FILE_ERROR"| "SCRAPE_ACTION_ERROR"| "SCRAPE_RACED_REDIRECT_ERROR"+ | "SCRAPE_NO_CACHED_DATA"| "SCRAPE_SITEMAP_ERROR"| "CRAWL_DENIAL"| "BAD_REQUEST_INVALID_JSON"diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex e343d223c..7906fc9ac 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -13,7 +13,7 @@ import {generateDomainSplits,addOMCEJob,} from "../../../../services";-import { EngineError, IndexMissError } from "../../error";+import { EngineError, IndexMissError, NoCachedDataError } from "../../error";import { shouldParsePDF } from "../../../../controllers/v2/types";export async function sendDocumentToIndex(meta: Meta, document: Document) {@@ -250,7 +250,7 @@ export async function scrapeURLWithIndex(const checkpoint1 = Date.now();const { data, error } = await index_supabase_service.rpc(- "index_get_recent_2",+ "index_get_recent_3",{p_url_hash: urlHash,p_max_age_ms: maxAge,@@ -266,6 +266,7 @@ export async function scrapeURLWithIndex(? meta.options.location?.languages: null,p_wait_time_ms: meta.options.waitFor,+ p_min_age_ms: meta.options.minAge ?? null,},);@@ -303,6 +304,12 @@ export async function scrapeURLWithIndex(timingsMaxAge: checkpoint1 - startTime,timingsSupa: Date.now() - checkpoint1,});++ // when minAge is specified, don't waterfall to other engines+ if (meta.options.minAge !== undefined) {+ throw new NoCachedDataError();+ }+throw new IndexMissError();}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 77e246a55..f4d840368 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -274,6 +274,28 @@ export class IndexMissError extends Error {}}+export class NoCachedDataError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_NO_CACHED_DATA",+ "No cached data available that meets the specified age requirements.",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new NoCachedDataError();+ x.stack = data.stack;+ return x;+ }+}+export class ZDRViolationError extends TransportableError {constructor(public feature: string) {super(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0ba49a4b2..a474607a0 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -35,6 +35,7 @@ import {SSLError,PDFInsufficientTimeError,IndexMissError,+ NoCachedDataError,DNSResolutionError,ZDRViolationError,PDFPrefetchFailed,@@ -636,7 +637,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof PDFAntibotError ||error.error instanceof DocumentAntibotError ||error.error instanceof PDFInsufficientTimeError ||- error.error instanceof ProxySelectionError+ error.error instanceof ProxySelectionError ||+ error.error instanceof NoCachedDataError) {throw error.error;} else if (error.error instanceof LLMRefusalError) {diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 1b62e15b9..e7215f00a 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.7.0",+ "version": "4.8.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 9af559f91..badb39fd1 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -155,6 +155,7 @@ export interface ScrapeOptions {blockAds?: boolean;proxy?: 'basic' | 'stealth' | 'auto' | string;maxAge?: number;+ minAge?: number;storeInCache?: boolean;integration?: string;}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex e616fcee9..2d8182dca 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.9.0"+__version__ = "4.10.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 96bac96d3..afdfd270d 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -504,6 +504,7 @@ class ScrapeOptions(BaseModel):block_ads: Optional[bool] = Noneproxy: Optional[Literal["basic", "stealth", "auto"]] = Nonemax_age: Optional[int] = None+ min_age: Optional[int] = Nonestore_in_cache: Optional[bool] = Noneintegration: Optional[str] = None