Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add minAge parameter to scrape (#2452)

tom9f4f011a7834
diff --git a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
index e6d3b5bb8..529e6b8c9 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
@@ -1,5 +1,5 @@
import { describeIf, TEST_PRODUCTION } from "../lib";
-import { Identity, idmux, scrapeTimeout, scrape } from "./lib";
+import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";
import crypto from "crypto";
describeIf(TEST_PRODUCTION)("V2 Scrape Default maxAge", () => {
@@ -119,4 +119,76 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Default maxAge", () => {
},
scrapeTimeout * 2 + 20000,
);
+
+ test(
+ "should return error if cached data does not meet minAge requirement",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ // First scrape to populate cache
+ const data1 = await scrape(
+ {
+ url,
+ },
+ identity,
+ );
+
+ expect(data1).toBeDefined();
+ expect(data1.metadata.cacheState).toBe("miss");
+
+ // Wait for index to be populated
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ // Second scrape with minAge should fail
+ const response = await scrapeRaw(
+ {
+ url,
+ minAge: 60000,
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(404);
+ expect(response.body.success).toBe(false);
+ expect(response.body.code).toBe("SCRAPE_NO_CACHED_DATA");
+ },
+ scrapeTimeout * 2 + 20000,
+ );
+
+ test(
+ "should return cached data if it meets minAge requirement",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?testId=" + id;
+
+ // First scrape to populate cache
+ const data1 = await scrape(
+ {
+ url,
+ },
+ identity,
+ );
+
+ expect(data1).toBeDefined();
+ expect(data1.metadata.cacheState).toBe("miss");
+
+ // Wait for index to be populated and for data to age
+ await new Promise(resolve => setTimeout(resolve, 35000));
+
+ // Second scrape with minAge should hit cache
+ const data2 = await scrape(
+ {
+ url,
+ minAge: 30000,
+ },
+ identity,
+ );
+
+ expect(data2).toBeDefined();
+ expect(data2.metadata.cacheState).toBe("hit");
+ expect(data2.metadata.cachedAt).toBeDefined();
+ },
+ scrapeTimeout * 2 + 35000,
+ );
});
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index de66d59b2..d70be9cca 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -243,6 +243,17 @@ export async function scrapeController(
});
}
+ if (e.code === "SCRAPE_NO_CACHED_DATA") {
+ setSpanAttributes(span, {
+ "scrape.status_code": 404,
+ });
+ return res.status(404).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;
setSpanAttributes(span, {
"scrape.status_code": statusCode,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 6593c1308..1599426c5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -553,6 +553,7 @@ const baseScrapeOptions = z
blockAds: z.boolean().default(true),
proxy: z.enum(["basic", "stealth", "auto"]).default("auto"),
maxAge: z.number().int().gte(0).safe().optional(),
+ minAge: z.number().int().gte(0).safe().optional(),
storeInCache: z.boolean().default(true),
// @deprecated
__searchPreviewToken: z.string().optional(),
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 680c2137b..21936f81c 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -22,6 +22,7 @@ import {
SiteError,
SSLError,
ProxySelectionError,
+ NoCachedDataError,
} from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -41,6 +42,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,
SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,
SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,
+ SCRAPE_NO_CACHED_DATA: NoCachedDataError,
SCRAPE_ACTION_ERROR: ActionError,
SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,
SCRAPE_SITEMAP_ERROR: SitemapError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 9bf3c90cd..6076748b9 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -16,6 +16,7 @@ export type ErrorCodes =
| "SCRAPE_UNSUPPORTED_FILE_ERROR"
| "SCRAPE_ACTION_ERROR"
| "SCRAPE_RACED_REDIRECT_ERROR"
+ | "SCRAPE_NO_CACHED_DATA"
| "SCRAPE_SITEMAP_ERROR"
| "CRAWL_DENIAL"
| "BAD_REQUEST_INVALID_JSON"
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index e343d223c..7906fc9ac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -13,7 +13,7 @@ import {
generateDomainSplits,
addOMCEJob,
} from "../../../../services";
-import { EngineError, IndexMissError } from "../../error";
+import { EngineError, IndexMissError, NoCachedDataError } from "../../error";
import { shouldParsePDF } from "../../../../controllers/v2/types";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -250,7 +250,7 @@ export async function scrapeURLWithIndex(
const checkpoint1 = Date.now();
const { data, error } = await index_supabase_service.rpc(
- "index_get_recent_2",
+ "index_get_recent_3",
{
p_url_hash: urlHash,
p_max_age_ms: maxAge,
@@ -266,6 +266,7 @@ export async function scrapeURLWithIndex(
? meta.options.location?.languages
: null,
p_wait_time_ms: meta.options.waitFor,
+ p_min_age_ms: meta.options.minAge ?? null,
},
);
@@ -303,6 +304,12 @@ export async function scrapeURLWithIndex(
timingsMaxAge: checkpoint1 - startTime,
timingsSupa: Date.now() - checkpoint1,
});
+
+ // when minAge is specified, don't waterfall to other engines
+ if (meta.options.minAge !== undefined) {
+ throw new NoCachedDataError();
+ }
+
throw new IndexMissError();
}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 77e246a55..f4d840368 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -274,6 +274,28 @@ export class IndexMissError extends Error {
}
}
+export class NoCachedDataError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_NO_CACHED_DATA",
+ "No cached data available that meets the specified age requirements.",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new NoCachedDataError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class ZDRViolationError extends TransportableError {
constructor(public feature: string) {
super(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0ba49a4b2..a474607a0 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -35,6 +35,7 @@ import {
SSLError,
PDFInsufficientTimeError,
IndexMissError,
+ NoCachedDataError,
DNSResolutionError,
ZDRViolationError,
PDFPrefetchFailed,
@@ -636,7 +637,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof PDFAntibotError ||
error.error instanceof DocumentAntibotError ||
error.error instanceof PDFInsufficientTimeError ||
- error.error instanceof ProxySelectionError
+ error.error instanceof ProxySelectionError ||
+ error.error instanceof NoCachedDataError
) {
throw error.error;
} else if (error.error instanceof LLMRefusalError) {
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 1b62e15b9..e7215f00a 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.7.0",
+ "version": "4.8.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 9af559f91..badb39fd1 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -155,6 +155,7 @@ export interface ScrapeOptions {
blockAds?: boolean;
proxy?: 'basic' | 'stealth' | 'auto' | string;
maxAge?: number;
+ minAge?: number;
storeInCache?: boolean;
integration?: string;
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index e616fcee9..2d8182dca 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.9.0"
+__version__ = "4.10.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 96bac96d3..afdfd270d 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -504,6 +504,7 @@ class ScrapeOptions(BaseModel):
block_ads: Optional[bool] = None
proxy: Optional[Literal["basic", "stealth", "auto"]] = None
max_age: Optional[int] = None
+ min_age: Optional[int] = None
store_in_cache: Optional[bool] = None
integration: Optional[str] = None