Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sdks): add lockdown scrape option to all SDKs (ENG-4815) (#3409)

Abimael Martell8802eadf8872
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index 392ad2ab1..5e8ba9ea5 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -913,12 +913,13 @@ defmodule Firecrawl do
remove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],
skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],
store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],
+ lockdown: [type: :boolean, doc: "If true, only previously cached results are served and no outbound request is ever made. If nothing is cached for the URL, a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS is returned. Other options are accepted but ignored in lockdown mode."],
timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],
wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this scrape. To enable this feature, please contact [email protected]"]
])
- @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
+ @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
@doc """
Scrape a single URL and optionally extract information using an LLM
@@ -977,12 +978,13 @@ defmodule Firecrawl do
remove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],
skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],
store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],
+ lockdown: [type: :boolean, doc: "If true, only previously cached results are served and no outbound request is ever made. If nothing is cached for the URL, a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS is returned. Other options are accepted but ignored in lockdown mode."],
timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],
wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],
zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this batch scrape. To enable this feature, please contact [email protected]"]
])
- @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
+ @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}
@doc """
Scrape multiple URLs and optionally extract information using an LLM
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 174f62fe8..17950d843 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.0.1"
+ @version "1.1.0"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go
index 93559aafe..a824be667 100644
--- a/apps/go-sdk/options.go
+++ b/apps/go-sdk/options.go
@@ -19,6 +19,7 @@ type ScrapeOptions struct {
Proxy *string `json:"proxy,omitempty"`
MaxAge *int64 `json:"maxAge,omitempty"`
StoreInCache *bool `json:"storeInCache,omitempty"`
+ Lockdown *bool `json:"lockdown,omitempty"`
Integration *string `json:"integration,omitempty"`
JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
}
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index f81359cc4..316fca43c 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.2.2"
+version = "1.3.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
index a8560bc76..222704900 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
@@ -32,6 +32,7 @@ public class ScrapeOptions {
@JsonProperty("maxAge")
private Long maxAge;
private Boolean storeInCache;
+ private Boolean lockdown;
private String integration;
private ScrapeOptions() {}
@@ -53,6 +54,7 @@ public class ScrapeOptions {
public String getProxy() { return proxy; }
public Long getMaxAge() { return maxAge; }
public Boolean getStoreInCache() { return storeInCache; }
+ public Boolean getLockdown() { return lockdown; }
public String getIntegration() { return integration; }
public static Builder builder() { return new Builder(); }
@@ -76,6 +78,7 @@ public class ScrapeOptions {
b.proxy = this.proxy;
b.maxAge = this.maxAge;
b.storeInCache = this.storeInCache;
+ b.lockdown = this.lockdown;
b.integration = this.integration;
return b;
}
@@ -98,6 +101,7 @@ public class ScrapeOptions {
private String proxy;
private Long maxAge;
private Boolean storeInCache;
+ private Boolean lockdown;
private String integration;
private Builder() {}
@@ -157,6 +161,9 @@ public class ScrapeOptions {
/** Whether to cache the result. */
public Builder storeInCache(Boolean storeInCache) { this.storeInCache = storeInCache; return this; }
+ /** Lockdown mode: serve only previously cached results, never make outbound requests. */
+ public Builder lockdown(Boolean lockdown) { this.lockdown = lockdown; return this; }
+
/** Integration identifier. */
public Builder integration(String integration) { this.integration = integration; return this; }
@@ -179,6 +186,7 @@ public class ScrapeOptions {
o.proxy = this.proxy;
o.maxAge = this.maxAge;
o.storeInCache = this.storeInCache;
+ o.lockdown = this.lockdown;
o.integration = this.integration;
return o;
}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 6bdf3619d..55ee202d1 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.18.5",
+ "version": "4.19.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
index 22e7540e1..cd0f16a2b 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
@@ -37,4 +37,22 @@ describe("v2.parse unit", () => {
),
).rejects.toThrow("parse does not support changeTracking format");
});
+
+ test("rejects lockdown option before making requests", async () => {
+ const client = new FirecrawlClient({
+ apiKey: "test-key",
+ apiUrl: "https://localhost:3002",
+ });
+
+ await expect(
+ client.parse(
+ {
+ data: "<html><body>test</body></html>",
+ filename: "upload.html",
+ contentType: "text/html",
+ },
+ { formats: ["markdown"], lockdown: true } as any,
+ ),
+ ).rejects.toThrow("parse does not support cache/index options");
+ });
});
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 0dd88048d..51ce6c262 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -180,6 +180,7 @@ export interface ScrapeOptions {
maxAge?: number;
minAge?: number;
storeInCache?: boolean;
+ lockdown?: boolean;
profile?: {
name: string;
saveChanges?: boolean;
@@ -212,6 +213,7 @@ export type ParseOptions = Omit<
| 'maxAge'
| 'minAge'
| 'storeInCache'
+ | 'lockdown'
| 'proxy'
> & {
formats?: ParseFormatOption[];
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
index 5222d0325..afb91a8cc 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
@@ -139,7 +139,12 @@ export function ensureValidParseOptions(options?: ParseOptions): void {
if (raw.mobile !== undefined) {
throw new Error("parse does not support mobile rendering");
}
- if (raw.maxAge !== undefined || raw.minAge !== undefined || raw.storeInCache !== undefined) {
+ if (
+ raw.maxAge !== undefined ||
+ raw.minAge !== undefined ||
+ raw.storeInCache !== undefined ||
+ raw.lockdown !== undefined
+ ) {
throw new Error("parse does not support cache/index options");
}
if (raw.proxy !== undefined && raw.proxy !== "basic" && raw.proxy !== "auto") {
diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.php
index b91126530..28153bf57 100644
--- a/apps/php-sdk/src/Models/ScrapeOptions.php
+++ b/apps/php-sdk/src/Models/ScrapeOptions.php
@@ -32,6 +32,7 @@ final class ScrapeOptions
private readonly ?string $proxy = null,
private readonly ?int $maxAge = null,
private readonly ?bool $storeInCache = null,
+ private readonly ?bool $lockdown = null,
private readonly ?string $integration = null,
) {}
@@ -62,12 +63,13 @@ final class ScrapeOptions
?int $maxAge = null,
?bool $storeInCache = null,
?string $integration = null,
+ ?bool $lockdown = null,
): self {
return new self(
$formats, $headers, $includeTags, $excludeTags, $onlyMainContent,
$timeout, $waitFor, $mobile, $parsers, $actions, $location,
$skipTlsVerification, $removeBase64Images, $blockAds, $proxy,
- $maxAge, $storeInCache, $integration,
+ $maxAge, $storeInCache, $lockdown, $integration,
);
}
@@ -100,6 +102,7 @@ final class ScrapeOptions
'proxy' => $this->proxy,
'maxAge' => $this->maxAge,
'storeInCache' => $this->storeInCache,
+ 'lockdown' => $this->lockdown,
'integration' => $this->integration,
];
@@ -203,6 +206,11 @@ final class ScrapeOptions
return $this->storeInCache;
}
+ public function getLockdown(): ?bool
+ {
+ return $this->lockdown;
+ }
+
public function getIntegration(): ?string
{
return $this->integration;
diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php
index 6dc8d7e42..d3a6b78de 100644
--- a/apps/php-sdk/tests/Unit/ModelsTest.php
+++ b/apps/php-sdk/tests/Unit/ModelsTest.php
@@ -6,6 +6,7 @@ use Firecrawl\Models\CreditUsage;
use Firecrawl\Models\MapData;
use Firecrawl\Models\BatchScrapeJob;
use Firecrawl\Models\CrawlJob;
+use Firecrawl\Models\ScrapeOptions;
it('hydrates CreditUsage from nested data key', function (): void {
$response = [
@@ -88,3 +89,47 @@ it('preserves null creditsUsed in CrawlJob', function (): void {
expect($job->getCreditsUsed())->toBeNull();
});
+
+it('preserves positional integration in ScrapeOptions::with', function (): void {
+ $options = ScrapeOptions::with(
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ false,
+ 'php-sdk',
+ );
+
+ expect($options->getStoreInCache())->toBeFalse();
+ expect($options->getIntegration())->toBe('php-sdk');
+ expect($options->getLockdown())->toBeNull();
+ expect($options->toArray())->toMatchArray([
+ 'storeInCache' => false,
+ 'integration' => 'php-sdk',
+ ]);
+});
+
+it('serializes lockdown in ScrapeOptions', function (): void {
+ $options = ScrapeOptions::with(
+ lockdown: true,
+ integration: 'php-sdk',
+ );
+
+ expect($options->getLockdown())->toBeTrue();
+ expect($options->toArray())->toMatchArray([
+ 'lockdown' => true,
+ 'integration' => 'php-sdk',
+ ]);
+});
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 5aa9ffd73..5d36a0802 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.22.3"
+__version__ = "4.23.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py
index 1ecb54ab5..603ea8f8c 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py
@@ -67,6 +67,7 @@ class TestAsyncScrapeRequestPreparation:
proxy="basic",
max_age=1000,
store_in_cache=False,
+ lockdown=True,
)
payload = await _prepare_scrape_request("https://example.com", opts)
assert payload["url"] == "https://example.com"
@@ -85,6 +86,7 @@ class TestAsyncScrapeRequestPreparation:
assert payload["proxy"] == "basic"
assert payload["maxAge"] == 1000
assert payload["storeInCache"] is False
+ assert payload["lockdown"] is True
@pytest.mark.asyncio
async def test_interact_request_and_response_normalization(self):
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
index 540afb787..ccf24e99f 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
@@ -28,6 +28,7 @@ class TestParseRequestPreparation:
assert payload["origin"].startswith("python-sdk@")
assert "maxAge" not in payload
assert "storeInCache" not in payload
+ assert "lockdown" not in payload
assert "file" in files
filename, file_bytes, mime_type = files["file"]
@@ -64,3 +65,15 @@ class TestParseRequestPreparation:
filename="upload.html",
content_type="text/html",
)
+
+ def test_prepare_parse_request_strips_lockdown(self):
+ options = ParseOptions(formats=["markdown"], lockdown=True)
+ fields, _ = _prepare_parse_request(
+ b"<html><body><h1>Parse</h1></body></html>",
+ options,
+ filename="upload.html",
+ content_type="text/html",
+ )
+
+ payload = json.loads(fields["options"])
+ assert "lockdown" not in payload
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
index 9842cd1a3..615fafb7e 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
@@ -248,13 +248,14 @@ class TestPrepareScrapeOptions:
use_mock="test-mock",
block_ads=False,
store_in_cache=False,
+ lockdown=True,
max_age=7200000, # 2 hours
actions=[screenshot_action],
parsers=["pdf"]
)
-
+
result = prepare_scrape_options(options)
-
+
# Check new field conversions
assert "fastMode" in result
assert result["fastMode"] is True
@@ -264,6 +265,8 @@ class TestPrepareScrapeOptions:
assert result["blockAds"] is False
assert "storeInCache" in result
assert result["storeInCache"] is False
+ assert "lockdown" in result
+ assert result["lockdown"] is True
assert "maxAge" in result
assert result["maxAge"] == 7200000
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index d7d54a2f7..88539b542 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -134,6 +134,7 @@ class FirecrawlClient:
proxy: Optional[str] = None,
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
+ lockdown: Optional[bool] = None,
profile: Optional[Dict[str, Any]] = None,
integration: Optional[str] = None,
) -> Document:
@@ -160,6 +161,7 @@ class FirecrawlClient:
proxy: Proxy to use
max_age: Maximum age of the cache
store_in_cache: Whether to store the result in the cache
+ lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.
profile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})
Returns:
Document
@@ -185,10 +187,11 @@ class FirecrawlClient:
proxy=proxy,
max_age=max_age,
store_in_cache=store_in_cache,
+ lockdown=lockdown,
profile=profile,
integration=integration,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, profile, integration]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, profile, integration]) else None
return scrape_module.scrape(self.http_client, url, options)
def interact(
@@ -811,6 +814,7 @@ class FirecrawlClient:
proxy: Optional[str] = None,
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
+ lockdown: Optional[bool] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -831,7 +835,7 @@ class FirecrawlClient:
timeout: Per-request timeout in milliseconds
wait_for: Wait condition in milliseconds
mobile: Emulate mobile viewport
- parsers: Parser list (e.g., ["pdf"])
+ parsers: Parser list (e.g., ["pdf"])
actions: Browser actions to perform
location: Location settings
skip_tls_verification: Skip TLS verification
@@ -842,6 +846,7 @@ class FirecrawlClient:
proxy: Proxy setting
max_age: Cache max age
store_in_cache: Whether to store results in cache
+ lockdown: Serve only previously cached results; never make outbound requests.
webhook: Webhook configuration
append_to_id: Append to an existing batch job
ignore_invalid_urls: Skip invalid URLs without failing
@@ -874,8 +879,9 @@ class FirecrawlClient:
proxy=proxy,
max_age=max_age,
store_in_cache=store_in_cache,
+ lockdown=lockdown,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else None
return batch_module.start_batch_scrape(
self.http_client,
@@ -1224,6 +1230,7 @@ class FirecrawlClient:
proxy: Optional[str] = None,
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
+ lockdown: Optional[bool] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -1258,8 +1265,9 @@ class FirecrawlClient:
proxy=proxy,
max_age=max_age,
store_in_cache=store_in_cache,
+ lockdown=lockdown,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else None
return batch_module.batch_scrape(
self.http_client,
diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.py
index 8f1508a49..9886a7dbd 100644
--- a/apps/python-sdk/firecrawl/v2/methods/parse.py
+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py
@@ -70,6 +70,7 @@ def _prepare_parse_options_payload(
opts.pop("maxAge", None)
opts.pop("minAge", None)
opts.pop("storeInCache", None)
+ opts.pop("lockdown", None)
request_data.update(opts)
request_data["origin"] = request_data.get("origin") or f"python-sdk@{version}"
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index c27a7ff9f..50776997e 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -545,6 +545,7 @@ class ScrapeOptions(BaseModel):
max_age: Optional[int] = None
min_age: Optional[int] = None
store_in_cache: Optional[bool] = None
+ lockdown: Optional[bool] = None
profile: Optional[Dict[str, Any]] = None
integration: Optional[str] = None
diff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
index 1e1afc2bb..e251b217b 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb
@@ -8,7 +8,7 @@ module Firecrawl
formats headers include_tags exclude_tags only_main_content
timeout wait_for mobile parsers actions location
skip_tls_verification remove_base64_images block_ads proxy
- max_age store_in_cache integration
+ max_age store_in_cache lockdown integration
].freeze
attr_reader(*FIELDS)
@@ -37,6 +37,7 @@ module Firecrawl
"proxy" => proxy,
"maxAge" => max_age,
"storeInCache" => store_in_cache,
+ "lockdown" => lockdown,
"integration" => integration,
}.compact
end
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index b482a3271..79bf04f3c 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.1.0"
+ VERSION = "1.2.0"
end
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index f7209018b..ee65270a9 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.1.0"
+version = "2.2.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 84e51405c..650ed92d8 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -73,6 +73,9 @@ pub struct ScrapeOptions {
/// Store the result in cache for future requests.
pub store_in_cache: Option<bool>,
+ /// Lockdown mode: serve only previously cached results, never make outbound requests.
+ pub lockdown: Option<bool>,
+
/// Persistent browser profile for maintaining state across scrapes.
pub profile: Option<ProfileConfig>,