Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdks): add lockdown scrape option to all SDKs (ENG-4815) (#3409)
Abimael Martell8802eadf8872
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 392ad2ab1..5e8ba9ea5 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -913,12 +913,13 @@ defmodule Firecrawl doremove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],+ lockdown: [type: :boolean, doc: "If true, only previously cached results are served and no outbound request is ever made. If nothing is cached for the URL, a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS is returned. Other options are accepted but ignored in lockdown mode."],timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this scrape. To enable this feature, please contact [email protected]"]])- @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}+ @scrape_and_extract_from_url_key_mapping %{url: "url", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}@doc """Scrape a single URL and optionally extract information using an LLM@@ -977,12 +978,13 @@ defmodule Firecrawl doremove_base64_images: [type: :boolean, doc: "Removes all base 64 images from the markdown output, which may be overwhelmingly long. This does not affect html or rawHtml formats. The image's alt text remains in the output, but the URL is replaced with a placeholder."],skip_tls_verification: [type: :boolean, doc: "Skip TLS certificate verification when making requests."],store_in_cache: [type: :boolean, doc: "If true, the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns. Using some parameters associated with sensitive scraping (e.g. actions, headers) will force this parameter to be false."],+ lockdown: [type: :boolean, doc: "If true, only previously cached results are served and no outbound request is ever made. If nothing is cached for the URL, a 404 with error code SCRAPE_LOCKDOWN_CACHE_MISS is returned. Other options are accepted but ignored in lockdown mode."],timeout: [type: :integer, doc: "Timeout in milliseconds for the request. Minimum is 1000 (1 second). Default is 60000 (60 seconds). Maximum is 300000 (300 seconds)."],wait_for: [type: :integer, doc: "Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load. This waiting time is in addition to Firecrawl's smart wait feature."],zero_data_retention: [type: :boolean, doc: "If true, this will enable zero data retention for this batch scrape. To enable this feature, please contact [email protected]"]])- @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}+ @scrape_and_extract_from_urls_key_mapping %{ignore_invalid_urls: "ignoreInvalidURLs", max_concurrency: "maxConcurrency", urls: "urls", webhook: "webhook", actions: "actions", block_ads: "blockAds", exclude_tags: "excludeTags", formats: "formats", headers: "headers", include_tags: "includeTags", location: "location", max_age: "maxAge", min_age: "minAge", mobile: "mobile", only_main_content: "onlyMainContent", parsers: "parsers", profile: "profile", proxy: "proxy", remove_base64_images: "removeBase64Images", skip_tls_verification: "skipTlsVerification", store_in_cache: "storeInCache", lockdown: "lockdown", timeout: "timeout", wait_for: "waitFor", zero_data_retention: "zeroDataRetention"}@doc """Scrape multiple URLs and optionally extract information using an LLMdiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 174f62fe8..17950d843 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.0.1"+ @version "1.1.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.goindex 93559aafe..a824be667 100644--- a/apps/go-sdk/options.go+++ b/apps/go-sdk/options.go@@ -19,6 +19,7 @@ type ScrapeOptions struct {Proxy *string `json:"proxy,omitempty"`MaxAge *int64 `json:"maxAge,omitempty"`StoreInCache *bool `json:"storeInCache,omitempty"`+ Lockdown *bool `json:"lockdown,omitempty"`Integration *string `json:"integration,omitempty"`JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`}diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex f81359cc4..316fca43c 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.2.2"+version = "1.3.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.javaindex a8560bc76..222704900 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java@@ -32,6 +32,7 @@ public class ScrapeOptions {@JsonProperty("maxAge")private Long maxAge;private Boolean storeInCache;+ private Boolean lockdown;private String integration;private ScrapeOptions() {}@@ -53,6 +54,7 @@ public class ScrapeOptions {public String getProxy() { return proxy; }public Long getMaxAge() { return maxAge; }public Boolean getStoreInCache() { return storeInCache; }+ public Boolean getLockdown() { return lockdown; }public String getIntegration() { return integration; }public static Builder builder() { return new Builder(); }@@ -76,6 +78,7 @@ public class ScrapeOptions {b.proxy = this.proxy;b.maxAge = this.maxAge;b.storeInCache = this.storeInCache;+ b.lockdown = this.lockdown;b.integration = this.integration;return b;}@@ -98,6 +101,7 @@ public class ScrapeOptions {private String proxy;private Long maxAge;private Boolean storeInCache;+ private Boolean lockdown;private String integration;private Builder() {}@@ -157,6 +161,9 @@ public class ScrapeOptions {/** Whether to cache the result. */public Builder storeInCache(Boolean storeInCache) { this.storeInCache = storeInCache; return this; }+ /** Lockdown mode: serve only previously cached results, never make outbound requests. */+ public Builder lockdown(Boolean lockdown) { this.lockdown = lockdown; return this; }+/** Integration identifier. */public Builder integration(String integration) { this.integration = integration; return this; }@@ -179,6 +186,7 @@ public class ScrapeOptions {o.proxy = this.proxy;o.maxAge = this.maxAge;o.storeInCache = this.storeInCache;+ o.lockdown = this.lockdown;o.integration = this.integration;return o;}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 6bdf3619d..55ee202d1 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.18.5",+ "version": "4.19.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.tsindex 22e7540e1..cd0f16a2b 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts@@ -37,4 +37,22 @@ describe("v2.parse unit", () => {),).rejects.toThrow("parse does not support changeTracking format");});++ test("rejects lockdown option before making requests", async () => {+ const client = new FirecrawlClient({+ apiKey: "test-key",+ apiUrl: "https://localhost:3002",+ });++ await expect(+ client.parse(+ {+ data: "<html><body>test</body></html>",+ filename: "upload.html",+ contentType: "text/html",+ },+ { formats: ["markdown"], lockdown: true } as any,+ ),+ ).rejects.toThrow("parse does not support cache/index options");+ });});diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 0dd88048d..51ce6c262 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -180,6 +180,7 @@ export interface ScrapeOptions {maxAge?: number;minAge?: number;storeInCache?: boolean;+ lockdown?: boolean;profile?: {name: string;saveChanges?: boolean;@@ -212,6 +213,7 @@ export type ParseOptions = Omit<| 'maxAge'| 'minAge'| 'storeInCache'+ | 'lockdown'| 'proxy'> & {formats?: ParseFormatOption[];diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.tsindex 5222d0325..afb91a8cc 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts@@ -139,7 +139,12 @@ export function ensureValidParseOptions(options?: ParseOptions): void {if (raw.mobile !== undefined) {throw new Error("parse does not support mobile rendering");}- if (raw.maxAge !== undefined || raw.minAge !== undefined || raw.storeInCache !== undefined) {+ if (+ raw.maxAge !== undefined ||+ raw.minAge !== undefined ||+ raw.storeInCache !== undefined ||+ raw.lockdown !== undefined+ ) {throw new Error("parse does not support cache/index options");}if (raw.proxy !== undefined && raw.proxy !== "basic" && raw.proxy !== "auto") {diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.phpindex b91126530..28153bf57 100644--- a/apps/php-sdk/src/Models/ScrapeOptions.php+++ b/apps/php-sdk/src/Models/ScrapeOptions.php@@ -32,6 +32,7 @@ final class ScrapeOptionsprivate readonly ?string $proxy = null,private readonly ?int $maxAge = null,private readonly ?bool $storeInCache = null,+ private readonly ?bool $lockdown = null,private readonly ?string $integration = null,) {}@@ -62,12 +63,13 @@ final class ScrapeOptions?int $maxAge = null,?bool $storeInCache = null,?string $integration = null,+ ?bool $lockdown = null,): self {return new self($formats, $headers, $includeTags, $excludeTags, $onlyMainContent,$timeout, $waitFor, $mobile, $parsers, $actions, $location,$skipTlsVerification, $removeBase64Images, $blockAds, $proxy,- $maxAge, $storeInCache, $integration,+ $maxAge, $storeInCache, $lockdown, $integration,);}@@ -100,6 +102,7 @@ final class ScrapeOptions'proxy' => $this->proxy,'maxAge' => $this->maxAge,'storeInCache' => $this->storeInCache,+ 'lockdown' => $this->lockdown,'integration' => $this->integration,];@@ -203,6 +206,11 @@ final class ScrapeOptionsreturn $this->storeInCache;}+ public function getLockdown(): ?bool+ {+ return $this->lockdown;+ }+public function getIntegration(): ?string{return $this->integration;diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.phpindex 6dc8d7e42..d3a6b78de 100644--- a/apps/php-sdk/tests/Unit/ModelsTest.php+++ b/apps/php-sdk/tests/Unit/ModelsTest.php@@ -6,6 +6,7 @@ use Firecrawl\Models\CreditUsage;use Firecrawl\Models\MapData;use Firecrawl\Models\BatchScrapeJob;use Firecrawl\Models\CrawlJob;+use Firecrawl\Models\ScrapeOptions;it('hydrates CreditUsage from nested data key', function (): void {$response = [@@ -88,3 +89,47 @@ it('preserves null creditsUsed in CrawlJob', function (): void {expect($job->getCreditsUsed())->toBeNull();});++it('preserves positional integration in ScrapeOptions::with', function (): void {+ $options = ScrapeOptions::with(+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ null,+ false,+ 'php-sdk',+ );++ expect($options->getStoreInCache())->toBeFalse();+ expect($options->getIntegration())->toBe('php-sdk');+ expect($options->getLockdown())->toBeNull();+ expect($options->toArray())->toMatchArray([+ 'storeInCache' => false,+ 'integration' => 'php-sdk',+ ]);+});++it('serializes lockdown in ScrapeOptions', function (): void {+ $options = ScrapeOptions::with(+ lockdown: true,+ integration: 'php-sdk',+ );++ expect($options->getLockdown())->toBeTrue();+ expect($options->toArray())->toMatchArray([+ 'lockdown' => true,+ 'integration' => 'php-sdk',+ ]);+});diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5aa9ffd73..5d36a0802 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.22.3"+__version__ = "4.23.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.pyindex 1ecb54ab5..603ea8f8c 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_scrape_request_preparation.py@@ -67,6 +67,7 @@ class TestAsyncScrapeRequestPreparation:proxy="basic",max_age=1000,store_in_cache=False,+ lockdown=True,)payload = await _prepare_scrape_request("https://example.com", opts)assert payload["url"] == "https://example.com"@@ -85,6 +86,7 @@ class TestAsyncScrapeRequestPreparation:assert payload["proxy"] == "basic"assert payload["maxAge"] == 1000assert payload["storeInCache"] is False+ assert payload["lockdown"] is True@pytest.mark.asyncioasync def test_interact_request_and_response_normalization(self):diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.pyindex 540afb787..ccf24e99f 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py@@ -28,6 +28,7 @@ class TestParseRequestPreparation:assert payload["origin"].startswith("python-sdk@")assert "maxAge" not in payloadassert "storeInCache" not in payload+ assert "lockdown" not in payloadassert "file" in filesfilename, file_bytes, mime_type = files["file"]@@ -64,3 +65,15 @@ class TestParseRequestPreparation:filename="upload.html",content_type="text/html",)++ def test_prepare_parse_request_strips_lockdown(self):+ options = ParseOptions(formats=["markdown"], lockdown=True)+ fields, _ = _prepare_parse_request(+ b"<html><body><h1>Parse</h1></body></html>",+ options,+ filename="upload.html",+ content_type="text/html",+ )++ payload = json.loads(fields["options"])+ assert "lockdown" not in payloaddiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.pyindex 9842cd1a3..615fafb7e 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py@@ -248,13 +248,14 @@ class TestPrepareScrapeOptions:use_mock="test-mock",block_ads=False,store_in_cache=False,+ lockdown=True,max_age=7200000, # 2 hoursactions=[screenshot_action],parsers=["pdf"])-+result = prepare_scrape_options(options)-+# Check new field conversionsassert "fastMode" in resultassert result["fastMode"] is True@@ -264,6 +265,8 @@ class TestPrepareScrapeOptions:assert result["blockAds"] is Falseassert "storeInCache" in resultassert result["storeInCache"] is False+ assert "lockdown" in result+ assert result["lockdown"] is Trueassert "maxAge" in resultassert result["maxAge"] == 7200000diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex d7d54a2f7..88539b542 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -134,6 +134,7 @@ class FirecrawlClient:proxy: Optional[str] = None,max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,+ lockdown: Optional[bool] = None,profile: Optional[Dict[str, Any]] = None,integration: Optional[str] = None,) -> Document:@@ -160,6 +161,7 @@ class FirecrawlClient:proxy: Proxy to usemax_age: Maximum age of the cachestore_in_cache: Whether to store the result in the cache+ lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.profile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})Returns:Document@@ -185,10 +187,11 @@ class FirecrawlClient:proxy=proxy,max_age=max_age,store_in_cache=store_in_cache,+ lockdown=lockdown,profile=profile,integration=integration,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, profile, integration]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, profile, integration]) else Nonereturn scrape_module.scrape(self.http_client, url, options)def interact(@@ -811,6 +814,7 @@ class FirecrawlClient:proxy: Optional[str] = None,max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,+ lockdown: Optional[bool] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,@@ -831,7 +835,7 @@ class FirecrawlClient:timeout: Per-request timeout in millisecondswait_for: Wait condition in millisecondsmobile: Emulate mobile viewport- parsers: Parser list (e.g., ["pdf"])+ parsers: Parser list (e.g., ["pdf"])actions: Browser actions to performlocation: Location settingsskip_tls_verification: Skip TLS verification@@ -842,6 +846,7 @@ class FirecrawlClient:proxy: Proxy settingmax_age: Cache max agestore_in_cache: Whether to store results in cache+ lockdown: Serve only previously cached results; never make outbound requests.webhook: Webhook configurationappend_to_id: Append to an existing batch jobignore_invalid_urls: Skip invalid URLs without failing@@ -874,8 +879,9 @@ class FirecrawlClient:proxy=proxy,max_age=max_age,store_in_cache=store_in_cache,+ lockdown=lockdown,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else Nonereturn batch_module.start_batch_scrape(self.http_client,@@ -1224,6 +1230,7 @@ class FirecrawlClient:proxy: Optional[str] = None,max_age: Optional[int] = None,store_in_cache: Optional[bool] = None,+ lockdown: Optional[bool] = None,webhook: Optional[Union[str, WebhookConfig]] = None,append_to_id: Optional[str] = None,ignore_invalid_urls: Optional[bool] = None,@@ -1258,8 +1265,9 @@ class FirecrawlClient:proxy=proxy,max_age=max_age,store_in_cache=store_in_cache,+ lockdown=lockdown,).items() if v is not None}- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else Nonereturn batch_module.batch_scrape(self.http_client,diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.pyindex 8f1508a49..9886a7dbd 100644--- a/apps/python-sdk/firecrawl/v2/methods/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py@@ -70,6 +70,7 @@ def _prepare_parse_options_payload(opts.pop("maxAge", None)opts.pop("minAge", None)opts.pop("storeInCache", None)+ opts.pop("lockdown", None)request_data.update(opts)request_data["origin"] = request_data.get("origin") or f"python-sdk@{version}"diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex c27a7ff9f..50776997e 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -545,6 +545,7 @@ class ScrapeOptions(BaseModel):max_age: Optional[int] = Nonemin_age: Optional[int] = Nonestore_in_cache: Optional[bool] = None+ lockdown: Optional[bool] = Noneprofile: Optional[Dict[str, Any]] = Noneintegration: Optional[str] = Nonediff --git a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rbindex 1e1afc2bb..e251b217b 100644--- a/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/scrape_options.rb@@ -8,7 +8,7 @@ module Firecrawlformats headers include_tags exclude_tags only_main_contenttimeout wait_for mobile parsers actions locationskip_tls_verification remove_base64_images block_ads proxy- max_age store_in_cache integration+ max_age store_in_cache lockdown integration].freezeattr_reader(*FIELDS)@@ -37,6 +37,7 @@ module Firecrawl"proxy" => proxy,"maxAge" => max_age,"storeInCache" => store_in_cache,+ "lockdown" => lockdown,"integration" => integration,}.compactenddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex b482a3271..79bf04f3c 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.1.0"+ VERSION = "1.2.0"enddiff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex f7209018b..ee65270a9 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.1.0"+version = "2.2.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 84e51405c..650ed92d8 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -73,6 +73,9 @@ pub struct ScrapeOptions {/// Store the result in cache for future requests.pub store_in_cache: Option<bool>,+ /// Lockdown mode: serve only previously cached results, never make outbound requests.+ pub lockdown: Option<bool>,+/// Persistent browser profile for maintaining state across scrapes.pub profile: Option<ProfileConfig>,